From 54c8f0103454da4e870d88fa787351e535bdb0c7 Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 1 Feb 2024 16:05:20 +0800 Subject: [PATCH 01/28] update test case for ts4479 and td28163 by charles --- tests/parallel_test/cases.task | 3 +- tests/system-test/1-insert/test_ts4479.py | 75 ++++++ tests/system-test/2-query/test_td28163.py | 265 ++++++++++++++++++++++ 3 files changed, 342 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/1-insert/test_ts4479.py create mode 100644 tests/system-test/2-query/test_td28163.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 91a0ac46e5..0060841ff5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -350,6 +350,7 @@ fi ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/ts-4272.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4295.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_td27388.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4479.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_timestamp.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py @@ -567,7 +568,7 @@ fi ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4382.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4403.py - +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_td28163.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py diff --git a/tests/system-test/1-insert/test_ts4479.py b/tests/system-test/1-insert/test_ts4479.py new file mode 100644 index 0000000000..be9789b5fc --- /dev/null +++ b/tests/system-test/1-insert/test_ts4479.py @@ -0,0 +1,75 @@ +import os +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf +import taos + + +class TDTestCase: + """Verify inserting varbinary type data of ts-4479 + """ + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.conn = conn + self.db_name = "db" + self.stable_name = "st" + + def run(self): + tdSql.execute("create database if not exists %s" % self.db_name) + tdSql.execute("use %s" % self.db_name) + # create super table + tdSql.execute("create table %s (ts timestamp, c1 varbinary(65517)) tags (t1 varbinary(16382))" % self.stable_name) + + # varbinary tag length is more than 16382 + tag = os.urandom(16383).hex() + tdSql.error("create table ct using st tags(%s);" % ('\\x' + tag)) + + # create child table with max column and tag length + child_table_list = [] + for i in range(2): + child_table_name = "ct_" + str(i+1) + child_table_list.append(child_table_name) + tag = os.urandom(16382).hex() + tdSql.execute("create table %s using st tags('%s');" % (child_table_name, '\\x' + tag)) + tdLog.info("create table %s successfully" % child_table_name) + + # varbinary column length is more than 65517 + value = os.urandom(65518).hex() + tdSql.error("insert into ct_1 values(now, '\\x%s');" % value) + + # insert data + for i in range(10): + sql = "insert into table_name values" + for j in range(5): + value = os.urandom(65517).hex() + sql += "(now+%ss, '%s')," % (str(j+1), '\\x' + value) + for child_table in child_table_list: + tdSql.execute(sql.replace("table_name", child_table)) + tdLog.info("Insert data into %s successfully" % child_table) + tdLog.info("Insert data round %s successfully" % str(i+1)) + tdSql.execute("flush database %s" % self.db_name) + + # insert \\x to varbinary column + tdSql.execute("insert into ct_1 values(now, '\\x');") + tdSql.query("select * from ct_1 where c1 = '\\x';") + tdSql.checkRows(1) + tdSql.checkData(0, 1, b'') + + # insert \\x to varbinary tag + tdSql.execute("create table ct_3 using st tags('\\x');") + tdSql.execute("insert into ct_3 values(now, '\\x45');") + tdSql.query("select * from st where t1='';") + tdSql.checkRows(1) + tdSql.checkData(0, 2, b'') + + def stop(self): + tdSql.execute("drop database if exists %s" % self.db_name) + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/test_td28163.py b/tests/system-test/2-query/test_td28163.py new file mode 100644 index 0000000000..df727f6c5a --- /dev/null +++ b/tests/system-test/2-query/test_td28163.py @@ -0,0 +1,265 @@ +import random +import itertools +from util.log import * +from util.cases import * +from util.sql import * +from util.sqlset import * +from util import constant +from util.common import * + + +class TDTestCase: + """Verify the jira TD-28163 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def prepareData(self): + # db + tdSql.execute("create database if not exists db") + tdSql.execute("use db") + + # super table + tdSql.execute("create stable st(ts timestamp, c_ts_empty timestamp, c_int int, c_int_empty int, c_unsigned_int int unsigned, \ + c_unsigned_int_empty int unsigned, c_bigint bigint, c_bigint_empty bigint, c_unsigned_bigint bigint unsigned, \ + c_unsigned_bigint_empty bigint unsigned, c_float float, c_float_empty float, c_double double, c_double_empty double, \ + c_binary binary(16), c_binary_empty binary(16), c_smallint smallint, c_smallint_empty smallint, \ + c_smallint_unsigned smallint unsigned, c_smallint_unsigned_empty smallint unsigned, c_tinyint tinyint, \ + c_tinyint_empty tinyint, c_tinyint_unsigned tinyint unsigned, c_tinyint_unsigned_empty tinyint unsigned, \ + c_bool bool, c_bool_empty bool, c_nchar nchar(16), c_nchar_empty nchar(16), c_varchar varchar(16), \ + c_varchar_empty varchar(16), c_varbinary varbinary(16), c_varbinary_empty varbinary(16)) \ + tags(t_timestamp timestamp, t_timestamp_empty timestamp, t_int int, t_int_empty int, \ + t_unsigned_int int unsigned, t_unsigned_int_empty int unsigned, t_bigint bigint, t_bigint_empty bigint, \ + t_unsigned_bigint bigint unsigned, t_unsigned_bigint_empty bigint unsigned, t_float float, t_float_empty float, \ + t_double double, t_double_empty double, t_binary binary(16), t_binary_empty binary(16), t_smallint smallint, \ + t_smallint_empty smallint, t_smallint_unsigned smallint unsigned, t_smallint_unsigned_empty smallint unsigned, \ + t_tinyint tinyint, t_tinyint_empty tinyint, t_tinyint_unsigned tinyint unsigned, t_tinyint_unsigned_empty tinyint unsigned, \ + t_bool bool, t_bool_empty bool, t_nchar nchar(16), t_nchar_empty nchar(16), t_varchar varchar(16), \ + t_varchar_empty varchar(16), t_varbinary varbinary(16), t_varbinary_empty varbinary(16));") + + # child tables + start_ts = 1704085200000 + tags = [ + "'2024-01-01 13:00:01', null, 1, null, 1, null, 1111111111111111, null, 1111111111111111, null, 1.1, null, 1.11, null, 'aaaaaaaa', '', 1, null, 1, null, 1, null, 1, null, True, null, 'ncharaa', null, 'varcharaa', null, '0x7661726331', null", + "'2024-01-01 13:00:02', null, 2, null, 2, null, 2222222222222222, null, 2222222222222222, null, 2.2, null, 2.22, null, 'bbbbbbbb', '', 2, null, 2, null, 2, null, 2, null, False, null, 'ncharbb', null, 'varcharbb', null, '0x7661726332', null", + "'2024-01-01 13:00:03', null, 3, null, 3, null, 3333333333333333, null, 3333333333333333, null, 3.3, null, 3.33, null, 'cccccccc', '', 3, null, 3, null, 3, null, 3, null, True, null, 'ncharcc', null, 'varcharcc', null, '0x7661726333', null", + "'2024-01-01 13:00:04', null, 4, null, 4, null, 4444444444444444, null, 4444444444444444, null, 4.4, null, 4.44, null, 'dddddddd', '', 4, null, 4, null, 4, null, 4, null, False, null, 'nchardd', null, 'varchardd', null, '0x7661726334', null", + "'2024-01-01 13:00:05', null, 5, null, 5, null, 5555555555555555, null, 5555555555555555, null, 5.5, null, 5.55, null, 'eeeeeeee', '', 5, null, 5, null, 5, null, 5, null, True, null, 'ncharee', null, 'varcharee', null, '0x7661726335', null", + ] + for i in range(5): + tdSql.execute(f"create table ct{i+1} using st tags({tags[i]});") + + # insert data + data = "null, 1, null, 1, null, 1111111111111111, null, 1111111111111111, null, 1.1, null, 1.11, null, 'aaaaaaaa', null, 1, null, 1, null, 1, null, 1, null, True, null, 'ncharaa', null, 'varcharaa', null, '0x7661726331', null" + for round in range(100): + sql = f"insert into ct{i+1} values" + for j in range(100): + sql += f"({start_ts + (round * 100 + j + 1) * 1000}, {data})" + sql += ";" + tdSql.execute(sql) + tdLog.debug("Prepare data successfully") + + def test_query_with_filter(self): + # total row number + tdSql.query("select count(*) from st;") + total_rows = tdSql.queryResult[0][0] + tdLog.debug("Total row number is %s" % total_rows) + + # start_ts and end_ts + tdSql.query("select first(ts), last(ts) from st;") + start_ts = tdSql.queryResult[0][0] + end_ts = tdSql.queryResult[0][1] + tdLog.debug("start_ts is %s, end_ts is %s" % (start_ts, end_ts)) + + filter_dic = { + "all_filter_list": ["ts <= now", "t_timestamp <= now", f"ts between '{start_ts}' and '{end_ts}'", + f"t_timestamp between '{start_ts}' and '{end_ts}'", "c_ts_empty is null", + "t_timestamp_empty is null", "ts > '1970-01-01 00:00:00'", "t_int in (1, 2, 3, 4, 5)", + "c_int=1", "c_int_empty is null", "c_unsigned_int=1", "c_unsigned_int_empty is null", + "c_unsigned_int in (1, 2, 3, 4, 5)", "c_unsigned_int_empty is null", "c_bigint=1111111111111111", + "c_bigint_empty is null", "c_unsigned_bigint in (1111111111111111)", "c_unsigned_bigint_empty is null", + "c_float=1.1", "c_float_empty is null", "c_double=1.11", "c_double_empty is null", "c_binary='aaaaaaaa'", + "c_binary_empty is null", "c_smallint=1", "c_smallint_empty is null", "c_smallint_unsigned=1", + "c_smallint_unsigned_empty is null", "c_tinyint=1", "c_tinyint_empty is null", "c_tinyint_unsigned=1", + "c_tinyint_unsigned_empty is null", "c_bool=True", "c_bool_empty is null", "c_nchar='ncharaa'", + "c_nchar_empty is null", "c_varchar='varcharaa'", "c_varchar_empty is null", "c_varbinary='0x7661726331'", + "c_varbinary_empty is null"], + "empty_filter_list": ["ts > now", "t_timestamp > now", "c_ts_empty is not null","t_timestamp_empty is not null", + "ts <= '1970-01-01 00:00:00'", "c_ts_empty < '1970-01-01 00:00:00'", "c_int <> 1", "c_int_empty is not null", + "t_int in (10, 11)", "t_int_empty is not null"] + } + for filter in filter_dic["all_filter_list"]: + tdLog.debug("Execute query with filter '%s'" % filter) + tdSql.query(f"select * from st where {filter};") + tdSql.checkRows(total_rows) + + for filter in filter_dic["empty_filter_list"]: + tdLog.debug("Execute query with filter '%s'" % filter) + tdSql.query(f"select * from st where {filter};") + tdSql.checkRows(0) + + def test_query_with_groupby(self): + tdSql.query("select count(*) from st group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(c_unsigned_int_empty + c_int_empty * c_float_empty - c_double_empty + c_smallint_empty / c_tinyint_empty) from st where c_int_empty is null group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 0) + + tdSql.query("select sum(t_unsigned_int_empty + t_int_empty * t_float_empty - t_double_empty + t_smallint_empty / t_tinyint_empty) from st where t_int_empty is null group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + + tdSql.query("select max(c_bigint_empty) from st group by tbname, t_bigint_empty, t_float_empty, t_double_empty;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + + tdSql.query("select min(t_double) as v from st where c_nchar like '%aa%' and t_double is not null group by tbname, t_bigint_empty, t_float_empty, t_double_empty order by v limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1.11) + + tdSql.query("select top(c_float, 1) as v from st where c_nchar like '%aa%' group by tbname order by v desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1.1) + + tdSql.query("select first(ts) from st where c_varchar is not null partition by tbname order by ts slimit 1;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + + tdSql.query("select first(c_nchar_empty) from st group by tbname;") + tdSql.checkRows(0) + + tdSql.query("select first(ts), first(c_nchar_empty) from st group by tbname, ts order by ts slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, None) + + tdSql.query("select first(c_nchar_empty) from st group by t_timestamp_empty order by t_timestamp;") + tdSql.checkRows(0) + + tdSql.query("select last(ts), last(c_nchar_empty) from st group by tbname, ts order by ts slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, None) + + tdSql.query("select elapsed(ts, 1s) t from st where c_int = 1 and c_nchar like '%aa%' group by tbname order by t desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9999) + + tdSql.query("select elapsed(ts, 1s) t from st where c_int_empty is not null and c_nchar like '%aa%' group by tbname order by t desc slimit 1 limit 1;") + tdSql.checkRows(0) + + def test_query_with_join(self): + tdSql.query("select count(*) from st as t1 join st as t2 on t1.ts = t2.ts and t1.c_float_empty is not null;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select count(t1.c_ts_empty) as v from st as t1 join st as t2 on t1.ts = t2.ts and t1.c_float_empty is null order by v desc;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select avg(t1.c_tinyint), sum(t2.c_bigint) from st t1, st t2 where t1.ts=t2.ts and t1.c_int > t2.c_int;") + tdSql.checkRows(0) + + tdSql.query("select avg(t1.c_tinyint), sum(t2.c_bigint) from st t1, st t2 where t1.ts=t2.ts and t1.c_int <= t2.c_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1076616672134475760) + + tdSql.query("select count(t1.c_float_empty) from st t1, st t2 where t1.ts=t2.ts and t1.c_int = t2.c_int and t1.t_int_empty=t2.t_int_empty;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + def test_query_with_window(self): + # time window + tdSql.query("select sum(c_int_empty) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m) fill(value, 10);") + tdSql.checkRows(841) + tdSql.checkData(0, 0, 10) + + tdSql.query("select _wstart, _wend, sum(c_int) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m);") + tdSql.checkRows(65) + + # status window + tdSql.error("select _wstart, count(*) from st state_window(t_bool);") + tdSql.query("select _wstart, count(*) from st partition by tbname state_window(c_bool);") + tdSql.checkRows(5) + tdSql.checkData(0, 1, 10000) + + # session window + tdSql.query("select _wstart, count(*) from st partition by tbname, t_int session(ts, 1m);") + tdSql.checkRows(5) + tdSql.checkData(0, 1, 10000) + + # event window + tdSql.query("select _wstart, _wend, count(*) from (select * from st order by ts, tbname) event_window start with t_bool=true end with t_bool=false;") + tdSql.checkRows(20000) + + def test_query_with_union(self): + tdSql.query("select count(ts) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(ts) from (select * from ct1 union all select * from ct2 union all select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 30000) + + tdSql.query("select count(*) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(c_ts_empty) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select count(*) from (select ts from st union select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10001) + + tdSql.query("select count(*) from (select ts from st union all select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 100000) + + tdSql.query("select count(ts) from (select ts from st union select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(ts) from (select ts from st union all select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 50000) + + def test_nested_query(self): + tdSql.query("select elapsed(ts, 1s) from (select * from (select * from st where c_int = 1) where c_int_empty is null);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9999) + + tdSql.query("select first(ts) as t, avg(c_int) as v from (select * from (select * from st where c_int = 1) where c_int_empty is null) group by t_timestamp order by t_timestamp desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, 1) + + tdSql.query("select max(c_tinyint) from (select c_tinyint, tbname from st where c_float_empty is null or t_int_empty is null) group by tbname order by c_tinyint desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select top(c_int, 3) from (select c_int, tbname from st where t_int in (2, 3)) group by tbname slimit 3;") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 1) + + def run(self): + self.prepareData() + self.test_query_with_filter() + self.test_query_with_groupby() + self.test_query_with_join() + self.test_query_with_window() + self.test_query_with_union() + self.test_nested_query() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 4d8ba0f60f324d82b84746d4f974d28429b5e0f2 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 1 Feb 2024 17:02:08 +0800 Subject: [PATCH 02/28] feat: support uniq grant --- include/common/tgrant.h | 2 +- source/util/src/tbase64.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index dbca2ac90c..688fac858e 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -69,7 +69,7 @@ int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, ch {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "state", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ + {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ diff --git a/source/util/src/tbase64.c b/source/util/src/tbase64.c index f6f12fef97..a2f4ddbc51 100644 --- a/source/util/src/tbase64.c +++ b/source/util/src/tbase64.c @@ -15,8 +15,6 @@ #define _DEFAULT_SOURCE #include "tbase64.h" -#include -#include static char basis_64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; From adee720128159c1f3236f28959d81c4e432bfb27 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 1 Feb 2024 17:48:29 +0800 Subject: [PATCH 03/28] feat: support uniq grant --- include/common/tgrant.h | 3 +- include/common/tmsg.h | 8 +++++ include/libs/catalog/catalog.h | 4 +++ source/client/src/clientHb.c | 40 +++++++++++++++++++++++ source/common/src/tgrant.c | 1 + source/common/src/tmsg.c | 29 ++++++++++++++++ source/dnode/mnode/impl/inc/mndGrant.h | 6 ++-- source/dnode/mnode/impl/src/mndConsumer.c | 7 ++-- source/dnode/mnode/impl/src/mndProfile.c | 11 +++++++ source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndStreamHb.c | 2 +- source/libs/parser/src/parInsertSql.c | 12 +++++-- 12 files changed, 114 insertions(+), 11 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 688fac858e..9f7572ed63 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -54,7 +54,8 @@ typedef enum { TSDB_GRANT_BACKUP_RESTORE, } EGrantType; -int32_t grantCheck(EGrantType grant); +int32_t grantCheck(EGrantType grant); // less +int32_t grantCheckLE(EGrantType grant); // less or equal char* tGetMachineId(); #ifndef TD_UNIQ_GRANT int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 04502eb64a..9b493c4a40 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -110,6 +110,7 @@ enum { HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_DYN_VIEW, HEARTBEAT_KEY_VIEWINFO, + HEARTBEAT_KEY_GRANT, }; typedef enum _mgmt_table { @@ -1885,6 +1886,13 @@ int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); void tFreeSViewHbRsp(SViewHbRsp* pRsp); +typedef struct { + uint32_t flags; +} SGrantHbRsp; + +int32_t tSerializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); +int32_t tDeserializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); + typedef struct { int32_t numOfTables; int32_t numOfVgroup; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index b34b998d76..3649f369b7 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -145,6 +145,10 @@ typedef struct SSTableVersion { int32_t smaVer; } SSTableVersion; +typedef struct SGrantVersion { + int32_t version; +} SGrantVersion; + typedef struct SDbCacheInfo { char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 63a65d7c95..f4c2fbec5a 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,6 +327,37 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } +#if 0 +static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t code = 0; + + SGrantHbRsp hbRsp = {0}; + if (tDeserializeSGrantHbRsp(value, valueLen, &hbRsp) != 0) { + taosArrayDestroyEx(hbRsp.pViewRsp, hbFreeSViewMetaInRsp); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t numOfMeta = taosArrayGetSize(hbRsp.pViewRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + SViewMetaRsp *rsp = taosArrayGetP(hbRsp.pViewRsp, i); + + if (rsp->numOfCols < 0) { + tscDebug("hb to remove view, db:%s, view:%s", rsp->dbFName, rsp->name); + catalogRemoveViewMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->name, rsp->viewId); + tFreeSViewMetaRsp(rsp); + taosMemoryFreeClear(rsp); + } else { + tscDebug("hb to update view, db:%s, view:%s", rsp->dbFName, rsp->name); + catalogUpdateViewMeta(pCatalog, rsp); + } + } + + taosArrayDestroy(hbRsp.pViewRsp); + return TSDB_CODE_SUCCESS; +} +#endif + static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { @@ -378,6 +409,15 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p hbProcessViewInfoRsp(kv->value, kv->valueLen, pCatalog); break; } + case HEARTBEAT_KEY_GRANT: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + // hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); + break; + } #endif default: tscError("invalid hb key type:%d", kv->key); diff --git a/source/common/src/tgrant.c b/source/common/src/tgrant.c index f212d71362..8e4fe9febb 100644 --- a/source/common/src/tgrant.c +++ b/source/common/src/tgrant.c @@ -19,5 +19,6 @@ #ifndef _GRANT int32_t grantCheck(EGrantType grant) {return TSDB_CODE_SUCCESS;} +int32_t grantCheckLE(EGrantType grant) {return TSDB_CODE_SUCCESS;} #endif \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index de505ab3d2..c866aae209 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9303,3 +9303,32 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) { taosArrayDestroy(pRsp->pViewRsp); } + +int32_t tSerializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeU32v(&encoder, pRsp->flags) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeU32v(&decoder, &pRsp->flags) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index 82b3260860..a012d7a8f5 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -41,9 +41,9 @@ char **mergeActive); #endif - int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); - int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); - + int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); + int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); + int32_t mndValidateGrant(SMnode * pMnode, SGrantVersion * pGrantVersion, void **ppRsp, int32_t *pRspLen); int32_t mndGrantGetLastState(SMnode * pMnode, SGrantState * pState); SGrantLogObj *mndAcquireGrant(SMnode * pMnode, void **ppIter); void mndReleaseGrant(SMnode * pMnode, SGrantLogObj * pGrant, void *pIter); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 3ce548a4f6..753076f1f3 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -107,7 +107,7 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode * goto FAILED; } - if ((terrno = grantCheck(TSDB_GRANT_SUBSCRIPTION)) < 0) { + if ((terrno = grantCheckLE(TSDB_GRANT_SUBSCRIPTION)) < 0) { code = terrno; goto FAILED; } @@ -240,9 +240,10 @@ static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbR } STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1); strcpy(data->topic, topic); - if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || grantCheck(TSDB_GRANT_SUBSCRIPTION) < 0) { + if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || + grantCheckLE(TSDB_GRANT_SUBSCRIPTION) < 0) { data->noPrivilege = 1; - } else{ + } else { data->noPrivilege = 0; } mndReleaseTopic(pMnode, pTopic); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e0559b4c48..d7e6d9609c 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -18,6 +18,7 @@ #include "audit.h" #include "mndDb.h" #include "mndDnode.h" +#include "mndGrant.h" #include "mndMnode.h" #include "mndPrivilege.h" #include "mndQnode.h" @@ -605,6 +606,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } + case HEARTBEAT_KEY_GRANT: { + void *rspMsg = NULL; + int32_t rspLen = 0; + mndValidateGrant(pMnode, kv->value, &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv1 = {.key = HEARTBEAT_KEY_GRANT, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); + } + break; + } #endif default: mError("invalid kv key:%d", kv->key); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5f5fbdba13..530b3d5bcd 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1628,7 +1628,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; - if(grantCheck(TSDB_GRANT_STREAMS) < 0){ + if(grantCheckLE(TSDB_GRANT_STREAMS) < 0){ terrno = TSDB_CODE_GRANT_EXPIRED; return -1; } diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 4426ab0672..005caea31b 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -225,7 +225,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SArray *pFailedTasks = taosArrayInit(4, sizeof(SFailedCheckpointInfo)); SArray *pOrphanTasks = taosArrayInit(3, sizeof(SOrphanTask)); - if(grantCheck(TSDB_GRANT_STREAMS) < 0){ + if(grantCheckLE(TSDB_GRANT_STREAMS) < 0){ if(suspendAllStreams(pMnode, &pReq->info) < 0){ return -1; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 512dfdaef2..8e32eca28e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2193,6 +2193,10 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); } + // if ((terrno = grantCheck(TSDB_GRANT_CSV)) < 0) { + // return buildInvalidOperationMsg(&pCxt->msg, terrstr()); + // } + NEXT_TOKEN(pStmt->pSql, *pToken); if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); @@ -2754,11 +2758,15 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOp } static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SRowsDataContext rowsDataCxt; + // if ((code = grantCheck(TSDB_GRANT_CSV)) < 0) { + // return code; + // } + if (!pStmt->stbSyntax) { - STableDataCxt* pTableCxt = NULL; + STableDataCxt* pTableCxt = NULL; code = getTableDataCxt(pCxt, pStmt, &pTableCxt); rowsDataCxt.pTableDataCxt = pTableCxt; } else { From 44d264456d951d1ce28770569bd5196cf44f59f3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 08:41:45 +0800 Subject: [PATCH 04/28] feat: support uniq grant --- include/libs/catalog/catalog.h | 1 + source/client/src/clientHb.c | 33 +++---- source/libs/catalog/inc/catalogInt.h | 12 +++ source/libs/catalog/src/catalog.c | 15 +++ source/libs/catalog/src/ctgCache.c | 131 +++++++++++++++++++++++++++ 5 files changed, 170 insertions(+), 22 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 3649f369b7..2e013413a3 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -383,6 +383,7 @@ int32_t catalogRemoveViewMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId int32_t catalogUpdateDynViewVer(SCatalog* pCtg, SDynViewVersion* pVer); int32_t catalogUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); +int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg); int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index f4c2fbec5a..755ab0eb91 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,37 +327,26 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } -#if 0 static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; - SGrantHbRsp hbRsp = {0}; - if (tDeserializeSGrantHbRsp(value, valueLen, &hbRsp) != 0) { - taosArrayDestroyEx(hbRsp.pViewRsp, hbFreeSViewMetaInRsp); + SGrantHbRsp *hbRsp = taosMemoryCalloc(1, sizeof(SGrantHbRsp)); + if (!hbRsp) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (tDeserializeSGrantHbRsp(value, valueLen, hbRsp) != 0) { + taosMemoryFree(hbRsp); terrno = TSDB_CODE_INVALID_MSG; return -1; } - int32_t numOfMeta = taosArrayGetSize(hbRsp.pViewRsp); - for (int32_t i = 0; i < numOfMeta; ++i) { - SViewMetaRsp *rsp = taosArrayGetP(hbRsp.pViewRsp, i); + tscInfo("hb to update grant info:%u", hbRsp->flags); + catalogUpdateGrantInfo(pCatalog, hbRsp); - if (rsp->numOfCols < 0) { - tscDebug("hb to remove view, db:%s, view:%s", rsp->dbFName, rsp->name); - catalogRemoveViewMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->name, rsp->viewId); - tFreeSViewMetaRsp(rsp); - taosMemoryFreeClear(rsp); - } else { - tscDebug("hb to update view, db:%s, view:%s", rsp->dbFName, rsp->name); - catalogUpdateViewMeta(pCatalog, rsp); - } - } - - taosArrayDestroy(hbRsp.pViewRsp); return TSDB_CODE_SUCCESS; } -#endif - static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { @@ -415,7 +404,7 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p break; } - // hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); + hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); break; } #endif diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index f9f4ee7dfc..b9a0752178 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -69,6 +69,7 @@ typedef enum { CTG_CI_UDF, CTG_CI_SVR_VER, CTG_CI_VIEW, + CTG_CI_GRANT_INFO, CTG_CI_MAX_VALUE, } CTG_CACHE_ITEM; @@ -101,6 +102,8 @@ enum { CTG_OP_DROP_TB_INDEX, CTG_OP_UPDATE_VIEW_META, CTG_OP_DROP_VIEW_META, + CTG_OP_UPDATE_GRANT_INFO, + CTG_OP_DROP_GRANT_INFO, CTG_OP_CLEAR_CACHE, CTG_OP_MAX }; @@ -123,6 +126,7 @@ typedef enum { CTG_TASK_GET_TB_HASH_BATCH, CTG_TASK_GET_TB_TAG, CTG_TASK_GET_VIEW, + CTG_TASK_GET_GRANT_INFO, } CTG_TASK_TYPE; typedef enum { @@ -322,9 +326,12 @@ typedef struct SCatalog { SDynViewVersion dynViewVer; SHashObj* userCache; // key:user, value:SCtgUserAuth SHashObj* dbCache; // key:dbname, value:SCtgDBCache + SHashObj* grantCache; + SGrantHbRsp _grantCache; SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; + SCtgRentMgmt grantRent; SCtgCacheStat cacheStat; } SCatalog; @@ -548,6 +555,10 @@ typedef struct SCtgDropViewMetaMsg { uint64_t viewId; } SCtgDropViewMetaMsg; +typedef struct SCtgUpdateGrantInfoMsg { + SCatalog* pCtg; + SGrantHbRsp* pRsp; +} SCtgUpdateGrantInfoMsg; typedef struct SCtgCacheOperation { int32_t opId; @@ -948,6 +959,7 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, SCtgViewCache *pCache); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); +int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask); void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache* dbCache); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index c2d88e5ce3..f52787a61e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1747,6 +1747,21 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pMsg) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgUpdateGrantInfoToCache(pCtg, pMsg, true)); + +_return: + + CTG_API_LEAVE(code); +} + int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 1b693b4e07..05078fabb4 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -32,6 +32,8 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update v {CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex}, {CTG_OP_UPDATE_VIEW_META, "update viewMeta", ctgOpUpdateViewMeta}, {CTG_OP_DROP_VIEW_META, "drop viewMeta", ctgOpDropViewMeta}, + {CTG_OP_UPDATE_GRANT_INFO, "update grantInfo", ctgOpUpdateGrantInfo}, + {CTG_OP_DROP_GRANT_INFO, "drop grantInfo", ctgOpDropGrantInfo}, {CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}}; SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = { @@ -1323,6 +1325,33 @@ _return: CTG_RET(code); } +int32_t ctgUpdateGrantInfoEnqueue(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_UPDATE_GRANT_INFO; + op->syncOp = syncOp; + + SCtgUpdateGrantInfoMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateGrantInfoMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateGrantInfoMsg)); + taosMemoryFree(op); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + msg->pCtg = pCtg; + msg->pRsp = pRsp; + + op->data = msg; + + CTG_ERR_RET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; +_return: + + taosMemoryFree(pRsp); + CTG_RET(code); +} + int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *viewName, uint64_t viewId, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); @@ -1715,6 +1744,10 @@ int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncRe CTG_RET(ctgUpdateViewMetaEnqueue(pCtg, pRsp, syncReq)); } +int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq) { + CTG_RET(ctgUpdateGrantInfoEnqueue(pCtg, pRsp, syncReq)); +} + void ctgClearAllHandles(void) { SCatalog *pCtg = NULL; @@ -2450,6 +2483,104 @@ _return: CTG_RET(code); } +int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgUpdateGrantInfoMsg *msg = operation->data; + SCatalog *pCtg = msg->pCtg; + SGrantHbRsp *pRsp = msg->pRsp; + SCtgDBCache *dbCache = NULL; + SViewMeta *pMeta = NULL; + + taosMemoryFreeClear(msg); + + if (pCtg->stopUpdate) { + goto _return; + } + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pRsp->dbFName, pRsp->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, pRsp->dbFName, pRsp->dbId); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + pMeta = taosMemoryCalloc(1, sizeof(SViewMeta)); + if (NULL == pMeta) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + CTG_ERR_JRET(dupViewMetaFromRsp(pRsp, pMeta)); + + code = ctgWriteViewMetaToCache(pCtg, dbCache, pRsp->dbFName, pRsp->name, pMeta); + pMeta = NULL; + +_return: + + tFreeSViewMetaRsp(pRsp); + taosMemoryFree(pRsp); + ctgFreeSViewMeta(pMeta); + taosMemoryFree(pMeta); + + CTG_RET(code); +} + +int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgDropViewMetaMsg *msg = operation->data; + SCatalog *pCtg = msg->pCtg; + int32_t tblType = 0; + + if (pCtg->stopUpdate) { + goto _return; + } + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + goto _return; + } + + if ((0 != msg->dbId) && (dbCache->dbId != msg->dbId)) { + ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", dbFName:%s, viewName:%s", msg->dbId, dbCache->dbId, + msg->dbFName, msg->viewName); + goto _return; + } + + SCtgViewCache *pViewCache = taosHashGet(dbCache->viewCache, msg->viewName, strlen(msg->viewName)); + if (NULL == pViewCache) { + ctgDebug("view %s already not in cache", msg->viewName); + goto _return; + } + + int64_t viewId = pViewCache->pMeta->viewId; + if (0 != msg->viewId && viewId != msg->viewId) { + ctgDebug("viewId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", viewName:%s", msg->viewId, viewId, msg->viewName); + goto _return; + } + + atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pViewCache->pMeta)); + ctgFreeViewCacheImpl(pViewCache, true); + + if (taosHashRemove(dbCache->viewCache, msg->viewName, strlen(msg->viewName))) { + ctgError("view %s not exist in cache, dbFName:%s", msg->viewName, msg->dbFName); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } else { + atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(SCtgViewCache) + strlen(msg->viewName)); + CTG_DB_NUM_DEC(CTG_CI_VIEW); + } + + ctgDebug("view %s removed from cache, dbFName:%s", msg->viewName, msg->dbFName); + + CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->viewRent, viewId, ctgViewVersionSortCompare, ctgViewVersionSearchCompare)); + + ctgDebug("view %s removed from rent, dbFName:%s, viewId:0x%" PRIx64, msg->viewName, msg->dbFName, viewId); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; From 57148a7731038e3497d3a4577f74d86fb17c04f3 Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 1 Feb 2024 16:05:20 +0800 Subject: [PATCH 05/28] udpate checkAsan.sh to skip asan_malloc_linux.cpp error --- tests/parallel_test/cases.task | 3 +- tests/script/sh/checkAsan.sh | 2 +- tests/system-test/1-insert/test_ts4479.py | 75 ++++++ tests/system-test/2-query/test_td28163.py | 265 ++++++++++++++++++++++ 4 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 tests/system-test/1-insert/test_ts4479.py create mode 100644 tests/system-test/2-query/test_td28163.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 91a0ac46e5..0060841ff5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -350,6 +350,7 @@ fi ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/ts-4272.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4295.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_td27388.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4479.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_timestamp.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py @@ -567,7 +568,7 @@ fi ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4382.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4403.py - +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_td28163.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 9f67d437e2..2c1ecd650f 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -72,7 +72,7 @@ python_error=$(cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l) #0 0x7f2d64f5a808 in __interceptor_malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cc:144 #1 0x7f2d63fcf459 in strerror /build/glibc-SzIz7B/glibc-2.31/string/strerror.c:38 -runtime_error=$(cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | grep -v "signed integer overflow" | grep -v "strerror.c" | grep -v "asan_malloc_linux.cc" | grep -v "strerror.c" | wc -l) +runtime_error=$(cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | grep -v "signed integer overflow" | grep -v "strerror.c" | grep -v "asan_malloc_linux.cc" | grep -v "strerror.c" | grep -v "asan_malloc_linux.cpp" | wc -l) echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" diff --git a/tests/system-test/1-insert/test_ts4479.py b/tests/system-test/1-insert/test_ts4479.py new file mode 100644 index 0000000000..be9789b5fc --- /dev/null +++ b/tests/system-test/1-insert/test_ts4479.py @@ -0,0 +1,75 @@ +import os +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf +import taos + + +class TDTestCase: + """Verify inserting varbinary type data of ts-4479 + """ + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.conn = conn + self.db_name = "db" + self.stable_name = "st" + + def run(self): + tdSql.execute("create database if not exists %s" % self.db_name) + tdSql.execute("use %s" % self.db_name) + # create super table + tdSql.execute("create table %s (ts timestamp, c1 varbinary(65517)) tags (t1 varbinary(16382))" % self.stable_name) + + # varbinary tag length is more than 16382 + tag = os.urandom(16383).hex() + tdSql.error("create table ct using st tags(%s);" % ('\\x' + tag)) + + # create child table with max column and tag length + child_table_list = [] + for i in range(2): + child_table_name = "ct_" + str(i+1) + child_table_list.append(child_table_name) + tag = os.urandom(16382).hex() + tdSql.execute("create table %s using st tags('%s');" % (child_table_name, '\\x' + tag)) + tdLog.info("create table %s successfully" % child_table_name) + + # varbinary column length is more than 65517 + value = os.urandom(65518).hex() + tdSql.error("insert into ct_1 values(now, '\\x%s');" % value) + + # insert data + for i in range(10): + sql = "insert into table_name values" + for j in range(5): + value = os.urandom(65517).hex() + sql += "(now+%ss, '%s')," % (str(j+1), '\\x' + value) + for child_table in child_table_list: + tdSql.execute(sql.replace("table_name", child_table)) + tdLog.info("Insert data into %s successfully" % child_table) + tdLog.info("Insert data round %s successfully" % str(i+1)) + tdSql.execute("flush database %s" % self.db_name) + + # insert \\x to varbinary column + tdSql.execute("insert into ct_1 values(now, '\\x');") + tdSql.query("select * from ct_1 where c1 = '\\x';") + tdSql.checkRows(1) + tdSql.checkData(0, 1, b'') + + # insert \\x to varbinary tag + tdSql.execute("create table ct_3 using st tags('\\x');") + tdSql.execute("insert into ct_3 values(now, '\\x45');") + tdSql.query("select * from st where t1='';") + tdSql.checkRows(1) + tdSql.checkData(0, 2, b'') + + def stop(self): + tdSql.execute("drop database if exists %s" % self.db_name) + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/test_td28163.py b/tests/system-test/2-query/test_td28163.py new file mode 100644 index 0000000000..df727f6c5a --- /dev/null +++ b/tests/system-test/2-query/test_td28163.py @@ -0,0 +1,265 @@ +import random +import itertools +from util.log import * +from util.cases import * +from util.sql import * +from util.sqlset import * +from util import constant +from util.common import * + + +class TDTestCase: + """Verify the jira TD-28163 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def prepareData(self): + # db + tdSql.execute("create database if not exists db") + tdSql.execute("use db") + + # super table + tdSql.execute("create stable st(ts timestamp, c_ts_empty timestamp, c_int int, c_int_empty int, c_unsigned_int int unsigned, \ + c_unsigned_int_empty int unsigned, c_bigint bigint, c_bigint_empty bigint, c_unsigned_bigint bigint unsigned, \ + c_unsigned_bigint_empty bigint unsigned, c_float float, c_float_empty float, c_double double, c_double_empty double, \ + c_binary binary(16), c_binary_empty binary(16), c_smallint smallint, c_smallint_empty smallint, \ + c_smallint_unsigned smallint unsigned, c_smallint_unsigned_empty smallint unsigned, c_tinyint tinyint, \ + c_tinyint_empty tinyint, c_tinyint_unsigned tinyint unsigned, c_tinyint_unsigned_empty tinyint unsigned, \ + c_bool bool, c_bool_empty bool, c_nchar nchar(16), c_nchar_empty nchar(16), c_varchar varchar(16), \ + c_varchar_empty varchar(16), c_varbinary varbinary(16), c_varbinary_empty varbinary(16)) \ + tags(t_timestamp timestamp, t_timestamp_empty timestamp, t_int int, t_int_empty int, \ + t_unsigned_int int unsigned, t_unsigned_int_empty int unsigned, t_bigint bigint, t_bigint_empty bigint, \ + t_unsigned_bigint bigint unsigned, t_unsigned_bigint_empty bigint unsigned, t_float float, t_float_empty float, \ + t_double double, t_double_empty double, t_binary binary(16), t_binary_empty binary(16), t_smallint smallint, \ + t_smallint_empty smallint, t_smallint_unsigned smallint unsigned, t_smallint_unsigned_empty smallint unsigned, \ + t_tinyint tinyint, t_tinyint_empty tinyint, t_tinyint_unsigned tinyint unsigned, t_tinyint_unsigned_empty tinyint unsigned, \ + t_bool bool, t_bool_empty bool, t_nchar nchar(16), t_nchar_empty nchar(16), t_varchar varchar(16), \ + t_varchar_empty varchar(16), t_varbinary varbinary(16), t_varbinary_empty varbinary(16));") + + # child tables + start_ts = 1704085200000 + tags = [ + "'2024-01-01 13:00:01', null, 1, null, 1, null, 1111111111111111, null, 1111111111111111, null, 1.1, null, 1.11, null, 'aaaaaaaa', '', 1, null, 1, null, 1, null, 1, null, True, null, 'ncharaa', null, 'varcharaa', null, '0x7661726331', null", + "'2024-01-01 13:00:02', null, 2, null, 2, null, 2222222222222222, null, 2222222222222222, null, 2.2, null, 2.22, null, 'bbbbbbbb', '', 2, null, 2, null, 2, null, 2, null, False, null, 'ncharbb', null, 'varcharbb', null, '0x7661726332', null", + "'2024-01-01 13:00:03', null, 3, null, 3, null, 3333333333333333, null, 3333333333333333, null, 3.3, null, 3.33, null, 'cccccccc', '', 3, null, 3, null, 3, null, 3, null, True, null, 'ncharcc', null, 'varcharcc', null, '0x7661726333', null", + "'2024-01-01 13:00:04', null, 4, null, 4, null, 4444444444444444, null, 4444444444444444, null, 4.4, null, 4.44, null, 'dddddddd', '', 4, null, 4, null, 4, null, 4, null, False, null, 'nchardd', null, 'varchardd', null, '0x7661726334', null", + "'2024-01-01 13:00:05', null, 5, null, 5, null, 5555555555555555, null, 5555555555555555, null, 5.5, null, 5.55, null, 'eeeeeeee', '', 5, null, 5, null, 5, null, 5, null, True, null, 'ncharee', null, 'varcharee', null, '0x7661726335', null", + ] + for i in range(5): + tdSql.execute(f"create table ct{i+1} using st tags({tags[i]});") + + # insert data + data = "null, 1, null, 1, null, 1111111111111111, null, 1111111111111111, null, 1.1, null, 1.11, null, 'aaaaaaaa', null, 1, null, 1, null, 1, null, 1, null, True, null, 'ncharaa', null, 'varcharaa', null, '0x7661726331', null" + for round in range(100): + sql = f"insert into ct{i+1} values" + for j in range(100): + sql += f"({start_ts + (round * 100 + j + 1) * 1000}, {data})" + sql += ";" + tdSql.execute(sql) + tdLog.debug("Prepare data successfully") + + def test_query_with_filter(self): + # total row number + tdSql.query("select count(*) from st;") + total_rows = tdSql.queryResult[0][0] + tdLog.debug("Total row number is %s" % total_rows) + + # start_ts and end_ts + tdSql.query("select first(ts), last(ts) from st;") + start_ts = tdSql.queryResult[0][0] + end_ts = tdSql.queryResult[0][1] + tdLog.debug("start_ts is %s, end_ts is %s" % (start_ts, end_ts)) + + filter_dic = { + "all_filter_list": ["ts <= now", "t_timestamp <= now", f"ts between '{start_ts}' and '{end_ts}'", + f"t_timestamp between '{start_ts}' and '{end_ts}'", "c_ts_empty is null", + "t_timestamp_empty is null", "ts > '1970-01-01 00:00:00'", "t_int in (1, 2, 3, 4, 5)", + "c_int=1", "c_int_empty is null", "c_unsigned_int=1", "c_unsigned_int_empty is null", + "c_unsigned_int in (1, 2, 3, 4, 5)", "c_unsigned_int_empty is null", "c_bigint=1111111111111111", + "c_bigint_empty is null", "c_unsigned_bigint in (1111111111111111)", "c_unsigned_bigint_empty is null", + "c_float=1.1", "c_float_empty is null", "c_double=1.11", "c_double_empty is null", "c_binary='aaaaaaaa'", + "c_binary_empty is null", "c_smallint=1", "c_smallint_empty is null", "c_smallint_unsigned=1", + "c_smallint_unsigned_empty is null", "c_tinyint=1", "c_tinyint_empty is null", "c_tinyint_unsigned=1", + "c_tinyint_unsigned_empty is null", "c_bool=True", "c_bool_empty is null", "c_nchar='ncharaa'", + "c_nchar_empty is null", "c_varchar='varcharaa'", "c_varchar_empty is null", "c_varbinary='0x7661726331'", + "c_varbinary_empty is null"], + "empty_filter_list": ["ts > now", "t_timestamp > now", "c_ts_empty is not null","t_timestamp_empty is not null", + "ts <= '1970-01-01 00:00:00'", "c_ts_empty < '1970-01-01 00:00:00'", "c_int <> 1", "c_int_empty is not null", + "t_int in (10, 11)", "t_int_empty is not null"] + } + for filter in filter_dic["all_filter_list"]: + tdLog.debug("Execute query with filter '%s'" % filter) + tdSql.query(f"select * from st where {filter};") + tdSql.checkRows(total_rows) + + for filter in filter_dic["empty_filter_list"]: + tdLog.debug("Execute query with filter '%s'" % filter) + tdSql.query(f"select * from st where {filter};") + tdSql.checkRows(0) + + def test_query_with_groupby(self): + tdSql.query("select count(*) from st group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(c_unsigned_int_empty + c_int_empty * c_float_empty - c_double_empty + c_smallint_empty / c_tinyint_empty) from st where c_int_empty is null group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 0) + + tdSql.query("select sum(t_unsigned_int_empty + t_int_empty * t_float_empty - t_double_empty + t_smallint_empty / t_tinyint_empty) from st where t_int_empty is null group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + + tdSql.query("select max(c_bigint_empty) from st group by tbname, t_bigint_empty, t_float_empty, t_double_empty;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + + tdSql.query("select min(t_double) as v from st where c_nchar like '%aa%' and t_double is not null group by tbname, t_bigint_empty, t_float_empty, t_double_empty order by v limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1.11) + + tdSql.query("select top(c_float, 1) as v from st where c_nchar like '%aa%' group by tbname order by v desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1.1) + + tdSql.query("select first(ts) from st where c_varchar is not null partition by tbname order by ts slimit 1;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + + tdSql.query("select first(c_nchar_empty) from st group by tbname;") + tdSql.checkRows(0) + + tdSql.query("select first(ts), first(c_nchar_empty) from st group by tbname, ts order by ts slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, None) + + tdSql.query("select first(c_nchar_empty) from st group by t_timestamp_empty order by t_timestamp;") + tdSql.checkRows(0) + + tdSql.query("select last(ts), last(c_nchar_empty) from st group by tbname, ts order by ts slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, None) + + tdSql.query("select elapsed(ts, 1s) t from st where c_int = 1 and c_nchar like '%aa%' group by tbname order by t desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9999) + + tdSql.query("select elapsed(ts, 1s) t from st where c_int_empty is not null and c_nchar like '%aa%' group by tbname order by t desc slimit 1 limit 1;") + tdSql.checkRows(0) + + def test_query_with_join(self): + tdSql.query("select count(*) from st as t1 join st as t2 on t1.ts = t2.ts and t1.c_float_empty is not null;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select count(t1.c_ts_empty) as v from st as t1 join st as t2 on t1.ts = t2.ts and t1.c_float_empty is null order by v desc;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select avg(t1.c_tinyint), sum(t2.c_bigint) from st t1, st t2 where t1.ts=t2.ts and t1.c_int > t2.c_int;") + tdSql.checkRows(0) + + tdSql.query("select avg(t1.c_tinyint), sum(t2.c_bigint) from st t1, st t2 where t1.ts=t2.ts and t1.c_int <= t2.c_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1076616672134475760) + + tdSql.query("select count(t1.c_float_empty) from st t1, st t2 where t1.ts=t2.ts and t1.c_int = t2.c_int and t1.t_int_empty=t2.t_int_empty;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + def test_query_with_window(self): + # time window + tdSql.query("select sum(c_int_empty) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m) fill(value, 10);") + tdSql.checkRows(841) + tdSql.checkData(0, 0, 10) + + tdSql.query("select _wstart, _wend, sum(c_int) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m);") + tdSql.checkRows(65) + + # status window + tdSql.error("select _wstart, count(*) from st state_window(t_bool);") + tdSql.query("select _wstart, count(*) from st partition by tbname state_window(c_bool);") + tdSql.checkRows(5) + tdSql.checkData(0, 1, 10000) + + # session window + tdSql.query("select _wstart, count(*) from st partition by tbname, t_int session(ts, 1m);") + tdSql.checkRows(5) + tdSql.checkData(0, 1, 10000) + + # event window + tdSql.query("select _wstart, _wend, count(*) from (select * from st order by ts, tbname) event_window start with t_bool=true end with t_bool=false;") + tdSql.checkRows(20000) + + def test_query_with_union(self): + tdSql.query("select count(ts) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(ts) from (select * from ct1 union all select * from ct2 union all select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 30000) + + tdSql.query("select count(*) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(c_ts_empty) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select count(*) from (select ts from st union select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10001) + + tdSql.query("select count(*) from (select ts from st union all select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 100000) + + tdSql.query("select count(ts) from (select ts from st union select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(ts) from (select ts from st union all select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 50000) + + def test_nested_query(self): + tdSql.query("select elapsed(ts, 1s) from (select * from (select * from st where c_int = 1) where c_int_empty is null);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9999) + + tdSql.query("select first(ts) as t, avg(c_int) as v from (select * from (select * from st where c_int = 1) where c_int_empty is null) group by t_timestamp order by t_timestamp desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, 1) + + tdSql.query("select max(c_tinyint) from (select c_tinyint, tbname from st where c_float_empty is null or t_int_empty is null) group by tbname order by c_tinyint desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select top(c_int, 3) from (select c_int, tbname from st where t_int in (2, 3)) group by tbname slimit 3;") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 1) + + def run(self): + self.prepareData() + self.test_query_with_filter() + self.test_query_with_groupby() + self.test_query_with_join() + self.test_query_with_window() + self.test_query_with_union() + self.test_nested_query() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 015fb4d679d9f825d2f99ab7ff28bf57d105ad2d Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 11:15:39 +0800 Subject: [PATCH 06/28] feat: support uniq grant --- source/libs/catalog/inc/catalogInt.h | 16 ++++- source/libs/catalog/src/ctgCache.c | 99 +++++++++++++++------------- 2 files changed, 68 insertions(+), 47 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index b9a0752178..d86825c216 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -320,14 +320,19 @@ typedef struct SCtgUserAuth { uint64_t userCacheSize; } SCtgUserAuth; +typedef struct SCtgGrantCache { + SRWLatch lock; + SGrantHbRsp grantInfo; + uint64_t grantCacheSize; +} SCtgGrantCache; + typedef struct SCatalog { uint64_t clusterId; bool stopUpdate; SDynViewVersion dynViewVer; SHashObj* userCache; // key:user, value:SCtgUserAuth SHashObj* dbCache; // key:dbname, value:SCtgDBCache - SHashObj* grantCache; - SGrantHbRsp _grantCache; + SCtgGrantCache grantCache; SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; @@ -377,6 +382,7 @@ typedef struct SCtgJob { int32_t tbCfgNum; int32_t svrVerNum; int32_t viewNum; + int32_t grantNum; } SCtgJob; typedef struct SCtgMsgCtx { @@ -560,6 +566,10 @@ typedef struct SCtgUpdateGrantInfoMsg { SGrantHbRsp* pRsp; } SCtgUpdateGrantInfoMsg; +typedef struct SCtgDropGrantInfoMsg { + SCatalog* pCtg; +} SCtgDropGrantInfoMsg; + typedef struct SCtgCacheOperation { int32_t opId; void* data; @@ -922,6 +932,7 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action); int32_t ctgOpDropStbMeta(SCtgCacheOperation* action); int32_t ctgOpDropTbMeta(SCtgCacheOperation* action); int32_t ctgOpDropViewMeta(SCtgCacheOperation* action); +int32_t ctgOpDropGrantInfo(SCtgCacheOperation* action); int32_t ctgOpUpdateUser(SCtgCacheOperation* action); int32_t ctgOpUpdateEpset(SCtgCacheOperation* operation); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char* dbFName, SCtgDBCache** pCache); @@ -969,6 +980,7 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation* operation); int32_t ctgOpUpdateTbIndex(SCtgCacheOperation* operation); int32_t ctgOpClearCache(SCtgCacheOperation* operation); int32_t ctgOpUpdateViewMeta(SCtgCacheOperation *operation); +int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation); int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char* tableName, int32_t* tbType); int32_t ctgGetTbHashVgroupFromCache(SCatalog* pCtg, const SName* pTableName, SVgroupInfo** pVgroup); int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsCtx *ctx, int32_t dbIdx, diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 05078fabb4..92bda82bd3 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -160,6 +160,16 @@ int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } +int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); +} + +void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { + CTG_UNLOCK(CTG_READ, &dbCache->dbLock); + taosHashRelease(pCtg->dbCache, dbCache); +} + + void ctgReleaseVgInfoToCache(SCatalog *pCtg, SCtgDBCache *dbCache) { ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); @@ -1512,6 +1522,47 @@ int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } +int32_t ctgGetAddGrantCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, dbFName, &dbCache); + + if (dbCache) { + // TODO OPEN IT +#if 0 + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#else + if (0 == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbId && (dbCache->dbId == 0)) { + dbCache->dbId = dbId; + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#endif + CTG_ERR_RET(ctgRemoveDBFromCache(pCtg, dbCache, dbFName)); + } + + CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); + + ctgGetDBCache(pCtg, dbFName, &dbCache); + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { @@ -2488,8 +2539,6 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { SCtgUpdateGrantInfoMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; SGrantHbRsp *pRsp = msg->pRsp; - SCtgDBCache *dbCache = NULL; - SViewMeta *pMeta = NULL; taosMemoryFreeClear(msg); @@ -2523,56 +2572,16 @@ _return: CTG_RET(code); } -int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) { +int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgDropViewMetaMsg *msg = operation->data; + SCtgDropGrantInfoMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; - int32_t tblType = 0; if (pCtg->stopUpdate) { goto _return; } - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); - if (NULL == dbCache) { - goto _return; - } - - if ((0 != msg->dbId) && (dbCache->dbId != msg->dbId)) { - ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", dbFName:%s, viewName:%s", msg->dbId, dbCache->dbId, - msg->dbFName, msg->viewName); - goto _return; - } - - SCtgViewCache *pViewCache = taosHashGet(dbCache->viewCache, msg->viewName, strlen(msg->viewName)); - if (NULL == pViewCache) { - ctgDebug("view %s already not in cache", msg->viewName); - goto _return; - } - - int64_t viewId = pViewCache->pMeta->viewId; - if (0 != msg->viewId && viewId != msg->viewId) { - ctgDebug("viewId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", viewName:%s", msg->viewId, viewId, msg->viewName); - goto _return; - } - - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pViewCache->pMeta)); - ctgFreeViewCacheImpl(pViewCache, true); - - if (taosHashRemove(dbCache->viewCache, msg->viewName, strlen(msg->viewName))) { - ctgError("view %s not exist in cache, dbFName:%s", msg->viewName, msg->dbFName); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } else { - atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(SCtgViewCache) + strlen(msg->viewName)); - CTG_DB_NUM_DEC(CTG_CI_VIEW); - } - - ctgDebug("view %s removed from cache, dbFName:%s", msg->viewName, msg->dbFName); - - CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->viewRent, viewId, ctgViewVersionSortCompare, ctgViewVersionSearchCompare)); - - ctgDebug("view %s removed from rent, dbFName:%s, viewId:0x%" PRIx64, msg->viewName, msg->dbFName, viewId); + printf("prop:%s:%d grant %s removed from rent\n"); _return: From a4524802497c61c170495bc102f8d9baeb09ecbf Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 12:28:51 +0800 Subject: [PATCH 07/28] feat: support uniq grant --- source/libs/catalog/src/ctgCache.c | 39 +++--------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 92bda82bd3..fc471d2513 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -161,14 +161,12 @@ int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) } int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { - CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); -} - -void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { - CTG_UNLOCK(CTG_READ, &dbCache->dbLock); - taosHashRelease(pCtg->dbCache, dbCache); + CTG_LOCK(CTG_READ, &pCtg->grantCache.lock); + *ppCache = &pCtg->grantCache; + CTG_CACHE_HIT_INC(CTG_CI_GRANT_INFO, 1); } +void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { CTG_UNLOCK(CTG_READ, &pCache->lock); } void ctgReleaseVgInfoToCache(SCatalog *pCtg, SCtgDBCache *dbCache) { ctgRUnlockVgInfo(dbCache); @@ -1527,36 +1525,7 @@ int32_t ctgGetAddGrantCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, dbFName, &dbCache); - if (dbCache) { - // TODO OPEN IT -#if 0 - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#else - if (0 == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - if (dbId && (dbCache->dbId == 0)) { - dbCache->dbId = dbId; - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#endif - CTG_ERR_RET(ctgRemoveDBFromCache(pCtg, dbCache, dbFName)); - } - - CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); - - ctgGetDBCache(pCtg, dbFName, &dbCache); *pCache = dbCache; From 3f061a8460c9c1e64b9d90a16deece217a440ac4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 13:44:33 +0800 Subject: [PATCH 08/28] feat: support uniq grant --- include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 1 + source/libs/catalog/src/ctgCache.c | 54 ++++++++++++++---------------- source/libs/catalog/src/ctgRent.c | 14 ++++++++ source/libs/catalog/src/ctgUtil.c | 19 +++++++++++ 5 files changed, 61 insertions(+), 28 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index dfbdc5a9f4..524f8864f3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1887,6 +1887,7 @@ int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); void tFreeSViewHbRsp(SViewHbRsp* pRsp); typedef struct { + int32_t version; uint32_t flags; } SGrantHbRsp; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 2e013413a3..ba59ab85e9 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -146,6 +146,7 @@ typedef struct SSTableVersion { } SSTableVersion; typedef struct SGrantVersion { + int32_t grantId; int32_t version; } SGrantVersion; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index fc471d2513..800e1edb80 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1520,18 +1520,6 @@ int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } -int32_t ctgGetAddGrantCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { - int32_t code = 0; - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, dbFName, &dbCache); - - - - *pCache = dbCache; - - return TSDB_CODE_SUCCESS; -} - int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { @@ -1743,6 +1731,30 @@ _return: CTG_RET(code); } +int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { + int32_t code = TSDB_CODE_SUCCESS; + SCtgGrantCache *pCache = &pCtg->grantCache; + + CTG_LOCK(CTG_WRITE, &pCache->lock); + pCache->grantInfo = *pRsp; + CTG_UNLOCK(CTG_WRITE, &pCache->lock); + + ctgDebug("grant info updated to cache, version"); + + CTG_ERR_RET(ctgUpdateRentViewVersion(pCtg, dbFName, viewName, dbCache->dbId, pMeta->viewId, pCache)); + + pMeta = NULL; + +_return: + + if (pMeta) { + ctgFreeSViewMeta(pMeta); + taosMemoryFree(pMeta); + } + + CTG_RET(code); +} + int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool syncReq) { STableMetaOutput *pOutput = NULL; int32_t code = 0; @@ -2508,6 +2520,7 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { SCtgUpdateGrantInfoMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; SGrantHbRsp *pRsp = msg->pRsp; + SCtgGrantCache *pGrantCache = NULL; taosMemoryFreeClear(msg); @@ -2515,29 +2528,14 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pRsp->dbFName, pRsp->dbId, &dbCache)); - if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, pRsp->dbFName, pRsp->dbId); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - pMeta = taosMemoryCalloc(1, sizeof(SViewMeta)); - if (NULL == pMeta) { - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - - CTG_ERR_JRET(dupViewMetaFromRsp(pRsp, pMeta)); + CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); code = ctgWriteViewMetaToCache(pCtg, dbCache, pRsp->dbFName, pRsp->name, pMeta); pMeta = NULL; _return: - tFreeSViewMetaRsp(pRsp); taosMemoryFree(pRsp); - ctgFreeSViewMeta(pMeta); - taosMemoryFree(pMeta); - CTG_RET(code); } diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 457285b147..799c76d5e5 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,6 +301,20 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } +int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, + SCtgGrantCache *pCache) { + SGrantVersion metaRent = {.grantId = grantId, .version = pCache->grantInfo.version}; + + + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), + ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); + + ctgDebug("grant %d version %d updated to grantRent", grantId, metaRent.version); + + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index cacdfcb8d6..e157028bee 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1328,6 +1328,16 @@ int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2) { } } +int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2) { + if (*(int32_t*)key1 == ((SGrantVersion*)key2)->grantId) { + return 0; + } else if (*(int32_t*)key1 < ((SGrantVersion*)key2)->grantId) { + return -1; + } else { + return 1; + } +} + int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) { @@ -1359,6 +1369,15 @@ int32_t ctgViewVersionSortCompare(const void* key1, const void* key2) { } } +int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2) { + if (((SGrantVersion*)key1)->grantId == ((SGrantVersion*)key2)->grantId) { + return 0; + } else if (((SGrantVersion*)key1)->grantId < ((SGrantVersion*)key2)->grantId) { + return -1; + } else { + return 1; + } +} int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { if (NULL == dbInfo) { From 3e7884a4f789efe75b121c9a30afc68dd90045cd Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 13:59:45 +0800 Subject: [PATCH 09/28] feat: support uniq grant --- include/libs/catalog/catalog.h | 2 +- source/libs/catalog/inc/catalogInt.h | 3 +++ source/libs/catalog/src/ctgCache.c | 12 +++++++----- source/libs/catalog/src/ctgRent.c | 9 +-------- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index ba59ab85e9..914ecb64ff 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -146,7 +146,7 @@ typedef struct SSTableVersion { } SSTableVersion; typedef struct SGrantVersion { - int32_t grantId; + int64_t grantId; int32_t version; } SGrantVersion; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index d86825c216..4d759d6bd7 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -568,6 +568,7 @@ typedef struct SCtgUpdateGrantInfoMsg { typedef struct SCtgDropGrantInfoMsg { SCatalog* pCtg; + int64_t grantId; } SCtgDropGrantInfoMsg; typedef struct SCtgCacheOperation { @@ -1040,10 +1041,12 @@ void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); int32_t ctgViewVersionSortCompare(const void* key1, const void* key2); +int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2); +int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 800e1edb80..0f93ba317a 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1739,7 +1739,7 @@ int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { pCache->grantInfo = *pRsp; CTG_UNLOCK(CTG_WRITE, &pCache->lock); - ctgDebug("grant info updated to cache, version"); + ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); CTG_ERR_RET(ctgUpdateRentViewVersion(pCtg, dbFName, viewName, dbCache->dbId, pMeta->viewId, pCache)); @@ -2540,15 +2540,18 @@ _return: } int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgDropGrantInfoMsg *msg = operation->data; - SCatalog *pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; if (pCtg->stopUpdate) { goto _return; } - printf("prop:%s:%d grant %s removed from rent\n"); + CTG_ERR_JRET( + ctgMetaRentRemove(&pCtg->grantRent, msg->grantId, ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); + + printf("prop:grant:0x%" PRIx64 "removed from rent", msg->grantId); _return: @@ -2557,7 +2560,6 @@ _return: CTG_RET(code); } - void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 799c76d5e5..9ef1cf5928 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,11 +301,9 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } -int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, - SCtgGrantCache *pCache) { +int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SCtgGrantCache *pCache) { SGrantVersion metaRent = {.grantId = grantId, .version = pCache->grantInfo.version}; - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); @@ -313,8 +311,3 @@ int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, return TSDB_CODE_SUCCESS; } - - - - - From 2ac374b2e01b300ef1b5a08106bf365ff31e14f3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 2 Feb 2024 14:02:13 +0800 Subject: [PATCH 10/28] max delay --- .../executor/src/streameventwindowoperator.c | 1 + .../executor/src/streamtimewindowoperator.c | 2 + tests/script/tsim/stream/windowClose.sim | 50 +++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 8aca76597b..d1138afb65 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -349,6 +349,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 02f8b90864..14fabaecea 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2083,6 +2083,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); @@ -3425,6 +3426,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index 67678963ea..e4b8fc7d84 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -290,6 +290,56 @@ if $data32 != $now32 then return -1 endi +print step 1 max delay 2s +sql create database test3 vgroups 4; +sql use test3; +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql create stream stream13 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 session(ts, 10s); + +sleep 1000 + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791223001,2,2,3,1.1); + +$loop_count = 0 + +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt13; + +if $rows != 2 then + print ======rows=$rows + goto loop2 +endi + +$now02 = $data02 +$now12 = $data12 + + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + print ======over system sh/exec.sh -n dnode1 -s stop -x SIGINT From 54f216b6c635453261d8d26c666c3bb34d6cb113 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 2 Feb 2024 15:19:58 +0800 Subject: [PATCH 11/28] add ci --- .../executor/src/streamtimewindowoperator.c | 4 + tests/script/tsim/stream/windowClose.sim | 161 +++++++++++++++++- 2 files changed, 160 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 14fabaecea..f26ff7156b 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2287,6 +2287,10 @@ int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { int32_t iter = 0; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { SResultWindowInfo* pWinInfo = pIte; + if (!pWinInfo->pStatePos->beUpdated) { + continue; + } + pWinInfo->pStatePos->beUpdated = false; saveResult(*pWinInfo, pStUpdated); } return TSDB_CODE_SUCCESS; diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index e4b8fc7d84..775ff81f51 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -290,17 +290,17 @@ if $data32 != $now32 then return -1 endi -print step 1 max delay 2s -sql create database test3 vgroups 4; -sql use test3; +print step 2 max delay 2s +sql create database test15 vgroups 4; +sql use test15; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream stream13 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 session(ts, 10s); +sql create stream stream15 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 session(ts, 10s); sleep 1000 sql insert into t1 values(1648791213000,1,2,3,1.0); -sql insert into t1 values(1648791223001,2,2,3,1.1); +sql insert into t1 values(1648791233001,2,2,3,1.1); $loop_count = 0 @@ -330,6 +330,157 @@ sleep 3000 sql select * from streamt13; +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +print step 3 max delay 2s +sql create database test16 vgroups 4; +sql use test16; +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql create stream stream16 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 state_window(a, 10s); + +sleep 1000 + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791233001,2,2,3,1.1); + +$loop_count = 0 + +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt13; + +if $rows != 2 then + print ======rows=$rows + goto loop2 +endi + +$now02 = $data02 +$now12 = $data12 + + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +print step 4 max delay 2s +sql create database test17 vgroups 4; +sql use test17; +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql create stream stream17 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 event_window start with a = 0 end with a = 9; + +sleep 1000 + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791213001,9,2,3,1.0); + +sql insert into t1 values(1648791233001,1,2,3,1.1); +sql insert into t1 values(1648791233009,9,2,3,1.1); + +$loop_count = 0 + +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt13; + +if $rows != 2 then + print ======rows=$rows + goto loop2 +endi + +$now02 = $data02 +$now12 = $data12 + + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + if $data02 != $now02 then print ======data02=$data02 return -1 From 092fc8d0110bb834968bb56ad703d2d50d8b27cf Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 3 Feb 2024 12:10:37 +0800 Subject: [PATCH 12/28] feat: support uniq grant --- include/util/tdef.h | 1 + source/common/src/systable.c | 16 ++++++++-------- source/common/src/tmsg.c | 2 ++ source/libs/catalog/inc/catalogInt.h | 4 +++- source/libs/catalog/src/ctgCache.c | 15 ++++----------- source/libs/catalog/src/ctgRent.c | 4 ++-- 6 files changed, 20 insertions(+), 22 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 4698d50e67..f136005026 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -287,6 +287,7 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_VALUE_LEN 256 #define TSDB_CLUSTER_VALUE_LEN 1000 +#define TSDB_GRANT_LOG_COL_LEN 15072 #define TSDB_ACTIVE_KEY_LEN 109 #define TSDB_CONN_ACTIVE_KEY_LEN 255 diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 77083d0425..47eac317ec 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -349,21 +349,21 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = { }; static const SSysDbTableSchema useGrantsFullSchema[] = { - {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema useGrantsLogsSchema[] = { - {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "machine", .bytes = 9088 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema useMachinesSchema[] = { {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "machine", .bytes = 6016 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysTableMeta infosMeta[] = { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index df62e8d8fc..72c17756b5 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9341,6 +9341,7 @@ int32_t tSerializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32v(&encoder, pRsp->version) < 0) return -1; if (tEncodeU32v(&encoder, pRsp->flags) < 0) return -1; tEndEncode(&encoder); @@ -9356,6 +9357,7 @@ int32_t tDeserializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32v(&decoder, &pRsp->version) < 0) return -1; if (tDecodeU32v(&decoder, &pRsp->flags) < 0) return -1; tEndDecode(&decoder); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 4d759d6bd7..b880c0a155 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -39,6 +39,7 @@ extern "C" { #define CTG_MAX_COMMAND_LEN 512 #define CTG_DEFAULT_CACHE_MON_MSEC 5000 #define CTG_CLEAR_CACHE_ROUND_TB_NUM 3000 +#define CGT_GRANT_ID 0 #define CTG_RENT_SLOT_SECOND 1.5 @@ -968,7 +969,8 @@ void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache); int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid, SCtgTbCache *pCache); int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, - SCtgViewCache *pCache); + SCtgViewCache *pCache); +int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 0f93ba317a..3b8a18e908 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -164,6 +164,7 @@ int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { CTG_LOCK(CTG_READ, &pCtg->grantCache.lock); *ppCache = &pCtg->grantCache; CTG_CACHE_HIT_INC(CTG_CI_GRANT_INFO, 1); + return TSDB_CODE_SUCCESS; } void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { CTG_UNLOCK(CTG_READ, &pCache->lock); } @@ -1741,17 +1742,10 @@ int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); - CTG_ERR_RET(ctgUpdateRentViewVersion(pCtg, dbFName, viewName, dbCache->dbId, pMeta->viewId, pCache)); - - pMeta = NULL; + CTG_ERR_RET(ctgUpdateRentGrantVersion(pCtg, CGT_GRANT_ID, pRsp)); _return: - if (pMeta) { - ctgFreeSViewMeta(pMeta); - taosMemoryFree(pMeta); - } - CTG_RET(code); } @@ -2528,10 +2522,9 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); + // CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); // TODO: inc/dec or cacheSize ? - code = ctgWriteViewMetaToCache(pCtg, dbCache, pRsp->dbFName, pRsp->name, pMeta); - pMeta = NULL; + code = ctgWriteGrantInfoToCache(pCtg, pRsp); _return: diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 9ef1cf5928..f757bd856f 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,8 +301,8 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } -int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SCtgGrantCache *pCache) { - SGrantVersion metaRent = {.grantId = grantId, .version = pCache->grantInfo.version}; +int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SGrantHbRsp *pGrant) { + SGrantVersion metaRent = {.grantId = grantId, .version = pGrant->version}; CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); From 3a0892631978a8126cf6b2531342503393b43511 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 3 Feb 2024 17:32:02 +0800 Subject: [PATCH 13/28] feat: support uniq grant --- include/common/tmsg.h | 2 +- include/libs/catalog/catalog.h | 3 +- source/client/src/clientHb.c | 54 ++++++++++++++++++++++-- source/dnode/mnode/impl/src/mndProfile.c | 4 +- source/libs/catalog/inc/catalogInt.h | 8 ++-- source/libs/catalog/src/catalog.c | 17 ++++++++ source/libs/catalog/src/ctgCache.c | 7 --- source/libs/catalog/src/ctgRent.c | 10 ----- source/libs/catalog/src/ctgUtil.c | 19 --------- 9 files changed, 76 insertions(+), 48 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 524f8864f3..96bf9677b7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -110,7 +110,7 @@ enum { HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_DYN_VIEW, HEARTBEAT_KEY_VIEWINFO, - HEARTBEAT_KEY_GRANT, + HEARTBEAT_KEY_GRANTINFO, }; typedef enum _mgmt_table { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 914ecb64ff..16deda205a 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -146,7 +146,6 @@ typedef struct SSTableVersion { } SSTableVersion; typedef struct SGrantVersion { - int64_t grantId; int32_t version; } SGrantVersion; @@ -343,6 +342,8 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion** stables, u int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* num, SDynViewVersion** dynViewVersion); +int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num); + int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* num); int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 755ab0eb91..6972ca3c38 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -342,7 +342,8 @@ static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCata return -1; } - tscInfo("hb to update grant info:%u", hbRsp->flags); + tscDebug("hb to update grant info, version:%d, flags:%u", hbRsp->version, hbRsp->flags); + catalogUpdateGrantInfo(pCatalog, hbRsp); return TSDB_CODE_SUCCESS; @@ -398,11 +399,12 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p hbProcessViewInfoRsp(kv->value, kv->valueLen, pCatalog); break; } - case HEARTBEAT_KEY_GRANT: { + case HEARTBEAT_KEY_GRANTINFO: { if (kv->valueLen <= 0 || NULL == kv->value) { tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); break; } + assert(0); hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); break; @@ -872,7 +874,7 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S view->version = htonl(view->version); } - tscDebug("hb got %d expired view, valueLen:%lu", viewNum, sizeof(SViewVersion) * viewNum); + tscDebug("hb got %u expired view, valueLen:%lu", viewNum, sizeof(SViewVersion) * viewNum); if (NULL == req->info) { req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); @@ -895,6 +897,47 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S return TSDB_CODE_SUCCESS; } +int32_t hbGetExpiredGrantInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SGrantVersion *grants = NULL; + uint32_t grantNum = 0; + int32_t code = 0; + + code = catalogGetExpiredGrants(pCatalog, &grants, &grantNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (grantNum <= 0) { + taosMemoryFree(grants); + return code; + } + + for (int32_t i = 0; i < grantNum; ++i) { + SGrantVersion *gv = &grants[i]; + gv->version = htonl(gv->version); + } + + tscDebug("hb got %d expired grant, valueLen:%u", grantNum, (int32_t)sizeof(SGrantVersion) * grantNum); + + if (!req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + if (!req->info) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(grants); + return code; + taosMemoryFree(grants); + } + } + + SKv kv = { + .key = HEARTBEAT_KEY_GRANTINFO, + .valueLen = sizeof(SGrantVersion) * grantNum, + .value = grants, + }; + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); @@ -958,6 +1001,11 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req if (TSDB_CODE_SUCCESS != code) { return code; } + + code = hbGetExpiredGrantInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } #endif } else { req->app.appId = 0; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d7e6d9609c..2d2d12a668 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -606,12 +606,12 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } - case HEARTBEAT_KEY_GRANT: { + case HEARTBEAT_KEY_GRANTINFO: { void *rspMsg = NULL; int32_t rspLen = 0; mndValidateGrant(pMnode, kv->value, &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { - SKv kv1 = {.key = HEARTBEAT_KEY_GRANT, .valueLen = rspLen, .value = rspMsg}; + SKv kv1 = {.key = HEARTBEAT_KEY_GRANTINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv1); } break; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index b880c0a155..89fd50657d 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -337,7 +337,6 @@ typedef struct SCatalog { SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; - SCtgRentMgmt grantRent; SCtgCacheStat cacheStat; } SCatalog; @@ -569,7 +568,6 @@ typedef struct SCtgUpdateGrantInfoMsg { typedef struct SCtgDropGrantInfoMsg { SCatalog* pCtg; - int64_t grantId; } SCtgDropGrantInfoMsg; typedef struct SCtgCacheOperation { @@ -970,7 +968,7 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uin SCtgTbCache *pCache); int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, SCtgViewCache *pCache); -int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); +// int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); @@ -1043,12 +1041,12 @@ void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); int32_t ctgViewVersionSortCompare(const void* key1, const void* key2); -int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); +// int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2); -int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); +// int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f52787a61e..c1bf641303 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1513,6 +1513,23 @@ int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* n CTG_API_LEAVE(ctgMetaRentGet(&pCtg->viewRent, (void**)views, num, sizeof(SViewVersion))); } +int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == grants || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + *num = 1; + *grants = taosMemoryCalloc(*num, sizeof(SGrantVersion)); + if (!(*grants)) { + ctgError("calloc %d grantVersion failed", *num); + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + (*grants)[0].version = pCtg->grantCache.grantInfo.version; + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 3b8a18e908..12c9907ffd 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1742,8 +1742,6 @@ int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); - CTG_ERR_RET(ctgUpdateRentGrantVersion(pCtg, CGT_GRANT_ID, pRsp)); - _return: CTG_RET(code); @@ -2541,11 +2539,6 @@ int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_JRET( - ctgMetaRentRemove(&pCtg->grantRent, msg->grantId, ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); - - printf("prop:grant:0x%" PRIx64 "removed from rent", msg->grantId); - _return: taosMemoryFreeClear(msg); diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index f757bd856f..0ac133b8e7 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,13 +301,3 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } -int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SGrantHbRsp *pGrant) { - SGrantVersion metaRent = {.grantId = grantId, .version = pGrant->version}; - - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), - ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); - - ctgDebug("grant %d version %d updated to grantRent", grantId, metaRent.version); - - return TSDB_CODE_SUCCESS; -} diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index e157028bee..cacdfcb8d6 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1328,16 +1328,6 @@ int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2) { } } -int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2) { - if (*(int32_t*)key1 == ((SGrantVersion*)key2)->grantId) { - return 0; - } else if (*(int32_t*)key1 < ((SGrantVersion*)key2)->grantId) { - return -1; - } else { - return 1; - } -} - int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) { @@ -1369,15 +1359,6 @@ int32_t ctgViewVersionSortCompare(const void* key1, const void* key2) { } } -int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2) { - if (((SGrantVersion*)key1)->grantId == ((SGrantVersion*)key2)->grantId) { - return 0; - } else if (((SGrantVersion*)key1)->grantId < ((SGrantVersion*)key2)->grantId) { - return -1; - } else { - return 1; - } -} int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { if (NULL == dbInfo) { From f53d0bce5d12669e21d75007b46a864a3ae84fca Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 00:24:42 +0800 Subject: [PATCH 14/28] feat: support uniq grant --- include/common/tgrant.h | 5 +++ include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 5 ++- include/util/taoserror.h | 2 -- source/dnode/vnode/src/vnd/vnodeSvr.c | 5 +++ source/libs/catalog/inc/catalogInt.h | 1 - source/libs/catalog/src/catalog.c | 47 ++++++++++++++++++++++++++ source/libs/catalog/src/ctgCache.c | 2 -- source/libs/parser/inc/parInsertUtil.h | 3 +- source/libs/parser/src/parInsertSml.c | 6 ++-- source/libs/parser/src/parInsertSql.c | 24 ++++++------- source/libs/parser/src/parInsertUtil.c | 9 ++--- source/util/src/terror.c | 2 -- 13 files changed, 84 insertions(+), 28 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 9f7572ed63..66408e9fa4 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -30,6 +30,10 @@ extern "C" { #define GRANT_HEART_BEAT_MIN 2 #define GRANT_ACTIVE_CODE "activeCode" +#define GRANT_ALL_FLAG (0x01) +#define GRANT_AUDIT_FLAG (0x02) +#define GRANT_CSV_FLAG (0x04) +#define GRANT_VIEW_FLAG (0x08) typedef enum { TSDB_GRANT_ALL, @@ -50,6 +54,7 @@ typedef enum { TSDB_GRANT_SUBSCRIPTION, TSDB_GRANT_AUDIT, TSDB_GRANT_CSV, + TSDB_GRANT_VIEW, TSDB_GRANT_MULTI_TIER, TSDB_GRANT_BACKUP_RESTORE, } EGrantType; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 96bf9677b7..899ce62a9d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3931,6 +3931,7 @@ int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq); #define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 +#define SUBMIT_REQ_FROM_FILE 0x4 typedef struct { int32_t flags; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 16deda205a..6ce1769f5f 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -25,6 +25,7 @@ extern "C" { #include "taosdef.h" #include "tarray.h" #include "tcommon.h" +#include "tgrant.h" #include "thash.h" #include "tmsg.h" #include "tname.h" @@ -364,7 +365,9 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes); -int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists); +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pAuth, SUserAuthRes* pRes, bool* exists); + +int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant); int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 94fe80b901..c0c20a0fde 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -575,8 +575,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0821) #define TSDB_CODE_GRANT_DUPLICATED_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x0822) #define TSDB_CODE_GRANT_VIEW_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0823) -#define TSDB_CODE_GRANT_CSV_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0824) -#define TSDB_CODE_GRANT_AUDIT_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0825) // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 3ec6adee41..e4d7b11176 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -238,6 +238,11 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int TSDB_CHECK_CODE(code, lino, _exit); } + if (submitTbData.flags & SUBMIT_REQ_FROM_FILE) { + code = grantCheck(TSDB_GRANT_CSV); + TSDB_CHECK_CODE(code, lino, _exit); + } + int64_t uid; if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { code = vnodePreprocessCreateTableReq(pVnode, pCoder, btimeMs, &uid); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 89fd50657d..0934aa0a27 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -324,7 +324,6 @@ typedef struct SCtgUserAuth { typedef struct SCtgGrantCache { SRWLatch lock; SGrantHbRsp grantInfo; - uint64_t grantCacheSize; } SCtgGrantCache; typedef struct SCatalog { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index c1bf641303..3d256f07d3 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -353,6 +353,39 @@ _return: CTG_RET(code); } +int32_t ctgChkGrant(SCatalog* pCtg, EGrantType grant) { + int32_t code = 0; + int32_t flag = 0; + SCtgGrantCache* pCache = &pCtg->grantCache; + + CTG_LOCK(CTG_READ, &pCache->lock); + switch (grant) { + case TSDB_GRANT_ALL: { + flag = pCache->grantInfo.flags & GRANT_ALL_FLAG; + break; + } + case TSDB_GRANT_AUDIT: { + flag = pCache->grantInfo.flags & GRANT_AUDIT_FLAG; + break; + } + case TSDB_GRANT_CSV: { + flag = pCache->grantInfo.flags & GRANT_CSV_FLAG; + break; + } + case TSDB_GRANT_VIEW: { + flag = pCache->grantInfo.flags & GRANT_VIEW_FLAG; + break; + } + } + CTG_UNLOCK(CTG_READ, &pCache->lock); + + if (flag) code = TSDB_CODE_GRANT_EXPIRED; + +_return: + + CTG_RET(code); +} + int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); @@ -1693,6 +1726,20 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant) { + CTG_API_ENTER(); + + if (NULL == pCtg) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgChkGrant(pCtg, grant)); + +_return: + + CTG_API_LEAVE(code); +} int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 12c9907ffd..ad48aa3fb1 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2520,8 +2520,6 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { goto _return; } - // CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); // TODO: inc/dec or cacheSize ? - code = ctgWriteGrantInfoToCache(pCtg, pRsp); _return: diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index b20587dd43..f1ba58f6de 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -48,7 +48,8 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); void insInitColValues(STableMeta* pTableMeta, SArray* aColValues); void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, - SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals); + SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals, + uint32_t insertType); int32_t initTableColSubmitData(STableDataCxt *pTableCxt); int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks, bool isRebuild); int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 13c4431b62..ffb4eacaea 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -169,7 +169,8 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { STableDataCxt* pTableCxt = NULL; SVCreateTbReq* pCreateTbReq = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false, + ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); if (ret != TSDB_CODE_SUCCESS) { return NULL; } @@ -313,7 +314,8 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc STableDataCxt* pTableCxt = NULL; ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false, + ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); goto end; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 8e32eca28e..b6df2e47ed 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1333,7 +1333,7 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), - pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false); + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false, pStmt->insertType); } char tbFName[TSDB_TABLE_FNAME_LEN]; @@ -1342,7 +1342,7 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS pStmt->pTableMeta->uid = 0; } return insGetTableDataCxt(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), pStmt->pTableMeta, - &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false); + &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false, pStmt->insertType); } static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { @@ -1931,7 +1931,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt } code = insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), - pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true); + pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true, pStmt->insertType); initTableColSubmitData(*ppTableDataCxt); if (code == TSDB_CODE_SUCCESS) { SRow** pRow = taosArrayReserve((*ppTableDataCxt)->pData->aRowP, 1); @@ -2139,14 +2139,20 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { + int32_t code = 0; + int32_t numOfRows = 0; + + if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { + return code; + } + // init only for file if (NULL == pStmt->pTableCxtHashObj) { pStmt->pTableCxtHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } - int32_t numOfRows = 0; - int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); + code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; @@ -2193,10 +2199,6 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); } - // if ((terrno = grantCheck(TSDB_GRANT_CSV)) < 0) { - // return buildInvalidOperationMsg(&pCxt->msg, terrstr()); - // } - NEXT_TOKEN(pStmt->pSql, *pToken); if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); @@ -2761,10 +2763,6 @@ static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpSt int32_t code = TSDB_CODE_SUCCESS; SRowsDataContext rowsDataCxt; - // if ((code = grantCheck(TSDB_GRANT_CSV)) < 0) { - // return code; - // } - if (!pStmt->stbSyntax) { STableDataCxt* pTableCxt = NULL; code = getTableDataCxt(pCxt, pStmt, &pTableCxt); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 6b655bfae6..5c7de2519c 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -208,7 +208,7 @@ void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { void insDestroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, - bool colMode, bool ignoreColVals) { + bool colMode, bool ignoreColVals, uint32_t insertType) { STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == pTableCxt) { return TSDB_CODE_OUT_OF_MEMORY; @@ -249,6 +249,7 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat } else { pTableCxt->pData->flags = (pCreateTbReq != NULL && NULL != *pCreateTbReq) ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; pTableCxt->pData->flags |= colMode ? SUBMIT_REQ_COLUMN_DATA_FORMAT : 0; + pTableCxt->pData->flags |= insertType == TSDB_QUERY_TYPE_FILE_INSERT ? SUBMIT_REQ_FROM_FILE : 0; pTableCxt->pData->suid = pTableMeta->suid; pTableCxt->pData->uid = pTableMeta->uid; pTableCxt->pData->sver = pTableMeta->sversion; @@ -330,7 +331,7 @@ static void resetColValues(SArray* pValues) { } int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, - SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals) { + SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals, uint32_t insertType) { STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); if (NULL != tmp) { *pTableCxt = *tmp; @@ -339,7 +340,7 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* } return TSDB_CODE_SUCCESS; } - int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals); + int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals, insertType); if (TSDB_CODE_SUCCESS == code) { void* pData = *pTableCxt; // deal scan coverity code = taosHashPut(pHash, id, idLen, &pData, POINTER_BYTES); @@ -645,7 +646,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false); + sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false, ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); if (ret != TSDB_CODE_SUCCESS) { uError("insGetTableDataCxt error"); goto end; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9fcca86744..7510b89736 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -462,8 +462,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MACHINES_MISMATCH, "Cluster machines mism TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE, "Expire time of optional grant item is too large") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUPLICATED_ACTIVE, "The active code can't be activated repeatedly") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED, "Number of view has reached the licensed upper limit") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CSV_LIMITED, "Csv has reached the licensed upper limit") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_AUDIT_LIMITED, "Audit has reached the licensed upper limit") // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") From a9316c37100a8d8aec51041d8a1ec7423eb34626 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 12:28:00 +0800 Subject: [PATCH 15/28] feat: support uniq grant --- include/libs/nodes/plannodes.h | 6 +++++- include/libs/nodes/querynodes.h | 2 ++ include/libs/parser/parser.h | 4 +++- include/libs/planner/planner.h | 2 ++ include/libs/qcom/query.h | 5 +++++ source/client/src/clientHb.c | 4 +--- source/client/src/clientImpl.c | 4 ++++ source/libs/nodes/src/nodesCloneFuncs.c | 2 ++ source/libs/nodes/src/nodesCodeFuncs.c | 12 ++++++++++++ source/libs/nodes/src/nodesMsgFuncs.c | 13 ++++++++++++- source/libs/parser/inc/parInsertUtil.h | 3 +-- source/libs/parser/inc/parTranslater.h | 2 ++ source/libs/parser/src/parInsertSml.c | 6 ++---- source/libs/parser/src/parInsertSql.c | 9 ++++++--- source/libs/parser/src/parInsertUtil.c | 9 ++++----- source/libs/parser/src/parUtil.c | 2 ++ source/libs/planner/src/planLogicCreater.c | 2 ++ source/libs/scheduler/src/schRemote.c | 3 +++ 18 files changed, 70 insertions(+), 20 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 8e375e9da8..3ff50ca2e8 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -92,6 +92,8 @@ typedef struct SScanLogicNode { STimeWindow scanRange; SName tableName; bool showRewrite; + bool isView; + bool isAudit; double ratio; SNodeList* pDynamicScanFuncs; int32_t dataRequired; @@ -713,8 +715,10 @@ typedef struct SSubplan { SNode* pTagCond; SNode* pTagIndexCond; bool showRewrite; - int32_t rowsThreshold; + bool isView; + bool isAudit; bool dynamicRowThreshold; + int32_t rowsThreshold; } SSubplan; typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 9647c0adac..97a20c2fc0 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -517,6 +517,8 @@ typedef struct SQuery { SArray* pTableList; SArray* pDbList; bool showRewrite; + // bool isView; + // bool isAudit; int32_t placeholderNum; SArray* pPlaceholderValues; SNode* pPrepareRoot; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 6a41f4607b..2ac2c3ccbd 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -86,8 +86,10 @@ typedef struct SParseContext { bool enableSysInfo; bool async; bool hasInvisibleCol; - const char* svrVer; + bool isView; + bool isAudit; bool nodeOffline; + const char* svrVer; SArray* pTableMetaPos; // sql table pos => catalog data pos SArray* pTableVgroupPos; // sql table pos => catalog data pos int64_t allocatorId; diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 1b523c0323..707d70b71b 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -32,6 +32,8 @@ typedef struct SPlanContext { bool streamQuery; bool rSmaQuery; bool showRewrite; + bool isView; + bool isAudit; int8_t triggerType; int64_t watermark; int64_t deleteMark; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index f79a0a0718..ad8de4707d 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -66,7 +66,12 @@ typedef enum { #define QUERY_RSP_POLICY_QUICK 1 #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) +#define QUERY_MSG_MASK_VIEW() (1 << 1) +#define QUERY_MSG_MASK_AUDIT() (1 << 2) #define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) +#define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) +#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_MASK()) != 0) + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6972ca3c38..e7529e7ed3 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -404,8 +404,6 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); break; } - assert(0); - hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); break; } @@ -917,7 +915,7 @@ int32_t hbGetExpiredGrantInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, gv->version = htonl(gv->version); } - tscDebug("hb got %d expired grant, valueLen:%u", grantNum, (int32_t)sizeof(SGrantVersion) * grantNum); + tscDebug("hb got %d expired grant, valueLen:%d", grantNum, (int32_t)(sizeof(SGrantVersion) * grantNum)); if (!req->info) { req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c1e1da617d..ddcf8ea105 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -458,6 +458,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .mgmtEpSet = getEpSet_s(&pAppInfo->mgmtEp), .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, + .isView = pQuery->isView; + .isAudit = pQuery->isAudit; .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, @@ -1154,6 +1156,8 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, + .isView = pWrapper->pParseCtx->isView, + .isAudit = pWrapper->pParseCtx->isAudit, .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index bc9839792c..b882271175 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -423,6 +423,8 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_OBJECT_FIELD(scanRange, sizeof(STimeWindow)); COPY_OBJECT_FIELD(tableName, sizeof(SName)); COPY_SCALAR_FIELD(showRewrite); + COPY_SCALAR_FIELD(isView); + COPY_SCALAR_FIELD(isAudit); COPY_SCALAR_FIELD(ratio); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); COPY_SCALAR_FIELD(dataRequired); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index cae6ffa5fa..b05c9b019e 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3332,6 +3332,12 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->showRewrite); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isAudit); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkSubplanRowsThreshold, pNode->rowsThreshold); } @@ -3379,6 +3385,12 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->showRewrite); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isAudit); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanRowsThreshold, &pNode->rowsThreshold); } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 3fd219b8d8..7d8cd72cdb 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -3930,6 +3930,12 @@ static int32_t subplanInlineToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueBool(pEncoder, pNode->dynamicRowThreshold); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->isAudit); + } return code; } @@ -3985,7 +3991,12 @@ static int32_t msgToSubplanInline(STlvDecoder* pDecoder, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueBool(pDecoder, &pNode->dynamicRowThreshold); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->isAudit); + } return code; } diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index f1ba58f6de..b20587dd43 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -48,8 +48,7 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); void insInitColValues(STableMeta* pTableMeta, SArray* aColValues); void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, - SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals, - uint32_t insertType); + SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals); int32_t initTableColSubmitData(STableDataCxt *pTableCxt); int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks, bool isRebuild); int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h index 55ea71a368..de885bb1be 100644 --- a/source/libs/parser/inc/parTranslater.h +++ b/source/libs/parser/inc/parTranslater.h @@ -43,6 +43,8 @@ typedef struct STranslateContext { bool createStream; bool stableQuery; bool showRewrite; + bool isView; + bool isAudit; SNode* pPrevRoot; SNode* pPostRoot; } STranslateContext; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index ffb4eacaea..13c4431b62 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -169,8 +169,7 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { STableDataCxt* pTableCxt = NULL; SVCreateTbReq* pCreateTbReq = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false, - ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); if (ret != TSDB_CODE_SUCCESS) { return NULL; } @@ -314,8 +313,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc STableDataCxt* pTableCxt = NULL; ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false, - ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); goto end; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index b6df2e47ed..83a38c4b15 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1333,7 +1333,7 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), - pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false, pStmt->insertType); + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false); } char tbFName[TSDB_TABLE_FNAME_LEN]; @@ -1342,7 +1342,7 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS pStmt->pTableMeta->uid = 0; } return insGetTableDataCxt(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), pStmt->pTableMeta, - &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false, pStmt->insertType); + &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false); } static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { @@ -1931,7 +1931,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt } code = insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), - pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true, pStmt->insertType); + pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true); initTableColSubmitData(*ppTableDataCxt); if (code == TSDB_CODE_SUCCESS) { SRow** pRow = taosArrayReserve((*ppTableDataCxt)->pData->aRowP, 1); @@ -2157,6 +2157,9 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT); + if (rowsDataCxt.pTableDataCxt && rowsDataCxt.pTableDataCxt->pData) { + rowsDataCxt.pTableDataCxt->pData->flags |= SUBMIT_REQ_FROM_FILE; + } if (!pStmt->fileProcessing) { taosCloseFile(&pStmt->fp); } else { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 5c7de2519c..6b655bfae6 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -208,7 +208,7 @@ void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { void insDestroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, - bool colMode, bool ignoreColVals, uint32_t insertType) { + bool colMode, bool ignoreColVals) { STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == pTableCxt) { return TSDB_CODE_OUT_OF_MEMORY; @@ -249,7 +249,6 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat } else { pTableCxt->pData->flags = (pCreateTbReq != NULL && NULL != *pCreateTbReq) ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; pTableCxt->pData->flags |= colMode ? SUBMIT_REQ_COLUMN_DATA_FORMAT : 0; - pTableCxt->pData->flags |= insertType == TSDB_QUERY_TYPE_FILE_INSERT ? SUBMIT_REQ_FROM_FILE : 0; pTableCxt->pData->suid = pTableMeta->suid; pTableCxt->pData->uid = pTableMeta->uid; pTableCxt->pData->sver = pTableMeta->sversion; @@ -331,7 +330,7 @@ static void resetColValues(SArray* pValues) { } int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, - SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals, uint32_t insertType) { + SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals) { STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); if (NULL != tmp) { *pTableCxt = *tmp; @@ -340,7 +339,7 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* } return TSDB_CODE_SUCCESS; } - int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals, insertType); + int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals); if (TSDB_CODE_SUCCESS == code) { void* pData = *pTableCxt; // deal scan coverity code = taosHashPut(pHash, id, idLen, &pData, POINTER_BYTES); @@ -646,7 +645,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false, ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); + sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false); if (ret != TSDB_CODE_SUCCESS) { uError("insGetTableDataCxt error"); goto end; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index dfe33ce55e..8aa37f5784 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -192,6 +192,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Out of memory"; case TSDB_CODE_PAR_ORDERBY_AMBIGUOUS: return "ORDER BY \"%s\" is ambiguous"; + case TSDB_CODE_GRANT_EXPIRED: + return "License expired"; default: return "Unknown error"; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 12b7360165..a5828af969 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -340,6 +340,8 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT strcpy(pScan->tableName.dbname, pRealTable->table.dbName); strcpy(pScan->tableName.tname, pRealTable->table.tableName); pScan->showRewrite = pCxt->pPlanCxt->showRewrite; + pScan->isView = pCxt->pPlanCxt->isView; + pScan->isAudit = pCxt->pPlanCxt->isAudit; pScan->ratio = pRealTable->ratio; pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; pScan->cacheLastMode = pRealTable->cacheLastMode; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 5c67c7974f..1a9c37ffe9 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1109,6 +1109,9 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.refId = pJob->refId; qMsg.execId = pTask->execId; qMsg.msgMask = (pTask->plan->showRewrite) ? QUERY_MSG_MASK_SHOW_REWRITE() : 0; + qMsg.msgMask |= (pTask->plan->isView) ? QUERY_MSG_MASK_VIEW() : 0; + qMsg.msgMask |= (pTask->plan->isAudit) ? QUERY_MSG_MASK_AUDIT() : 0; + qMsg.taskType = TASK_TYPE_TEMP; qMsg.explain = SCH_IS_EXPLAIN_JOB(pJob); qMsg.needFetch = SCH_TASK_NEED_FETCH(pTask); From 6cf7809e8d793de9ebf76300528dee1057c4c027 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 14:59:47 +0800 Subject: [PATCH 16/28] feat: support uniq grant --- include/common/tglobal.h | 2 +- include/libs/nodes/plannodes.h | 2 -- include/libs/nodes/querynodes.h | 2 -- include/libs/qcom/query.h | 2 +- source/client/src/clientImpl.c | 2 -- source/common/src/tglobal.c | 15 ++++++++++++++- source/libs/parser/inc/parTranslater.h | 4 ++-- source/libs/planner/src/planPhysiCreater.c | 2 ++ source/libs/qworker/src/qwMsg.c | 22 ++++++++++++++++++---- 9 files changed, 38 insertions(+), 15 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index f23bb4d51b..04e9b5a380 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -232,7 +232,7 @@ struct SConfig *taosGetCfg(); void taosSetAllDebugFlag(int32_t flag); void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); -int8_t taosGranted(); +int8_t taosGranted(int8_t type); #ifdef __cplusplus } diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 3ff50ca2e8..84934b4f4f 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -92,8 +92,6 @@ typedef struct SScanLogicNode { STimeWindow scanRange; SName tableName; bool showRewrite; - bool isView; - bool isAudit; double ratio; SNodeList* pDynamicScanFuncs; int32_t dataRequired; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 97a20c2fc0..9647c0adac 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -517,8 +517,6 @@ typedef struct SQuery { SArray* pTableList; SArray* pDbList; bool showRewrite; - // bool isView; - // bool isAudit; int32_t placeholderNum; SArray* pPlaceholderValues; SNode* pPrepareRoot; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index ad8de4707d..b63d3d5db4 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -70,7 +70,7 @@ typedef enum { #define QUERY_MSG_MASK_AUDIT() (1 << 2) #define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) #define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) -#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_MASK()) != 0) +#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_AUDIT()) != 0) typedef struct STableComInfo { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ddcf8ea105..d3d8ee1dc1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -458,8 +458,6 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .mgmtEpSet = getEpSet_s(&pAppInfo->mgmtEp), .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, - .isView = pQuery->isView; - .isAudit = pQuery->isAudit; .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 46c4d613fb..d46b91276b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1801,4 +1801,17 @@ void taosSetAllDebugFlag(int32_t flag) { if (terrno == TSDB_CODE_CFG_NOT_FOUND) terrno = TSDB_CODE_SUCCESS; // ignore not exist } -int8_t taosGranted() { return atomic_load_8(&tsGrant); } +int8_t taosGranted(int8_t type) { + switch (type) { + case TSDB_GRANT_ALL: + return atomic_load_8(&tsGrant) & GRANT_ALL_FLAG; + case TSDB_GRANT_AUDIT: + return atomic_load_8(&tsGrant) & GRANT_AUDIT_FLAG; + case TSDB_GRANT_VIEW: + return atomic_load_8(&tsGrant) & GRANT_VIEW_FLAG; + default: + ASSERTS(0, "undefined grant type:%" PRIi8, type); + break; + } + return 0; +} \ No newline at end of file diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h index de885bb1be..00f37cb002 100644 --- a/source/libs/parser/inc/parTranslater.h +++ b/source/libs/parser/inc/parTranslater.h @@ -43,8 +43,8 @@ typedef struct STranslateContext { bool createStream; bool stableQuery; bool showRewrite; - bool isView; - bool isAudit; + // bool isView; + // bool isAudit; SNode* pPrevRoot; SNode* pPostRoot; } STranslateContext; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 21c637116f..4b0024d098 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2164,6 +2164,8 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl pSubplan->level = pLogicSubplan->level; pSubplan->rowsThreshold = 4096; pSubplan->dynamicRowThreshold = false; + pSubplan->isView = pCxt->pPlanCxt->isView; + pSubplan->isAudit = pCxt->pPlanCxt->isAudit; if (NULL != pCxt->pPlanCxt->pUser) { snprintf(pSubplan->user, sizeof(pSubplan->user), "%s", pCxt->pPlanCxt->pUser); } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 9a1c309ab0..c9d3cb51e3 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -360,10 +360,24 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - if (chkGrant && (!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && !taosGranted()) { - QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + if (chkGrant) { + if ((!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_ALL)) { + QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + + if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) { + QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + + if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) { + QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } } uint64_t sId = msg.sId; From 3ebc26e1fb35dd0cb589ce859949905496b3b48a Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 16:33:43 +0800 Subject: [PATCH 17/28] feat: support uniq grant --- include/common/systable.h | 5 +++++ include/libs/qcom/query.h | 5 +++++ source/dnode/mnode/impl/src/mndCluster.c | 2 +- source/libs/nodes/src/nodesCloneFuncs.c | 2 -- source/libs/parser/src/parInsertSql.c | 6 +++--- source/libs/parser/src/parTranslater.c | 16 +++++++++++++++- source/libs/planner/src/planLogicCreater.c | 2 -- 7 files changed, 29 insertions(+), 9 deletions(-) diff --git a/include/common/systable.h b/include/common/systable.h index 37593144d8..f0f8ac8cf6 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -65,6 +65,11 @@ extern "C" { #define TSDB_PERFS_TABLE_TRANS "perf_trans" #define TSDB_PERFS_TABLE_APPS "perf_apps" +#define TSDB_AUDIT_DB "audit" +#define TSDB_AUDIT_STB_OPERATION "operations" +#define TSDB_AUDIT_CTB_OPERATION "t_operations_" +#define TSDB_AUDIT_CTB_OPERATION_LEN 13 + typedef struct SSysDbTableSchema { const char* name; const int32_t type; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index b63d3d5db4..4c33c36d1b 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -343,6 +343,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define IS_SYS_DBNAME(_dbname) (IS_INFORMATION_SCHEMA_DB(_dbname) || IS_PERFORMANCE_SCHEMA_DB(_dbname)) +#define IS_AUDIT_DBNAME(_dbname) ((*(_dbname) == 'a') && (0 == strcmp(_dbname, TSDB_AUDIT_DB))) +#define IS_AUDIT_STB_NAME(_stbname) ((*(_stbname) == 'o') && (0 == strcmp(_stbname, TSDB_AUDIT_STB_OPERATION))) +#define IS_AUDIT_CTB_NAME(_ctbname) \ + ((*(_ctbname) == 't') && (0 == strncmp(_ctbname, TSDB_AUDIT_CTB_OPERATION, TSDB_AUDIT_CTB_OPERATION_LEN))) + #define qFatal(...) \ do { \ if (qDebugFlag & DEBUG_FATAL) { \ diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 1a55a161bf..36b079d347 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -409,7 +409,7 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) { } { // audit - auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, cfgReq.sqlLen); + auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, MIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1)); } _exit: tFreeSMCfgClusterReq(&cfgReq); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index b882271175..bc9839792c 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -423,8 +423,6 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_OBJECT_FIELD(scanRange, sizeof(STimeWindow)); COPY_OBJECT_FIELD(tableName, sizeof(SName)); COPY_SCALAR_FIELD(showRewrite); - COPY_SCALAR_FIELD(isView); - COPY_SCALAR_FIELD(isAudit); COPY_SCALAR_FIELD(ratio); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); COPY_SCALAR_FIELD(dataRequired); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 83a38c4b15..a229d2658e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2142,9 +2142,9 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt int32_t code = 0; int32_t numOfRows = 0; - if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { - return code; - } + // if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { + // return code; + // } // init only for file if (NULL == pStmt->pTableCxtHashObj) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7dabae0c3a..de667b2f20 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3165,6 +3165,19 @@ static int32_t checkJoinTable(STranslateContext* pCxt, SJoinTableNode* pJoinTabl return TSDB_CODE_SUCCESS; } +static int32_t translateAudit(STranslateContext* pCxt, SRealTableNode* pRealTable, SName* pName) { + if (pRealTable->pMeta->tableType == TSDB_SUPER_TABLE) { + if (IS_AUDIT_DBNAME(pName->dbname) && IS_AUDIT_STB_NAME(pName->tname)) { + pCxt->pParseCxt->isAudit = true; + } + } else if (pRealTable->pMeta->tableType == TSDB_CHILD_TABLE) { + if (IS_AUDIT_DBNAME(pName->dbname) && IS_AUDIT_CTB_NAME(pName->tname)) { + pCxt->pParseCxt->isAudit = true; + } + } + return 0; +} + int32_t translateTable(STranslateContext* pCxt, SNode** pTable) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(*pTable)) { @@ -3184,7 +3197,8 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable) { if (TSDB_VIEW_TABLE == pRealTable->pMeta->tableType) { return translateView(pCxt, pTable, &name); } -#endif + translateAudit(pCxt, pRealTable, &name); +#endif code = setTableVgroupList(pCxt, &name, pRealTable); if (TSDB_CODE_SUCCESS == code) { code = setTableIndex(pCxt, &name, pRealTable); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index a5828af969..12b7360165 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -340,8 +340,6 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT strcpy(pScan->tableName.dbname, pRealTable->table.dbName); strcpy(pScan->tableName.tname, pRealTable->table.tableName); pScan->showRewrite = pCxt->pPlanCxt->showRewrite; - pScan->isView = pCxt->pPlanCxt->isView; - pScan->isAudit = pCxt->pPlanCxt->isAudit; pScan->ratio = pRealTable->ratio; pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; pScan->cacheLastMode = pRealTable->cacheLastMode; From d281bc70b0f0ed2618e23cda284439f9ae122109 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 16:46:24 +0800 Subject: [PATCH 18/28] feat: remove hb of grantInfo --- include/common/tmsg.h | 9 --- include/libs/catalog/catalog.h | 5 -- source/client/src/clientHb.c | 77 ------------------- source/common/src/tmsg.c | 31 -------- source/dnode/mnode/impl/inc/mndGrant.h | 1 - source/dnode/mnode/impl/src/mndProfile.c | 10 --- source/libs/catalog/inc/catalogInt.h | 27 ------- source/libs/catalog/src/catalog.c | 82 --------------------- source/libs/catalog/src/ctgCache.c | 94 ------------------------ source/libs/parser/src/parInsertSql.c | 4 - 10 files changed, 340 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 899ce62a9d..9cd9acaee4 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -110,7 +110,6 @@ enum { HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_DYN_VIEW, HEARTBEAT_KEY_VIEWINFO, - HEARTBEAT_KEY_GRANTINFO, }; typedef enum _mgmt_table { @@ -1886,14 +1885,6 @@ int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); void tFreeSViewHbRsp(SViewHbRsp* pRsp); -typedef struct { - int32_t version; - uint32_t flags; -} SGrantHbRsp; - -int32_t tSerializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); -int32_t tDeserializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); - typedef struct { int32_t numOfTables; int32_t numOfVgroup; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 6ce1769f5f..96a05ccbf9 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -343,8 +343,6 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion** stables, u int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* num, SDynViewVersion** dynViewVersion); -int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num); - int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* num); int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num); @@ -367,8 +365,6 @@ int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *p int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pAuth, SUserAuthRes* pRes, bool* exists); -int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant); - int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet* epSet); @@ -388,7 +384,6 @@ int32_t catalogRemoveViewMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId int32_t catalogUpdateDynViewVer(SCatalog* pCtg, SDynViewVersion* pVer); int32_t catalogUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); -int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg); int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index e7529e7ed3..53a64cec86 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,28 +327,6 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } -static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { - int32_t code = 0; - - SGrantHbRsp *hbRsp = taosMemoryCalloc(1, sizeof(SGrantHbRsp)); - if (!hbRsp) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (tDeserializeSGrantHbRsp(value, valueLen, hbRsp) != 0) { - taosMemoryFree(hbRsp); - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - tscDebug("hb to update grant info, version:%d, flags:%u", hbRsp->version, hbRsp->flags); - - catalogUpdateGrantInfo(pCatalog, hbRsp); - - return TSDB_CODE_SUCCESS; -} - static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pKvs, i); @@ -399,14 +377,6 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p hbProcessViewInfoRsp(kv->value, kv->valueLen, pCatalog); break; } - case HEARTBEAT_KEY_GRANTINFO: { - if (kv->valueLen <= 0 || NULL == kv->value) { - tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); - break; - } - hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); - break; - } #endif default: tscError("invalid hb key type:%d", kv->key); @@ -895,48 +865,6 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S return TSDB_CODE_SUCCESS; } -int32_t hbGetExpiredGrantInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { - SGrantVersion *grants = NULL; - uint32_t grantNum = 0; - int32_t code = 0; - - code = catalogGetExpiredGrants(pCatalog, &grants, &grantNum); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - - if (grantNum <= 0) { - taosMemoryFree(grants); - return code; - } - - for (int32_t i = 0; i < grantNum; ++i) { - SGrantVersion *gv = &grants[i]; - gv->version = htonl(gv->version); - } - - tscDebug("hb got %d expired grant, valueLen:%d", grantNum, (int32_t)(sizeof(SGrantVersion) * grantNum)); - - if (!req->info) { - req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); - if (!req->info) { - code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(grants); - return code; - taosMemoryFree(grants); - } - } - - SKv kv = { - .key = HEARTBEAT_KEY_GRANTINFO, - .valueLen = sizeof(SGrantVersion) * grantNum, - .value = grants, - }; - taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); - - return TSDB_CODE_SUCCESS; -} - int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL != pApp) { @@ -999,11 +927,6 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req if (TSDB_CODE_SUCCESS != code) { return code; } - - code = hbGetExpiredGrantInfo(connKey, pCatalog, req); - if (TSDB_CODE_SUCCESS != code) { - return code; - } #endif } else { req->app.appId = 0; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 72c17756b5..785cb1878e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9333,35 +9333,4 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) { } taosArrayDestroy(pRsp->pViewRsp); -} - -int32_t tSerializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - - if (tEncodeI32v(&encoder, pRsp->version) < 0) return -1; - if (tEncodeU32v(&encoder, pRsp->flags) < 0) return -1; - - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - - if (tDecodeI32v(&decoder, &pRsp->version) < 0) return -1; - if (tDecodeU32v(&decoder, &pRsp->flags) < 0) return -1; - - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index a012d7a8f5..bda5cebf01 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -43,7 +43,6 @@ int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); - int32_t mndValidateGrant(SMnode * pMnode, SGrantVersion * pGrantVersion, void **ppRsp, int32_t *pRspLen); int32_t mndGrantGetLastState(SMnode * pMnode, SGrantState * pState); SGrantLogObj *mndAcquireGrant(SMnode * pMnode, void **ppIter); void mndReleaseGrant(SMnode * pMnode, SGrantLogObj * pGrant, void *pIter); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 2d2d12a668..f42cdfa4f3 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -606,16 +606,6 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } - case HEARTBEAT_KEY_GRANTINFO: { - void *rspMsg = NULL; - int32_t rspLen = 0; - mndValidateGrant(pMnode, kv->value, &rspMsg, &rspLen); - if (rspMsg && rspLen > 0) { - SKv kv1 = {.key = HEARTBEAT_KEY_GRANTINFO, .valueLen = rspLen, .value = rspMsg}; - taosArrayPush(hbRsp.info, &kv1); - } - break; - } #endif default: mError("invalid kv key:%d", kv->key); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 0934aa0a27..a35eec0ec5 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -39,7 +39,6 @@ extern "C" { #define CTG_MAX_COMMAND_LEN 512 #define CTG_DEFAULT_CACHE_MON_MSEC 5000 #define CTG_CLEAR_CACHE_ROUND_TB_NUM 3000 -#define CGT_GRANT_ID 0 #define CTG_RENT_SLOT_SECOND 1.5 @@ -70,7 +69,6 @@ typedef enum { CTG_CI_UDF, CTG_CI_SVR_VER, CTG_CI_VIEW, - CTG_CI_GRANT_INFO, CTG_CI_MAX_VALUE, } CTG_CACHE_ITEM; @@ -103,8 +101,6 @@ enum { CTG_OP_DROP_TB_INDEX, CTG_OP_UPDATE_VIEW_META, CTG_OP_DROP_VIEW_META, - CTG_OP_UPDATE_GRANT_INFO, - CTG_OP_DROP_GRANT_INFO, CTG_OP_CLEAR_CACHE, CTG_OP_MAX }; @@ -127,7 +123,6 @@ typedef enum { CTG_TASK_GET_TB_HASH_BATCH, CTG_TASK_GET_TB_TAG, CTG_TASK_GET_VIEW, - CTG_TASK_GET_GRANT_INFO, } CTG_TASK_TYPE; typedef enum { @@ -321,18 +316,12 @@ typedef struct SCtgUserAuth { uint64_t userCacheSize; } SCtgUserAuth; -typedef struct SCtgGrantCache { - SRWLatch lock; - SGrantHbRsp grantInfo; -} SCtgGrantCache; - typedef struct SCatalog { uint64_t clusterId; bool stopUpdate; SDynViewVersion dynViewVer; SHashObj* userCache; // key:user, value:SCtgUserAuth SHashObj* dbCache; // key:dbname, value:SCtgDBCache - SCtgGrantCache grantCache; SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; @@ -381,7 +370,6 @@ typedef struct SCtgJob { int32_t tbCfgNum; int32_t svrVerNum; int32_t viewNum; - int32_t grantNum; } SCtgJob; typedef struct SCtgMsgCtx { @@ -560,15 +548,6 @@ typedef struct SCtgDropViewMetaMsg { uint64_t viewId; } SCtgDropViewMetaMsg; -typedef struct SCtgUpdateGrantInfoMsg { - SCatalog* pCtg; - SGrantHbRsp* pRsp; -} SCtgUpdateGrantInfoMsg; - -typedef struct SCtgDropGrantInfoMsg { - SCatalog* pCtg; -} SCtgDropGrantInfoMsg; - typedef struct SCtgCacheOperation { int32_t opId; void* data; @@ -931,7 +910,6 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action); int32_t ctgOpDropStbMeta(SCtgCacheOperation* action); int32_t ctgOpDropTbMeta(SCtgCacheOperation* action); int32_t ctgOpDropViewMeta(SCtgCacheOperation* action); -int32_t ctgOpDropGrantInfo(SCtgCacheOperation* action); int32_t ctgOpUpdateUser(SCtgCacheOperation* action); int32_t ctgOpUpdateEpset(SCtgCacheOperation* operation); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char* dbFName, SCtgDBCache** pCache); @@ -967,10 +945,8 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uin SCtgTbCache *pCache); int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, SCtgViewCache *pCache); -// int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); -int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask); void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache* dbCache); @@ -980,7 +956,6 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation* operation); int32_t ctgOpUpdateTbIndex(SCtgCacheOperation* operation); int32_t ctgOpClearCache(SCtgCacheOperation* operation); int32_t ctgOpUpdateViewMeta(SCtgCacheOperation *operation); -int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation); int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char* tableName, int32_t* tbType); int32_t ctgGetTbHashVgroupFromCache(SCatalog* pCtg, const SName* pTableName, SVgroupInfo** pVgroup); int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsCtx *ctx, int32_t dbIdx, @@ -1040,12 +1015,10 @@ void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); int32_t ctgViewVersionSortCompare(const void* key1, const void* key2); -// int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2); -// int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3d256f07d3..5a53f167c4 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -353,39 +353,6 @@ _return: CTG_RET(code); } -int32_t ctgChkGrant(SCatalog* pCtg, EGrantType grant) { - int32_t code = 0; - int32_t flag = 0; - SCtgGrantCache* pCache = &pCtg->grantCache; - - CTG_LOCK(CTG_READ, &pCache->lock); - switch (grant) { - case TSDB_GRANT_ALL: { - flag = pCache->grantInfo.flags & GRANT_ALL_FLAG; - break; - } - case TSDB_GRANT_AUDIT: { - flag = pCache->grantInfo.flags & GRANT_AUDIT_FLAG; - break; - } - case TSDB_GRANT_CSV: { - flag = pCache->grantInfo.flags & GRANT_CSV_FLAG; - break; - } - case TSDB_GRANT_VIEW: { - flag = pCache->grantInfo.flags & GRANT_VIEW_FLAG; - break; - } - } - CTG_UNLOCK(CTG_READ, &pCache->lock); - - if (flag) code = TSDB_CODE_GRANT_EXPIRED; - -_return: - - CTG_RET(code); -} - int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); @@ -1546,24 +1513,6 @@ int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* n CTG_API_LEAVE(ctgMetaRentGet(&pCtg->viewRent, (void**)views, num, sizeof(SViewVersion))); } -int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num) { - CTG_API_ENTER(); - - if (NULL == pCtg || NULL == grants || NULL == num) { - CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); - } - - *num = 1; - *grants = taosMemoryCalloc(*num, sizeof(SGrantVersion)); - if (!(*grants)) { - ctgError("calloc %d grantVersion failed", *num); - CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); - } - (*grants)[0].version = pCtg->grantCache.grantInfo.version; - - CTG_API_LEAVE(TSDB_CODE_SUCCESS); -} - int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { CTG_API_ENTER(); @@ -1726,21 +1675,6 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant) { - CTG_API_ENTER(); - - if (NULL == pCtg) { - CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); - } - - int32_t code = 0; - CTG_ERR_JRET(ctgChkGrant(pCtg, grant)); - -_return: - - CTG_API_LEAVE(code); -} - int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) { CTG_API_ENTER(); @@ -1811,22 +1745,6 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg) { - CTG_API_ENTER(); - - if (NULL == pCtg || NULL == pMsg) { - CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); - } - - int32_t code = 0; - CTG_ERR_JRET(ctgUpdateGrantInfoToCache(pCtg, pMsg, true)); - -_return: - - CTG_API_LEAVE(code); -} - - int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index ad48aa3fb1..08a8f348e9 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -32,8 +32,6 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update v {CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex}, {CTG_OP_UPDATE_VIEW_META, "update viewMeta", ctgOpUpdateViewMeta}, {CTG_OP_DROP_VIEW_META, "drop viewMeta", ctgOpDropViewMeta}, - {CTG_OP_UPDATE_GRANT_INFO, "update grantInfo", ctgOpUpdateGrantInfo}, - {CTG_OP_DROP_GRANT_INFO, "drop grantInfo", ctgOpDropGrantInfo}, {CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}}; SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = { @@ -160,15 +158,6 @@ int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } -int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { - CTG_LOCK(CTG_READ, &pCtg->grantCache.lock); - *ppCache = &pCtg->grantCache; - CTG_CACHE_HIT_INC(CTG_CI_GRANT_INFO, 1); - return TSDB_CODE_SUCCESS; -} - -void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { CTG_UNLOCK(CTG_READ, &pCache->lock); } - void ctgReleaseVgInfoToCache(SCatalog *pCtg, SCtgDBCache *dbCache) { ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); @@ -1334,33 +1323,6 @@ _return: CTG_RET(code); } -int32_t ctgUpdateGrantInfoEnqueue(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncOp) { - int32_t code = 0; - SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); - op->opId = CTG_OP_UPDATE_GRANT_INFO; - op->syncOp = syncOp; - - SCtgUpdateGrantInfoMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateGrantInfoMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateGrantInfoMsg)); - taosMemoryFree(op); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - - msg->pCtg = pCtg; - msg->pRsp = pRsp; - - op->data = msg; - - CTG_ERR_RET(ctgEnqueue(pCtg, op)); - - return TSDB_CODE_SUCCESS; -_return: - - taosMemoryFree(pRsp); - CTG_RET(code); -} - int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *viewName, uint64_t viewId, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); @@ -1732,21 +1694,6 @@ _return: CTG_RET(code); } -int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { - int32_t code = TSDB_CODE_SUCCESS; - SCtgGrantCache *pCache = &pCtg->grantCache; - - CTG_LOCK(CTG_WRITE, &pCache->lock); - pCache->grantInfo = *pRsp; - CTG_UNLOCK(CTG_WRITE, &pCache->lock); - - ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); - -_return: - - CTG_RET(code); -} - int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool syncReq) { STableMetaOutput *pOutput = NULL; int32_t code = 0; @@ -1768,10 +1715,6 @@ int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncRe CTG_RET(ctgUpdateViewMetaEnqueue(pCtg, pRsp, syncReq)); } -int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq) { - CTG_RET(ctgUpdateGrantInfoEnqueue(pCtg, pRsp, syncReq)); -} - void ctgClearAllHandles(void) { SCatalog *pCtg = NULL; @@ -2507,43 +2450,6 @@ _return: CTG_RET(code); } -int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { - int32_t code = 0; - SCtgUpdateGrantInfoMsg *msg = operation->data; - SCatalog *pCtg = msg->pCtg; - SGrantHbRsp *pRsp = msg->pRsp; - SCtgGrantCache *pGrantCache = NULL; - - taosMemoryFreeClear(msg); - - if (pCtg->stopUpdate) { - goto _return; - } - - code = ctgWriteGrantInfoToCache(pCtg, pRsp); - -_return: - - taosMemoryFree(pRsp); - CTG_RET(code); -} - -int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { - int32_t code = 0; - SCtgDropGrantInfoMsg *msg = operation->data; - SCatalog *pCtg = msg->pCtg; - - if (pCtg->stopUpdate) { - goto _return; - } - -_return: - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index a229d2658e..4ac8e0e541 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2142,10 +2142,6 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt int32_t code = 0; int32_t numOfRows = 0; - // if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { - // return code; - // } - // init only for file if (NULL == pStmt->pTableCxtHashObj) { pStmt->pTableCxtHashObj = From 60985381620e4e3ba028850724254a1195855f2a Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 18:00:02 +0800 Subject: [PATCH 19/28] feat: support uniq grant --- include/common/tgrant.h | 8 +++--- include/libs/catalog/catalog.h | 7 +----- source/client/src/clientHb.c | 2 ++ source/common/src/tglobal.c | 6 ++--- source/common/src/tmsg.c | 2 +- source/dnode/mnode/impl/inc/mndGrant.h | 5 ++-- source/dnode/mnode/impl/src/mndProfile.c | 1 - source/libs/catalog/inc/catalogInt.h | 5 ++-- source/libs/catalog/src/catalog.c | 3 +++ source/libs/catalog/src/ctgCache.c | 1 + source/libs/catalog/src/ctgRent.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 10 +++++--- source/libs/parser/inc/parTranslater.h | 2 -- source/libs/parser/src/parInsertSql.c | 11 +++----- source/libs/parser/src/parUtil.c | 2 -- source/libs/qworker/src/qwMsg.c | 32 ++++++++++++------------ 16 files changed, 48 insertions(+), 50 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 66408e9fa4..03c68ef59b 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -30,10 +30,10 @@ extern "C" { #define GRANT_HEART_BEAT_MIN 2 #define GRANT_ACTIVE_CODE "activeCode" -#define GRANT_ALL_FLAG (0x01) -#define GRANT_AUDIT_FLAG (0x02) -#define GRANT_CSV_FLAG (0x04) -#define GRANT_VIEW_FLAG (0x08) +#define GRANT_FLAG_ALL (0x01) +#define GRANT_FLAG_AUDIT (0x02) +#define GRANT_FLAG_CSV (0x04) +#define GRANT_FLAG_VIEW (0x08) typedef enum { TSDB_GRANT_ALL, diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 96a05ccbf9..1957bcbb24 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -25,7 +25,6 @@ extern "C" { #include "taosdef.h" #include "tarray.h" #include "tcommon.h" -#include "tgrant.h" #include "thash.h" #include "tmsg.h" #include "tname.h" @@ -146,10 +145,6 @@ typedef struct SSTableVersion { int32_t smaVer; } SSTableVersion; -typedef struct SGrantVersion { - int32_t version; -} SGrantVersion; - typedef struct SDbCacheInfo { char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; @@ -363,7 +358,7 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes); -int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pAuth, SUserAuthRes* pRes, bool* exists); +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists); int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 53a64cec86..a146712cab 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,6 +327,7 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } + static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pKvs, i); @@ -865,6 +866,7 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S return TSDB_CODE_SUCCESS; } + int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL != pApp) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d46b91276b..444a4c0ccc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1804,11 +1804,11 @@ void taosSetAllDebugFlag(int32_t flag) { int8_t taosGranted(int8_t type) { switch (type) { case TSDB_GRANT_ALL: - return atomic_load_8(&tsGrant) & GRANT_ALL_FLAG; + return atomic_load_8(&tsGrant) & GRANT_FLAG_ALL; case TSDB_GRANT_AUDIT: - return atomic_load_8(&tsGrant) & GRANT_AUDIT_FLAG; + return atomic_load_8(&tsGrant) & GRANT_FLAG_AUDIT; case TSDB_GRANT_VIEW: - return atomic_load_8(&tsGrant) & GRANT_VIEW_FLAG; + return atomic_load_8(&tsGrant) & GRANT_FLAG_VIEW; default: ASSERTS(0, "undefined grant type:%" PRIi8, type); break; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 785cb1878e..af1c1c4923 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9333,4 +9333,4 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) { } taosArrayDestroy(pRsp->pViewRsp); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index bda5cebf01..82b3260860 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -41,8 +41,9 @@ char **mergeActive); #endif - int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); - int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); + int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); + int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); + int32_t mndGrantGetLastState(SMnode * pMnode, SGrantState * pState); SGrantLogObj *mndAcquireGrant(SMnode * pMnode, void **ppIter); void mndReleaseGrant(SMnode * pMnode, SGrantLogObj * pGrant, void *pIter); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index f42cdfa4f3..e0559b4c48 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -18,7 +18,6 @@ #include "audit.h" #include "mndDb.h" #include "mndDnode.h" -#include "mndGrant.h" #include "mndMnode.h" #include "mndPrivilege.h" #include "mndQnode.h" diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a35eec0ec5..7474b3bf91 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -548,6 +548,7 @@ typedef struct SCtgDropViewMetaMsg { uint64_t viewId; } SCtgDropViewMetaMsg; + typedef struct SCtgCacheOperation { int32_t opId; void* data; @@ -943,8 +944,8 @@ void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache); void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache); int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid, SCtgTbCache *pCache); -int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, - SCtgViewCache *pCache); +int32_t ctgUpdateRentViewVersion(SCatalog* pCtg, char* dbFName, char* viewName, uint64_t dbId, uint64_t viewId, + SCtgViewCache* pCache); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 5a53f167c4..c2d88e5ce3 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1513,6 +1513,7 @@ int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* n CTG_API_LEAVE(ctgMetaRentGet(&pCtg->viewRent, (void**)views, num, sizeof(SViewVersion))); } + int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { CTG_API_ENTER(); @@ -1675,6 +1676,7 @@ _return: CTG_API_LEAVE(code); } + int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) { CTG_API_ENTER(); @@ -1745,6 +1747,7 @@ _return: CTG_API_LEAVE(code); } + int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 08a8f348e9..1b693b4e07 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2450,6 +2450,7 @@ _return: CTG_RET(code); } + void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 0ac133b8e7..f5016e9fef 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,3 +301,4 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } + diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index b05c9b019e..7533d00704 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3294,6 +3294,8 @@ static const char* jkSubplanTagIndexCond = "TagIndexCond"; static const char* jkSubplanShowRewrite = "ShowRewrite"; static const char* jkSubplanRowsThreshold = "RowThreshold"; static const char* jkSubplanDynamicRowsThreshold = "DyRowThreshold"; +static const char* jkSubplanIsView = "IsView"; +static const char* jkSubplanIsAudit = "IsAudit"; static int32_t subplanToJson(const void* pObj, SJson* pJson) { const SSubplan* pNode = (const SSubplan*)pObj; @@ -3333,10 +3335,10 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) { code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->showRewrite); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isView); + code = tjsonAddBoolToObject(pJson, jkSubplanIsView, pNode->isView); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isAudit); + code = tjsonAddBoolToObject(pJson, jkSubplanIsAudit, pNode->isAudit); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkSubplanRowsThreshold, pNode->rowsThreshold); @@ -3386,10 +3388,10 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->showRewrite); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isView); + code = tjsonGetBoolValue(pJson, jkSubplanIsView, &pNode->isView); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isAudit); + code = tjsonGetBoolValue(pJson, jkSubplanIsAudit, &pNode->isAudit); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanRowsThreshold, &pNode->rowsThreshold); diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h index 00f37cb002..55ea71a368 100644 --- a/source/libs/parser/inc/parTranslater.h +++ b/source/libs/parser/inc/parTranslater.h @@ -43,8 +43,6 @@ typedef struct STranslateContext { bool createStream; bool stableQuery; bool showRewrite; - // bool isView; - // bool isAudit; SNode* pPrevRoot; SNode* pPostRoot; } STranslateContext; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 4ac8e0e541..7d10d1f2df 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2139,16 +2139,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { - int32_t code = 0; - int32_t numOfRows = 0; - // init only for file if (NULL == pStmt->pTableCxtHashObj) { pStmt->pTableCxtHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } - - code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); + int32_t numOfRows = 0; + int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; @@ -2759,11 +2756,11 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOp } static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SRowsDataContext rowsDataCxt; if (!pStmt->stbSyntax) { - STableDataCxt* pTableCxt = NULL; + STableDataCxt* pTableCxt = NULL; code = getTableDataCxt(pCxt, pStmt, &pTableCxt); rowsDataCxt.pTableDataCxt = pTableCxt; } else { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 8aa37f5784..dfe33ce55e 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -192,8 +192,6 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Out of memory"; case TSDB_CODE_PAR_ORDERBY_AMBIGUOUS: return "ORDER BY \"%s\" is ambiguous"; - case TSDB_CODE_GRANT_EXPIRED: - return "License expired"; default: return "Unknown error"; } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index c9d3cb51e3..66ec460861 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -361,22 +361,22 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran } if (chkGrant) { - if ((!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_ALL)) { - QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); - } - - if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) { - QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); - } - - if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) { - QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + if ((!TEST_SHOW_REWRITE_MASK(msg.msgMask))) { + if (!taosGranted(TSDB_GRANT_ALL)) { + QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) { + QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) { + QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } } } From 021247c45bd7b88395184ab9b98b55ea4c9d6282 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 06:51:44 +0800 Subject: [PATCH 20/28] feat: support uniq grant --- include/common/tgrant.h | 3 +-- include/libs/qcom/query.h | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 03c68ef59b..2fa6dde8f6 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -32,8 +32,7 @@ extern "C" { #define GRANT_ACTIVE_CODE "activeCode" #define GRANT_FLAG_ALL (0x01) #define GRANT_FLAG_AUDIT (0x02) -#define GRANT_FLAG_CSV (0x04) -#define GRANT_FLAG_VIEW (0x08) +#define GRANT_FLAG_VIEW (0x04) typedef enum { TSDB_GRANT_ALL, diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 4c33c36d1b..e80268bb5f 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -66,8 +66,8 @@ typedef enum { #define QUERY_RSP_POLICY_QUICK 1 #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) -#define QUERY_MSG_MASK_VIEW() (1 << 1) -#define QUERY_MSG_MASK_AUDIT() (1 << 2) +#define QUERY_MSG_MASK_AUDIT() (1 << 1) +#define QUERY_MSG_MASK_VIEW() (1 << 2) #define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) #define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) #define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_AUDIT()) != 0) From 205525dd652dbf89de91f47d63e780e75f0ee2b6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 07:03:51 +0800 Subject: [PATCH 21/28] feat: support uniq grant --- include/libs/qcom/query.h | 7 +++---- source/libs/catalog/inc/catalogInt.h | 4 ++-- source/libs/catalog/src/ctgRent.c | 2 -- source/libs/scheduler/src/schRemote.c | 1 - 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index e80268bb5f..8d6cd6a3c0 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -68,10 +68,9 @@ typedef enum { #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) #define QUERY_MSG_MASK_AUDIT() (1 << 1) #define QUERY_MSG_MASK_VIEW() (1 << 2) -#define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) -#define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) -#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_AUDIT()) != 0) - +#define TEST_SHOW_REWRITE_MASK(m) (((m) & QUERY_MSG_MASK_SHOW_REWRITE()) != 0) +#define TEST_AUDIT_MASK(m) (((m) & QUERY_MSG_MASK_AUDIT()) != 0) +#define TEST_VIEW_MASK(m) (((m) & QUERY_MSG_MASK_VIEW()) != 0) typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 7474b3bf91..f9f4ee7dfc 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -944,8 +944,8 @@ void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache); void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache); int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid, SCtgTbCache *pCache); -int32_t ctgUpdateRentViewVersion(SCatalog* pCtg, char* dbFName, char* viewName, uint64_t dbId, uint64_t viewId, - SCtgViewCache* pCache); +int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, + SCtgViewCache *pCache); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index f5016e9fef..a19eb19c02 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -300,5 +300,3 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } - - diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1a9c37ffe9..1c0b31109e 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1111,7 +1111,6 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.msgMask = (pTask->plan->showRewrite) ? QUERY_MSG_MASK_SHOW_REWRITE() : 0; qMsg.msgMask |= (pTask->plan->isView) ? QUERY_MSG_MASK_VIEW() : 0; qMsg.msgMask |= (pTask->plan->isAudit) ? QUERY_MSG_MASK_AUDIT() : 0; - qMsg.taskType = TASK_TYPE_TEMP; qMsg.explain = SCH_IS_EXPLAIN_JOB(pJob); qMsg.needFetch = SCH_TASK_NEED_FETCH(pTask); From 739485a41970ba5e88eed18f00a13b6b61250e27 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 5 Feb 2024 09:10:04 +0800 Subject: [PATCH 22/28] test(stream): add unit test case for the mnode stream module. --- include/libs/stream/streamState.h | 6 ++--- include/libs/stream/tstream.h | 6 ++--- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/inc/mndStream.h | 1 + source/dnode/mnode/impl/src/mndStream.c | 26 ++----------------- source/dnode/mnode/impl/src/mndStreamHb.c | 12 ++++++--- source/dnode/mnode/impl/src/mndStreamUtil.c | 24 +++++++++++++++++ source/dnode/mnode/impl/test/CMakeLists.txt | 2 +- .../mnode/impl/test/stream/CMakeLists.txt | 13 ++++++++++ 9 files changed, 56 insertions(+), 36 deletions(-) create mode 100644 source/dnode/mnode/impl/test/stream/CMakeLists.txt diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 24222677a4..c2f7c6de2f 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -13,6 +13,9 @@ * along with this program. If not, see . */ +#ifndef _STREAM_STATE_H_ +#define _STREAM_STATE_H_ + #include "tdatablock.h" #include "rocksdb/c.h" @@ -20,9 +23,6 @@ #include "tsimplehash.h" #include "tstreamFileState.h" -#ifndef _STREAM_STATE_H_ -#define _STREAM_STATE_H_ - #ifdef __cplusplus extern "C" { #endif diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index dce8fffe11..2135bb706b 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -13,6 +13,9 @@ * along with this program. If not, see . */ +#ifndef _STREAM_H_ +#define _STREAM_H_ + #include "os.h" #include "streamState.h" #include "tdatablock.h" @@ -26,9 +29,6 @@ extern "C" { #endif -#ifndef _STREAM_H_ -#define _STREAM_H_ - #define ONE_MiB_F (1048576.0) #define ONE_KiB_F (1024.0) #define SIZE_IN_MiB(_v) ((_v) / ONE_MiB_F) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index b056d561c7..f2b94aa1d4 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -553,7 +553,7 @@ typedef struct { } SMqConsumerObj; SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]); -void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer, bool delete); +void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer, bool isDeleted); int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer, int8_t sver); diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 372612274f..4d1125a340 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -124,6 +124,7 @@ SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream); void destroyStreamTaskIter(SStreamTaskIter *pIter); bool streamTaskIterNextTask(SStreamTaskIter *pIter); SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter); +void mndInitExecInfo(); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 18cecddbdb..d51a740b70 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -62,8 +62,6 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP static void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot); static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len); -static void freeCheckpointCandEntry(void *); -static void freeTaskList(void *param); static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); SSdbRaw *mndStreamSeqActionEncode(SStreamObj *pStream); @@ -121,17 +119,7 @@ int32_t mndInitStream(SMnode *pMnode) { mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask); - taosThreadMutexInit(&execInfo.lock, NULL); - _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); - - execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId)); - execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK); - execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK); - execInfo.transMgmt.pWaitingList = taosHashInit(32, fn, true, HASH_NO_LOCK); - execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK); - - taosHashSetFreeFp(execInfo.transMgmt.pWaitingList, freeCheckpointCandEntry); - taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); + mndInitExecInfo(); if (sdbSetTable(pMnode->pSdb, table) != 0) { return -1; @@ -2118,16 +2106,6 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); } -void freeCheckpointCandEntry(void *param) { - SCheckpointCandEntry *pEntry = param; - taosMemoryFreeClear(pEntry->pName); -} - -void freeTaskList(void* param) { - SArray** pList = (SArray **)param; - taosArrayDestroy(*pList); -} - static void doAddTaskId(SArray* pList, int32_t taskId, int64_t uid, int32_t numOfTotal) { int32_t num = taosArrayGetSize(pList); for(int32_t i = 0; i < num; ++i) { @@ -2203,4 +2181,4 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { taosThreadMutexUnlock(&execInfo.lock); return 0; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 4426ab0672..49a931bba8 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -316,16 +316,20 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { // current checkpoint is failed, rollback from the checkpoint trans // kill the checkpoint trans and then set all tasks status to be normal if (taosArrayGetSize(pFailedTasks) > 0) { - bool allReady = true; - SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady); - taosArrayDestroy(p); + bool allReady = true; + if (pMnode != NULL) { + SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady); + taosArrayDestroy(p); + } else { + allReady = false; + } if (allReady || snodeChanged) { // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal for(int32_t i = 0; i < taosArrayGetSize(pFailedTasks); ++i) { SFailedCheckpointInfo *pInfo = taosArrayGet(pFailedTasks, i); mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status", - pInfo->checkpointId, pInfo->transId); + pInfo->checkpointId, pInfo->transId); mndResetStatusFromCheckpoint(pMnode, pInfo->streamUid, pInfo->transId); } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 235c604b27..3cabce2201 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -543,3 +543,27 @@ int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj * taosWUnLockLatch(&pStream->lock); return 0; } + +static void freeCheckpointCandEntry(void *param) { + SCheckpointCandEntry *pEntry = param; + taosMemoryFreeClear(pEntry->pName); +} + +static void freeTaskList(void* param) { + SArray** pList = (SArray **)param; + taosArrayDestroy(*pList); +} + +void mndInitExecInfo() { + taosThreadMutexInit(&execInfo.lock, NULL); + _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); + + execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId)); + execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK); + execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK); + execInfo.transMgmt.pWaitingList = taosHashInit(32, fn, true, HASH_NO_LOCK); + execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK); + + taosHashSetFreeFp(execInfo.transMgmt.pWaitingList, freeCheckpointCandEntry); + taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); +} diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index a002b20bde..bc5b5125f1 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -4,7 +4,7 @@ add_subdirectory(acct) #add_subdirectory(db) #add_subdirectory(dnode) add_subdirectory(func) -#add_subdirectory(mnode) +add_subdirectory(stream) add_subdirectory(profile) add_subdirectory(qnode) add_subdirectory(sdb) diff --git a/source/dnode/mnode/impl/test/stream/CMakeLists.txt b/source/dnode/mnode/impl/test/stream/CMakeLists.txt new file mode 100644 index 0000000000..b1bb62735f --- /dev/null +++ b/source/dnode/mnode/impl/test/stream/CMakeLists.txt @@ -0,0 +1,13 @@ +SET(CMAKE_CXX_STANDARD 11) + +aux_source_directory(. MNODE_STREAM_TEST_SRC) +add_executable(streamTest ${MNODE_STREAM_TEST_SRC}) +target_link_libraries( + streamTest + PRIVATE dnode gtest +) + +add_test( + NAME streamTest + COMMAND streamTest +) From 3111a01a1202fc1e76809d044741335391675e5a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 5 Feb 2024 09:10:43 +0800 Subject: [PATCH 23/28] test: add unit test files. --- .../dnode/mnode/impl/test/stream/stream.cpp | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 source/dnode/mnode/impl/test/stream/stream.cpp diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp new file mode 100644 index 0000000000..31ba89f3ca --- /dev/null +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include +#include +#include "../../inc/mndStream.h" + +namespace { +SRpcMsg buildHbReq() { + SStreamHbMsg msg = {0}; + msg.vgId = 1; + msg.numOfTasks = 5; + msg.pTaskStatus = taosArrayInit(4, sizeof(STaskStatusEntry)); + + for (int32_t i = 0; i < 4; ++i) { + STaskStatusEntry entry = {0}; + entry.nodeId = i + 1; + entry.stage = 1; + entry.id.taskId = i + 1; + entry.id.streamId = 999; + + if (i == 0) { + entry.stage = 4; + } + + taosArrayPush(msg.pTaskStatus, &entry); + } + + // (p->checkpointId != 0) && p->checkpointFailed + // add failed checkpoint info + { + STaskStatusEntry entry = {0}; + entry.nodeId = 5; + entry.stage = 1; + + entry.id.taskId = 5; + entry.id.streamId = 999; + + entry.checkpointId = 1; + entry.checkpointFailed = true; + + taosArrayPush(msg.pTaskStatus, &entry); + } + + int32_t tlen = 0; + int32_t code = 0; + SEncoder encoder; + void* buf = NULL; + SRpcMsg msg1 = {0}; + msg1.info.noResp = 1; + + tEncodeSize(tEncodeStreamHbMsg, &msg, tlen, code); + if (code < 0) { + goto _end; + } + + buf = rpcMallocCont(tlen); + if (buf == NULL) { + goto _end; + } + + tEncoderInit(&encoder, (uint8_t*)buf, tlen); + if ((code = tEncodeStreamHbMsg(&encoder, &msg)) < 0) { + rpcFreeCont(buf); + goto _end; + } + tEncoderClear(&encoder); + + initRpcMsg(&msg1, TDMT_MND_STREAM_HEARTBEAT, buf, tlen); + + taosArrayDestroy(msg.pTaskStatus); + return msg1; + +_end: + return msg1; +} + +void setTask(SStreamTask* pTask, int32_t nodeId, int64_t streamId, int32_t taskId) { + SStreamExecInfo* pExecNode = &execInfo; + + pTask->id.streamId = streamId; + pTask->id.taskId = taskId; + pTask->info.nodeId = nodeId; + + STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; + STaskStatusEntry entry = {0}; + streamTaskStatusInit(&entry, pTask); + + entry.stage = 1; + entry.status = TASK_STATUS__READY; + + taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry)); + taosArrayPush(pExecNode->pTaskList, &id); +} +void initStreamExecInfo() { + SStreamExecInfo* pExecNode = &execInfo; + + SStreamTask task = {0}; + setTask(&task, 1, 999, 1); + setTask(&task, 1, 999, 2); + setTask(&task, 1, 999, 3); + setTask(&task, 1, 999, 4); + setTask(&task, 2, 999, 5); +} + +void initNodeInfo() { + execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry)); + SNodeEntry entry = {0}; + entry.nodeId = 2; + entry.stageUpdated = true; + taosArrayPush(execInfo.pNodeList, &entry); +} +} // namespace + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(mndHbTest, handle_error_in_hb) { + mndInitExecInfo(); + initStreamExecInfo(); + initNodeInfo(); + + SRpcMsg msg = buildHbReq(); + int32_t code = mndProcessStreamHb(&msg); + + rpcFreeCont(msg.pCont); +} + +#pragma GCC diagnostic pop \ No newline at end of file From 7e5dc8a02425a7826ea7195a77c0a812ce509e30 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 09:21:11 +0800 Subject: [PATCH 24/28] feat: support uniq grant --- source/dnode/mnode/impl/src/mndCluster.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 36b079d347..f2b279276e 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -409,7 +409,7 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) { } { // audit - auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, MIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1)); + auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, TMIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1)); } _exit: tFreeSMCfgClusterReq(&cfgReq); From 196256d9af4b2de3effba0c1bafd7bf6b025b5d0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 09:23:19 +0800 Subject: [PATCH 25/28] feat: support uniq grant --- source/dnode/mnode/impl/src/mndGrant.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index 0b85f8fd5a..2ec6e09d12 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -79,11 +79,6 @@ char *tGetMachineId() { return NULL; }; int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } int32_t mndProcessConfigGrantReq(SMnode *pMnode, SRpcMsg *pReq, SMCfgClusterReq *pCfg) { return 0; } -#else -#ifndef TD_UNIQ_GRANT -char *tGetMachineId() { return NULL; }; -int32_t mndProcessConfigGrantReq(SMnode *pMnode, SRpcMsg *pReq, SMCfgClusterReq *pCfg) { return 0; } -#endif #endif void mndGenerateMachineCode() { grantParseParameter(); } \ No newline at end of file From f6fea32eebb542d84c0bd3cf6de4ed82e9ae329c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 5 Feb 2024 09:27:19 +0800 Subject: [PATCH 26/28] fix: fix some errors. --- source/libs/parser/src/parTranslater.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4ffb5bfae8..51d37e0a35 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8209,7 +8209,7 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta return code; } - code = nodesListAppend((*pSelect1)->pGroupByList, (SNode*)pNode1); + code = nodesListAppend((*pSelect1)->pGroupByList, nodesCloneNode((const SNode*)pNode1)); if (code) { return code; } @@ -8222,18 +8222,17 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta pNode2->groupingSetType = GP_TYPE_NORMAL; pNode2->pParameterList = nodesMakeList(); if (NULL == pNode2->pParameterList) { - nodesDestroyNode((SNode*)pNode1); + nodesDestroyNode((SNode*)pNode2); return TSDB_CODE_OUT_OF_MEMORY; } - code = nodesListAppend(pNode2->pParameterList, (SNode*)pFunc2); + code = nodesListAppend(pNode2->pParameterList, nodesCloneNode((const SNode*)pFunc2)); if (code) { nodesDestroyNode((SNode*)pNode2); return code; } - code = nodesListAppend((*pSelect1)->pGroupByList, (SNode*)pNode2); - return code; + return nodesListAppend((*pSelect1)->pGroupByList, (SNode*)pNode2); } static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { From c56cebdd49f9c89cd0363e2fbc5d344d6471f9e8 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 5 Feb 2024 09:34:18 +0800 Subject: [PATCH 27/28] opt ci --- tests/script/tsim/stream/windowClose.sim | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index 775ff81f51..ce5c57572e 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -304,7 +304,7 @@ sql insert into t1 values(1648791233001,2,2,3,1.1); $loop_count = 0 -loop2: +loop4: sleep 1000 @@ -317,7 +317,7 @@ sql select * from streamt13; if $rows != 2 then print ======rows=$rows - goto loop2 + goto loop4 endi $now02 = $data02 @@ -356,12 +356,14 @@ if $data12 != $now12 then return -1 endi +print session max delay over + print step 3 max delay 2s sql create database test16 vgroups 4; sql use test16; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream stream16 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 state_window(a, 10s); +sql create stream stream16 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 state_window(a); sleep 1000 @@ -370,7 +372,7 @@ sql insert into t1 values(1648791233001,2,2,3,1.1); $loop_count = 0 -loop2: +loop5: sleep 1000 @@ -383,7 +385,7 @@ sql select * from streamt13; if $rows != 2 then print ======rows=$rows - goto loop2 + goto loop5 endi $now02 = $data02 @@ -422,12 +424,14 @@ if $data12 != $now12 then return -1 endi +print state max delay over + print step 4 max delay 2s sql create database test17 vgroups 4; sql use test17; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream stream17 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 event_window start with a = 0 end with a = 9; +sql create stream stream17 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 event_window start with a = 1 end with a = 9; sleep 1000 @@ -439,7 +443,7 @@ sql insert into t1 values(1648791233009,9,2,3,1.1); $loop_count = 0 -loop2: +loop6: sleep 1000 @@ -452,7 +456,7 @@ sql select * from streamt13; if $rows != 2 then print ======rows=$rows - goto loop2 + goto loop6 endi $now02 = $data02 @@ -491,6 +495,8 @@ if $data12 != $now12 then return -1 endi +print event max delay over + print ======over system sh/exec.sh -n dnode1 -s stop -x SIGINT From fc218a1f69ecc497a5d03360b072836f7da646db Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 5 Feb 2024 11:09:19 +0800 Subject: [PATCH 28/28] fix(test): fix syntax errors on windows. --- source/dnode/mnode/impl/test/stream/stream.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index 31ba89f3ca..a3babad80c 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -103,8 +103,11 @@ void setTask(SStreamTask* pTask, int32_t nodeId, int64_t streamId, int32_t taskI pTask->id.taskId = taskId; pTask->info.nodeId = nodeId; - STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - STaskStatusEntry entry = {0}; + STaskId id; + id.streamId = pTask->id.streamId; + id.taskId = pTask->id.taskId; + + STaskStatusEntry entry; streamTaskStatusInit(&entry, pTask); entry.stage = 1;