From 54c8f0103454da4e870d88fa787351e535bdb0c7 Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 1 Feb 2024 16:05:20 +0800 Subject: [PATCH 01/45] update test case for ts4479 and td28163 by charles --- tests/parallel_test/cases.task | 3 +- tests/system-test/1-insert/test_ts4479.py | 75 ++++++ tests/system-test/2-query/test_td28163.py | 265 ++++++++++++++++++++++ 3 files changed, 342 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/1-insert/test_ts4479.py create mode 100644 tests/system-test/2-query/test_td28163.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 91a0ac46e5..0060841ff5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -350,6 +350,7 @@ fi ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/ts-4272.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4295.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_td27388.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4479.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_timestamp.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py @@ -567,7 +568,7 @@ fi ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4382.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4403.py - +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_td28163.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py diff --git a/tests/system-test/1-insert/test_ts4479.py b/tests/system-test/1-insert/test_ts4479.py new file mode 100644 index 0000000000..be9789b5fc --- /dev/null +++ b/tests/system-test/1-insert/test_ts4479.py @@ -0,0 +1,75 @@ +import os +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf +import taos + + +class TDTestCase: + """Verify inserting varbinary type data of ts-4479 + """ + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.conn = conn + self.db_name = "db" + self.stable_name = "st" + + def run(self): + tdSql.execute("create database if not exists %s" % self.db_name) + tdSql.execute("use %s" % self.db_name) + # create super table + tdSql.execute("create table %s (ts timestamp, c1 varbinary(65517)) tags (t1 varbinary(16382))" % self.stable_name) + + # varbinary tag length is more than 16382 + tag = os.urandom(16383).hex() + tdSql.error("create table ct using st tags(%s);" % ('\\x' + tag)) + + # create child table with max column and tag length + child_table_list = [] + for i in range(2): + child_table_name = "ct_" + str(i+1) + child_table_list.append(child_table_name) + tag = os.urandom(16382).hex() + tdSql.execute("create table %s using st tags('%s');" % (child_table_name, '\\x' + tag)) + tdLog.info("create table %s successfully" % child_table_name) + + # varbinary column length is more than 65517 + value = os.urandom(65518).hex() + tdSql.error("insert into ct_1 values(now, '\\x%s');" % value) + + # insert data + for i in range(10): + sql = "insert into table_name values" + for j in range(5): + value = os.urandom(65517).hex() + sql += "(now+%ss, '%s')," % (str(j+1), '\\x' + value) + for child_table in child_table_list: + tdSql.execute(sql.replace("table_name", child_table)) + tdLog.info("Insert data into %s successfully" % child_table) + tdLog.info("Insert data round %s successfully" % str(i+1)) + tdSql.execute("flush database %s" % self.db_name) + + # insert \\x to varbinary column + tdSql.execute("insert into ct_1 values(now, '\\x');") + tdSql.query("select * from ct_1 where c1 = '\\x';") + tdSql.checkRows(1) + tdSql.checkData(0, 1, b'') + + # insert \\x to varbinary tag + tdSql.execute("create table ct_3 using st tags('\\x');") + tdSql.execute("insert into ct_3 values(now, '\\x45');") + tdSql.query("select * from st where t1='';") + tdSql.checkRows(1) + tdSql.checkData(0, 2, b'') + + def stop(self): + tdSql.execute("drop database if exists %s" % self.db_name) + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/test_td28163.py b/tests/system-test/2-query/test_td28163.py new file mode 100644 index 0000000000..df727f6c5a --- /dev/null +++ b/tests/system-test/2-query/test_td28163.py @@ -0,0 +1,265 @@ +import random +import itertools +from util.log import * +from util.cases import * +from util.sql import * +from util.sqlset import * +from util import constant +from util.common import * + + +class TDTestCase: + """Verify the jira TD-28163 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def prepareData(self): + # db + tdSql.execute("create database if not exists db") + tdSql.execute("use db") + + # super table + tdSql.execute("create stable st(ts timestamp, c_ts_empty timestamp, c_int int, c_int_empty int, c_unsigned_int int unsigned, \ + c_unsigned_int_empty int unsigned, c_bigint bigint, c_bigint_empty bigint, c_unsigned_bigint bigint unsigned, \ + c_unsigned_bigint_empty bigint unsigned, c_float float, c_float_empty float, c_double double, c_double_empty double, \ + c_binary binary(16), c_binary_empty binary(16), c_smallint smallint, c_smallint_empty smallint, \ + c_smallint_unsigned smallint unsigned, c_smallint_unsigned_empty smallint unsigned, c_tinyint tinyint, \ + c_tinyint_empty tinyint, c_tinyint_unsigned tinyint unsigned, c_tinyint_unsigned_empty tinyint unsigned, \ + c_bool bool, c_bool_empty bool, c_nchar nchar(16), c_nchar_empty nchar(16), c_varchar varchar(16), \ + c_varchar_empty varchar(16), c_varbinary varbinary(16), c_varbinary_empty varbinary(16)) \ + tags(t_timestamp timestamp, t_timestamp_empty timestamp, t_int int, t_int_empty int, \ + t_unsigned_int int unsigned, t_unsigned_int_empty int unsigned, t_bigint bigint, t_bigint_empty bigint, \ + t_unsigned_bigint bigint unsigned, t_unsigned_bigint_empty bigint unsigned, t_float float, t_float_empty float, \ + t_double double, t_double_empty double, t_binary binary(16), t_binary_empty binary(16), t_smallint smallint, \ + t_smallint_empty smallint, t_smallint_unsigned smallint unsigned, t_smallint_unsigned_empty smallint unsigned, \ + t_tinyint tinyint, t_tinyint_empty tinyint, t_tinyint_unsigned tinyint unsigned, t_tinyint_unsigned_empty tinyint unsigned, \ + t_bool bool, t_bool_empty bool, t_nchar nchar(16), t_nchar_empty nchar(16), t_varchar varchar(16), \ + t_varchar_empty varchar(16), t_varbinary varbinary(16), t_varbinary_empty varbinary(16));") + + # child tables + start_ts = 1704085200000 + tags = [ + "'2024-01-01 13:00:01', null, 1, null, 1, null, 1111111111111111, null, 1111111111111111, null, 1.1, null, 1.11, null, 'aaaaaaaa', '', 1, null, 1, null, 1, null, 1, null, True, null, 'ncharaa', null, 'varcharaa', null, '0x7661726331', null", + "'2024-01-01 13:00:02', null, 2, null, 2, null, 2222222222222222, null, 2222222222222222, null, 2.2, null, 2.22, null, 'bbbbbbbb', '', 2, null, 2, null, 2, null, 2, null, False, null, 'ncharbb', null, 'varcharbb', null, '0x7661726332', null", + "'2024-01-01 13:00:03', null, 3, null, 3, null, 3333333333333333, null, 3333333333333333, null, 3.3, null, 3.33, null, 'cccccccc', '', 3, null, 3, null, 3, null, 3, null, True, null, 'ncharcc', null, 'varcharcc', null, '0x7661726333', null", + "'2024-01-01 13:00:04', null, 4, null, 4, null, 4444444444444444, null, 4444444444444444, null, 4.4, null, 4.44, null, 'dddddddd', '', 4, null, 4, null, 4, null, 4, null, False, null, 'nchardd', null, 'varchardd', null, '0x7661726334', null", + "'2024-01-01 13:00:05', null, 5, null, 5, null, 5555555555555555, null, 5555555555555555, null, 5.5, null, 5.55, null, 'eeeeeeee', '', 5, null, 5, null, 5, null, 5, null, True, null, 'ncharee', null, 'varcharee', null, '0x7661726335', null", + ] + for i in range(5): + tdSql.execute(f"create table ct{i+1} using st tags({tags[i]});") + + # insert data + data = "null, 1, null, 1, null, 1111111111111111, null, 1111111111111111, null, 1.1, null, 1.11, null, 'aaaaaaaa', null, 1, null, 1, null, 1, null, 1, null, True, null, 'ncharaa', null, 'varcharaa', null, '0x7661726331', null" + for round in range(100): + sql = f"insert into ct{i+1} values" + for j in range(100): + sql += f"({start_ts + (round * 100 + j + 1) * 1000}, {data})" + sql += ";" + tdSql.execute(sql) + tdLog.debug("Prepare data successfully") + + def test_query_with_filter(self): + # total row number + tdSql.query("select count(*) from st;") + total_rows = tdSql.queryResult[0][0] + tdLog.debug("Total row number is %s" % total_rows) + + # start_ts and end_ts + tdSql.query("select first(ts), last(ts) from st;") + start_ts = tdSql.queryResult[0][0] + end_ts = tdSql.queryResult[0][1] + tdLog.debug("start_ts is %s, end_ts is %s" % (start_ts, end_ts)) + + filter_dic = { + "all_filter_list": ["ts <= now", "t_timestamp <= now", f"ts between '{start_ts}' and '{end_ts}'", + f"t_timestamp between '{start_ts}' and '{end_ts}'", "c_ts_empty is null", + "t_timestamp_empty is null", "ts > '1970-01-01 00:00:00'", "t_int in (1, 2, 3, 4, 5)", + "c_int=1", "c_int_empty is null", "c_unsigned_int=1", "c_unsigned_int_empty is null", + "c_unsigned_int in (1, 2, 3, 4, 5)", "c_unsigned_int_empty is null", "c_bigint=1111111111111111", + "c_bigint_empty is null", "c_unsigned_bigint in (1111111111111111)", "c_unsigned_bigint_empty is null", + "c_float=1.1", "c_float_empty is null", "c_double=1.11", "c_double_empty is null", "c_binary='aaaaaaaa'", + "c_binary_empty is null", "c_smallint=1", "c_smallint_empty is null", "c_smallint_unsigned=1", + "c_smallint_unsigned_empty is null", "c_tinyint=1", "c_tinyint_empty is null", "c_tinyint_unsigned=1", + "c_tinyint_unsigned_empty is null", "c_bool=True", "c_bool_empty is null", "c_nchar='ncharaa'", + "c_nchar_empty is null", "c_varchar='varcharaa'", "c_varchar_empty is null", "c_varbinary='0x7661726331'", + "c_varbinary_empty is null"], + "empty_filter_list": ["ts > now", "t_timestamp > now", "c_ts_empty is not null","t_timestamp_empty is not null", + "ts <= '1970-01-01 00:00:00'", "c_ts_empty < '1970-01-01 00:00:00'", "c_int <> 1", "c_int_empty is not null", + "t_int in (10, 11)", "t_int_empty is not null"] + } + for filter in filter_dic["all_filter_list"]: + tdLog.debug("Execute query with filter '%s'" % filter) + tdSql.query(f"select * from st where {filter};") + tdSql.checkRows(total_rows) + + for filter in filter_dic["empty_filter_list"]: + tdLog.debug("Execute query with filter '%s'" % filter) + tdSql.query(f"select * from st where {filter};") + tdSql.checkRows(0) + + def test_query_with_groupby(self): + tdSql.query("select count(*) from st group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(c_unsigned_int_empty + c_int_empty * c_float_empty - c_double_empty + c_smallint_empty / c_tinyint_empty) from st where c_int_empty is null group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 0) + + tdSql.query("select sum(t_unsigned_int_empty + t_int_empty * t_float_empty - t_double_empty + t_smallint_empty / t_tinyint_empty) from st where t_int_empty is null group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + + tdSql.query("select max(c_bigint_empty) from st group by tbname, t_bigint_empty, t_float_empty, t_double_empty;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + + tdSql.query("select min(t_double) as v from st where c_nchar like '%aa%' and t_double is not null group by tbname, t_bigint_empty, t_float_empty, t_double_empty order by v limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1.11) + + tdSql.query("select top(c_float, 1) as v from st where c_nchar like '%aa%' group by tbname order by v desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1.1) + + tdSql.query("select first(ts) from st where c_varchar is not null partition by tbname order by ts slimit 1;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + + tdSql.query("select first(c_nchar_empty) from st group by tbname;") + tdSql.checkRows(0) + + tdSql.query("select first(ts), first(c_nchar_empty) from st group by tbname, ts order by ts slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, None) + + tdSql.query("select first(c_nchar_empty) from st group by t_timestamp_empty order by t_timestamp;") + tdSql.checkRows(0) + + tdSql.query("select last(ts), last(c_nchar_empty) from st group by tbname, ts order by ts slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, None) + + tdSql.query("select elapsed(ts, 1s) t from st where c_int = 1 and c_nchar like '%aa%' group by tbname order by t desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9999) + + tdSql.query("select elapsed(ts, 1s) t from st where c_int_empty is not null and c_nchar like '%aa%' group by tbname order by t desc slimit 1 limit 1;") + tdSql.checkRows(0) + + def test_query_with_join(self): + tdSql.query("select count(*) from st as t1 join st as t2 on t1.ts = t2.ts and t1.c_float_empty is not null;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select count(t1.c_ts_empty) as v from st as t1 join st as t2 on t1.ts = t2.ts and t1.c_float_empty is null order by v desc;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select avg(t1.c_tinyint), sum(t2.c_bigint) from st t1, st t2 where t1.ts=t2.ts and t1.c_int > t2.c_int;") + tdSql.checkRows(0) + + tdSql.query("select avg(t1.c_tinyint), sum(t2.c_bigint) from st t1, st t2 where t1.ts=t2.ts and t1.c_int <= t2.c_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1076616672134475760) + + tdSql.query("select count(t1.c_float_empty) from st t1, st t2 where t1.ts=t2.ts and t1.c_int = t2.c_int and t1.t_int_empty=t2.t_int_empty;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + def test_query_with_window(self): + # time window + tdSql.query("select sum(c_int_empty) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m) fill(value, 10);") + tdSql.checkRows(841) + tdSql.checkData(0, 0, 10) + + tdSql.query("select _wstart, _wend, sum(c_int) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m);") + tdSql.checkRows(65) + + # status window + tdSql.error("select _wstart, count(*) from st state_window(t_bool);") + tdSql.query("select _wstart, count(*) from st partition by tbname state_window(c_bool);") + tdSql.checkRows(5) + tdSql.checkData(0, 1, 10000) + + # session window + tdSql.query("select _wstart, count(*) from st partition by tbname, t_int session(ts, 1m);") + tdSql.checkRows(5) + tdSql.checkData(0, 1, 10000) + + # event window + tdSql.query("select _wstart, _wend, count(*) from (select * from st order by ts, tbname) event_window start with t_bool=true end with t_bool=false;") + tdSql.checkRows(20000) + + def test_query_with_union(self): + tdSql.query("select count(ts) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(ts) from (select * from ct1 union all select * from ct2 union all select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 30000) + + tdSql.query("select count(*) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(c_ts_empty) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select count(*) from (select ts from st union select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10001) + + tdSql.query("select count(*) from (select ts from st union all select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 100000) + + tdSql.query("select count(ts) from (select ts from st union select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(ts) from (select ts from st union all select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 50000) + + def test_nested_query(self): + tdSql.query("select elapsed(ts, 1s) from (select * from (select * from st where c_int = 1) where c_int_empty is null);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9999) + + tdSql.query("select first(ts) as t, avg(c_int) as v from (select * from (select * from st where c_int = 1) where c_int_empty is null) group by t_timestamp order by t_timestamp desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, 1) + + tdSql.query("select max(c_tinyint) from (select c_tinyint, tbname from st where c_float_empty is null or t_int_empty is null) group by tbname order by c_tinyint desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select top(c_int, 3) from (select c_int, tbname from st where t_int in (2, 3)) group by tbname slimit 3;") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 1) + + def run(self): + self.prepareData() + self.test_query_with_filter() + self.test_query_with_groupby() + self.test_query_with_join() + self.test_query_with_window() + self.test_query_with_union() + self.test_nested_query() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 4d8ba0f60f324d82b84746d4f974d28429b5e0f2 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 1 Feb 2024 17:02:08 +0800 Subject: [PATCH 02/45] feat: support uniq grant --- include/common/tgrant.h | 2 +- source/util/src/tbase64.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index dbca2ac90c..688fac858e 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -69,7 +69,7 @@ int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, ch {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "state", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ + {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ diff --git a/source/util/src/tbase64.c b/source/util/src/tbase64.c index f6f12fef97..a2f4ddbc51 100644 --- a/source/util/src/tbase64.c +++ b/source/util/src/tbase64.c @@ -15,8 +15,6 @@ #define _DEFAULT_SOURCE #include "tbase64.h" -#include -#include static char basis_64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; From adee720128159c1f3236f28959d81c4e432bfb27 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 1 Feb 2024 17:48:29 +0800 Subject: [PATCH 03/45] feat: support uniq grant --- include/common/tgrant.h | 3 +- include/common/tmsg.h | 8 +++++ include/libs/catalog/catalog.h | 4 +++ source/client/src/clientHb.c | 40 +++++++++++++++++++++++ source/common/src/tgrant.c | 1 + source/common/src/tmsg.c | 29 ++++++++++++++++ source/dnode/mnode/impl/inc/mndGrant.h | 6 ++-- source/dnode/mnode/impl/src/mndConsumer.c | 7 ++-- source/dnode/mnode/impl/src/mndProfile.c | 11 +++++++ source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndStreamHb.c | 2 +- source/libs/parser/src/parInsertSql.c | 12 +++++-- 12 files changed, 114 insertions(+), 11 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 688fac858e..9f7572ed63 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -54,7 +54,8 @@ typedef enum { TSDB_GRANT_BACKUP_RESTORE, } EGrantType; -int32_t grantCheck(EGrantType grant); +int32_t grantCheck(EGrantType grant); // less +int32_t grantCheckLE(EGrantType grant); // less or equal char* tGetMachineId(); #ifndef TD_UNIQ_GRANT int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 04502eb64a..9b493c4a40 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -110,6 +110,7 @@ enum { HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_DYN_VIEW, HEARTBEAT_KEY_VIEWINFO, + HEARTBEAT_KEY_GRANT, }; typedef enum _mgmt_table { @@ -1885,6 +1886,13 @@ int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); void tFreeSViewHbRsp(SViewHbRsp* pRsp); +typedef struct { + uint32_t flags; +} SGrantHbRsp; + +int32_t tSerializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); +int32_t tDeserializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); + typedef struct { int32_t numOfTables; int32_t numOfVgroup; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index b34b998d76..3649f369b7 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -145,6 +145,10 @@ typedef struct SSTableVersion { int32_t smaVer; } SSTableVersion; +typedef struct SGrantVersion { + int32_t version; +} SGrantVersion; + typedef struct SDbCacheInfo { char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 63a65d7c95..f4c2fbec5a 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,6 +327,37 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } +#if 0 +static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t code = 0; + + SGrantHbRsp hbRsp = {0}; + if (tDeserializeSGrantHbRsp(value, valueLen, &hbRsp) != 0) { + taosArrayDestroyEx(hbRsp.pViewRsp, hbFreeSViewMetaInRsp); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t numOfMeta = taosArrayGetSize(hbRsp.pViewRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + SViewMetaRsp *rsp = taosArrayGetP(hbRsp.pViewRsp, i); + + if (rsp->numOfCols < 0) { + tscDebug("hb to remove view, db:%s, view:%s", rsp->dbFName, rsp->name); + catalogRemoveViewMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->name, rsp->viewId); + tFreeSViewMetaRsp(rsp); + taosMemoryFreeClear(rsp); + } else { + tscDebug("hb to update view, db:%s, view:%s", rsp->dbFName, rsp->name); + catalogUpdateViewMeta(pCatalog, rsp); + } + } + + taosArrayDestroy(hbRsp.pViewRsp); + return TSDB_CODE_SUCCESS; +} +#endif + static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { @@ -378,6 +409,15 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p hbProcessViewInfoRsp(kv->value, kv->valueLen, pCatalog); break; } + case HEARTBEAT_KEY_GRANT: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + // hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); + break; + } #endif default: tscError("invalid hb key type:%d", kv->key); diff --git a/source/common/src/tgrant.c b/source/common/src/tgrant.c index f212d71362..8e4fe9febb 100644 --- a/source/common/src/tgrant.c +++ b/source/common/src/tgrant.c @@ -19,5 +19,6 @@ #ifndef _GRANT int32_t grantCheck(EGrantType grant) {return TSDB_CODE_SUCCESS;} +int32_t grantCheckLE(EGrantType grant) {return TSDB_CODE_SUCCESS;} #endif \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index de505ab3d2..c866aae209 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9303,3 +9303,32 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) { taosArrayDestroy(pRsp->pViewRsp); } + +int32_t tSerializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeU32v(&encoder, pRsp->flags) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeU32v(&decoder, &pRsp->flags) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index 82b3260860..a012d7a8f5 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -41,9 +41,9 @@ char **mergeActive); #endif - int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); - int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); - + int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); + int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); + int32_t mndValidateGrant(SMnode * pMnode, SGrantVersion * pGrantVersion, void **ppRsp, int32_t *pRspLen); int32_t mndGrantGetLastState(SMnode * pMnode, SGrantState * pState); SGrantLogObj *mndAcquireGrant(SMnode * pMnode, void **ppIter); void mndReleaseGrant(SMnode * pMnode, SGrantLogObj * pGrant, void *pIter); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 3ce548a4f6..753076f1f3 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -107,7 +107,7 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode * goto FAILED; } - if ((terrno = grantCheck(TSDB_GRANT_SUBSCRIPTION)) < 0) { + if ((terrno = grantCheckLE(TSDB_GRANT_SUBSCRIPTION)) < 0) { code = terrno; goto FAILED; } @@ -240,9 +240,10 @@ static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbR } STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1); strcpy(data->topic, topic); - if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || grantCheck(TSDB_GRANT_SUBSCRIPTION) < 0) { + if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || + grantCheckLE(TSDB_GRANT_SUBSCRIPTION) < 0) { data->noPrivilege = 1; - } else{ + } else { data->noPrivilege = 0; } mndReleaseTopic(pMnode, pTopic); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e0559b4c48..d7e6d9609c 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -18,6 +18,7 @@ #include "audit.h" #include "mndDb.h" #include "mndDnode.h" +#include "mndGrant.h" #include "mndMnode.h" #include "mndPrivilege.h" #include "mndQnode.h" @@ -605,6 +606,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } + case HEARTBEAT_KEY_GRANT: { + void *rspMsg = NULL; + int32_t rspLen = 0; + mndValidateGrant(pMnode, kv->value, &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv1 = {.key = HEARTBEAT_KEY_GRANT, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); + } + break; + } #endif default: mError("invalid kv key:%d", kv->key); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5f5fbdba13..530b3d5bcd 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1628,7 +1628,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; - if(grantCheck(TSDB_GRANT_STREAMS) < 0){ + if(grantCheckLE(TSDB_GRANT_STREAMS) < 0){ terrno = TSDB_CODE_GRANT_EXPIRED; return -1; } diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 4426ab0672..005caea31b 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -225,7 +225,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SArray *pFailedTasks = taosArrayInit(4, sizeof(SFailedCheckpointInfo)); SArray *pOrphanTasks = taosArrayInit(3, sizeof(SOrphanTask)); - if(grantCheck(TSDB_GRANT_STREAMS) < 0){ + if(grantCheckLE(TSDB_GRANT_STREAMS) < 0){ if(suspendAllStreams(pMnode, &pReq->info) < 0){ return -1; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 512dfdaef2..8e32eca28e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2193,6 +2193,10 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); } + // if ((terrno = grantCheck(TSDB_GRANT_CSV)) < 0) { + // return buildInvalidOperationMsg(&pCxt->msg, terrstr()); + // } + NEXT_TOKEN(pStmt->pSql, *pToken); if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); @@ -2754,11 +2758,15 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOp } static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SRowsDataContext rowsDataCxt; + // if ((code = grantCheck(TSDB_GRANT_CSV)) < 0) { + // return code; + // } + if (!pStmt->stbSyntax) { - STableDataCxt* pTableCxt = NULL; + STableDataCxt* pTableCxt = NULL; code = getTableDataCxt(pCxt, pStmt, &pTableCxt); rowsDataCxt.pTableDataCxt = pTableCxt; } else { From 68fe671cbc826d8392c84d77516eb6e345e66bce Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 1 Feb 2024 21:04:38 +0800 Subject: [PATCH 04/45] add tsz compress --- tests/army/community/cluster/snapshot.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 5b5457be75..26bb6b9377 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -29,7 +29,11 @@ from frame import * class TDTestCase(TBase): updatecfgDict = { - "countAlwaysReturnValue" : "0" + "countAlwaysReturnValue" : "0", + "lossyColumns" : "float|double", + "fPrecision" : "0.000000001", + "dPrecision" : "0.00000000000000001", + "ifAdtFse" : "1" } def insertData(self): From 44d264456d951d1ce28770569bd5196cf44f59f3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 08:41:45 +0800 Subject: [PATCH 05/45] feat: support uniq grant --- include/libs/catalog/catalog.h | 1 + source/client/src/clientHb.c | 33 +++---- source/libs/catalog/inc/catalogInt.h | 12 +++ source/libs/catalog/src/catalog.c | 15 +++ source/libs/catalog/src/ctgCache.c | 131 +++++++++++++++++++++++++++ 5 files changed, 170 insertions(+), 22 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 3649f369b7..2e013413a3 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -383,6 +383,7 @@ int32_t catalogRemoveViewMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId int32_t catalogUpdateDynViewVer(SCatalog* pCtg, SDynViewVersion* pVer); int32_t catalogUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); +int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg); int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index f4c2fbec5a..755ab0eb91 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,37 +327,26 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } -#if 0 static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; - SGrantHbRsp hbRsp = {0}; - if (tDeserializeSGrantHbRsp(value, valueLen, &hbRsp) != 0) { - taosArrayDestroyEx(hbRsp.pViewRsp, hbFreeSViewMetaInRsp); + SGrantHbRsp *hbRsp = taosMemoryCalloc(1, sizeof(SGrantHbRsp)); + if (!hbRsp) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (tDeserializeSGrantHbRsp(value, valueLen, hbRsp) != 0) { + taosMemoryFree(hbRsp); terrno = TSDB_CODE_INVALID_MSG; return -1; } - int32_t numOfMeta = taosArrayGetSize(hbRsp.pViewRsp); - for (int32_t i = 0; i < numOfMeta; ++i) { - SViewMetaRsp *rsp = taosArrayGetP(hbRsp.pViewRsp, i); + tscInfo("hb to update grant info:%u", hbRsp->flags); + catalogUpdateGrantInfo(pCatalog, hbRsp); - if (rsp->numOfCols < 0) { - tscDebug("hb to remove view, db:%s, view:%s", rsp->dbFName, rsp->name); - catalogRemoveViewMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->name, rsp->viewId); - tFreeSViewMetaRsp(rsp); - taosMemoryFreeClear(rsp); - } else { - tscDebug("hb to update view, db:%s, view:%s", rsp->dbFName, rsp->name); - catalogUpdateViewMeta(pCatalog, rsp); - } - } - - taosArrayDestroy(hbRsp.pViewRsp); return TSDB_CODE_SUCCESS; } -#endif - static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { @@ -415,7 +404,7 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p break; } - // hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); + hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); break; } #endif diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index f9f4ee7dfc..b9a0752178 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -69,6 +69,7 @@ typedef enum { CTG_CI_UDF, CTG_CI_SVR_VER, CTG_CI_VIEW, + CTG_CI_GRANT_INFO, CTG_CI_MAX_VALUE, } CTG_CACHE_ITEM; @@ -101,6 +102,8 @@ enum { CTG_OP_DROP_TB_INDEX, CTG_OP_UPDATE_VIEW_META, CTG_OP_DROP_VIEW_META, + CTG_OP_UPDATE_GRANT_INFO, + CTG_OP_DROP_GRANT_INFO, CTG_OP_CLEAR_CACHE, CTG_OP_MAX }; @@ -123,6 +126,7 @@ typedef enum { CTG_TASK_GET_TB_HASH_BATCH, CTG_TASK_GET_TB_TAG, CTG_TASK_GET_VIEW, + CTG_TASK_GET_GRANT_INFO, } CTG_TASK_TYPE; typedef enum { @@ -322,9 +326,12 @@ typedef struct SCatalog { SDynViewVersion dynViewVer; SHashObj* userCache; // key:user, value:SCtgUserAuth SHashObj* dbCache; // key:dbname, value:SCtgDBCache + SHashObj* grantCache; + SGrantHbRsp _grantCache; SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; + SCtgRentMgmt grantRent; SCtgCacheStat cacheStat; } SCatalog; @@ -548,6 +555,10 @@ typedef struct SCtgDropViewMetaMsg { uint64_t viewId; } SCtgDropViewMetaMsg; +typedef struct SCtgUpdateGrantInfoMsg { + SCatalog* pCtg; + SGrantHbRsp* pRsp; +} SCtgUpdateGrantInfoMsg; typedef struct SCtgCacheOperation { int32_t opId; @@ -948,6 +959,7 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, SCtgViewCache *pCache); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); +int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask); void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache* dbCache); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index c2d88e5ce3..f52787a61e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1747,6 +1747,21 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pMsg) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgUpdateGrantInfoToCache(pCtg, pMsg, true)); + +_return: + + CTG_API_LEAVE(code); +} + int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 1b693b4e07..05078fabb4 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -32,6 +32,8 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update v {CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex}, {CTG_OP_UPDATE_VIEW_META, "update viewMeta", ctgOpUpdateViewMeta}, {CTG_OP_DROP_VIEW_META, "drop viewMeta", ctgOpDropViewMeta}, + {CTG_OP_UPDATE_GRANT_INFO, "update grantInfo", ctgOpUpdateGrantInfo}, + {CTG_OP_DROP_GRANT_INFO, "drop grantInfo", ctgOpDropGrantInfo}, {CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}}; SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = { @@ -1323,6 +1325,33 @@ _return: CTG_RET(code); } +int32_t ctgUpdateGrantInfoEnqueue(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_UPDATE_GRANT_INFO; + op->syncOp = syncOp; + + SCtgUpdateGrantInfoMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateGrantInfoMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateGrantInfoMsg)); + taosMemoryFree(op); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + msg->pCtg = pCtg; + msg->pRsp = pRsp; + + op->data = msg; + + CTG_ERR_RET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; +_return: + + taosMemoryFree(pRsp); + CTG_RET(code); +} + int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *viewName, uint64_t viewId, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); @@ -1715,6 +1744,10 @@ int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncRe CTG_RET(ctgUpdateViewMetaEnqueue(pCtg, pRsp, syncReq)); } +int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq) { + CTG_RET(ctgUpdateGrantInfoEnqueue(pCtg, pRsp, syncReq)); +} + void ctgClearAllHandles(void) { SCatalog *pCtg = NULL; @@ -2450,6 +2483,104 @@ _return: CTG_RET(code); } +int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgUpdateGrantInfoMsg *msg = operation->data; + SCatalog *pCtg = msg->pCtg; + SGrantHbRsp *pRsp = msg->pRsp; + SCtgDBCache *dbCache = NULL; + SViewMeta *pMeta = NULL; + + taosMemoryFreeClear(msg); + + if (pCtg->stopUpdate) { + goto _return; + } + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pRsp->dbFName, pRsp->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, pRsp->dbFName, pRsp->dbId); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + pMeta = taosMemoryCalloc(1, sizeof(SViewMeta)); + if (NULL == pMeta) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + CTG_ERR_JRET(dupViewMetaFromRsp(pRsp, pMeta)); + + code = ctgWriteViewMetaToCache(pCtg, dbCache, pRsp->dbFName, pRsp->name, pMeta); + pMeta = NULL; + +_return: + + tFreeSViewMetaRsp(pRsp); + taosMemoryFree(pRsp); + ctgFreeSViewMeta(pMeta); + taosMemoryFree(pMeta); + + CTG_RET(code); +} + +int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgDropViewMetaMsg *msg = operation->data; + SCatalog *pCtg = msg->pCtg; + int32_t tblType = 0; + + if (pCtg->stopUpdate) { + goto _return; + } + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + goto _return; + } + + if ((0 != msg->dbId) && (dbCache->dbId != msg->dbId)) { + ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", dbFName:%s, viewName:%s", msg->dbId, dbCache->dbId, + msg->dbFName, msg->viewName); + goto _return; + } + + SCtgViewCache *pViewCache = taosHashGet(dbCache->viewCache, msg->viewName, strlen(msg->viewName)); + if (NULL == pViewCache) { + ctgDebug("view %s already not in cache", msg->viewName); + goto _return; + } + + int64_t viewId = pViewCache->pMeta->viewId; + if (0 != msg->viewId && viewId != msg->viewId) { + ctgDebug("viewId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", viewName:%s", msg->viewId, viewId, msg->viewName); + goto _return; + } + + atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pViewCache->pMeta)); + ctgFreeViewCacheImpl(pViewCache, true); + + if (taosHashRemove(dbCache->viewCache, msg->viewName, strlen(msg->viewName))) { + ctgError("view %s not exist in cache, dbFName:%s", msg->viewName, msg->dbFName); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } else { + atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(SCtgViewCache) + strlen(msg->viewName)); + CTG_DB_NUM_DEC(CTG_CI_VIEW); + } + + ctgDebug("view %s removed from cache, dbFName:%s", msg->viewName, msg->dbFName); + + CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->viewRent, viewId, ctgViewVersionSortCompare, ctgViewVersionSearchCompare)); + + ctgDebug("view %s removed from rent, dbFName:%s, viewId:0x%" PRIx64, msg->viewName, msg->dbFName, viewId); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; From 57148a7731038e3497d3a4577f74d86fb17c04f3 Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 1 Feb 2024 16:05:20 +0800 Subject: [PATCH 06/45] udpate checkAsan.sh to skip asan_malloc_linux.cpp error --- tests/parallel_test/cases.task | 3 +- tests/script/sh/checkAsan.sh | 2 +- tests/system-test/1-insert/test_ts4479.py | 75 ++++++ tests/system-test/2-query/test_td28163.py | 265 ++++++++++++++++++++++ 4 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 tests/system-test/1-insert/test_ts4479.py create mode 100644 tests/system-test/2-query/test_td28163.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 91a0ac46e5..0060841ff5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -350,6 +350,7 @@ fi ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/ts-4272.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4295.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_td27388.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4479.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_timestamp.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py @@ -567,7 +568,7 @@ fi ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4382.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4403.py - +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_td28163.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 9f67d437e2..2c1ecd650f 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -72,7 +72,7 @@ python_error=$(cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l) #0 0x7f2d64f5a808 in __interceptor_malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cc:144 #1 0x7f2d63fcf459 in strerror /build/glibc-SzIz7B/glibc-2.31/string/strerror.c:38 -runtime_error=$(cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | grep -v "signed integer overflow" | grep -v "strerror.c" | grep -v "asan_malloc_linux.cc" | grep -v "strerror.c" | wc -l) +runtime_error=$(cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | grep -v "signed integer overflow" | grep -v "strerror.c" | grep -v "asan_malloc_linux.cc" | grep -v "strerror.c" | grep -v "asan_malloc_linux.cpp" | wc -l) echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" diff --git a/tests/system-test/1-insert/test_ts4479.py b/tests/system-test/1-insert/test_ts4479.py new file mode 100644 index 0000000000..be9789b5fc --- /dev/null +++ b/tests/system-test/1-insert/test_ts4479.py @@ -0,0 +1,75 @@ +import os +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf +import taos + + +class TDTestCase: + """Verify inserting varbinary type data of ts-4479 + """ + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.conn = conn + self.db_name = "db" + self.stable_name = "st" + + def run(self): + tdSql.execute("create database if not exists %s" % self.db_name) + tdSql.execute("use %s" % self.db_name) + # create super table + tdSql.execute("create table %s (ts timestamp, c1 varbinary(65517)) tags (t1 varbinary(16382))" % self.stable_name) + + # varbinary tag length is more than 16382 + tag = os.urandom(16383).hex() + tdSql.error("create table ct using st tags(%s);" % ('\\x' + tag)) + + # create child table with max column and tag length + child_table_list = [] + for i in range(2): + child_table_name = "ct_" + str(i+1) + child_table_list.append(child_table_name) + tag = os.urandom(16382).hex() + tdSql.execute("create table %s using st tags('%s');" % (child_table_name, '\\x' + tag)) + tdLog.info("create table %s successfully" % child_table_name) + + # varbinary column length is more than 65517 + value = os.urandom(65518).hex() + tdSql.error("insert into ct_1 values(now, '\\x%s');" % value) + + # insert data + for i in range(10): + sql = "insert into table_name values" + for j in range(5): + value = os.urandom(65517).hex() + sql += "(now+%ss, '%s')," % (str(j+1), '\\x' + value) + for child_table in child_table_list: + tdSql.execute(sql.replace("table_name", child_table)) + tdLog.info("Insert data into %s successfully" % child_table) + tdLog.info("Insert data round %s successfully" % str(i+1)) + tdSql.execute("flush database %s" % self.db_name) + + # insert \\x to varbinary column + tdSql.execute("insert into ct_1 values(now, '\\x');") + tdSql.query("select * from ct_1 where c1 = '\\x';") + tdSql.checkRows(1) + tdSql.checkData(0, 1, b'') + + # insert \\x to varbinary tag + tdSql.execute("create table ct_3 using st tags('\\x');") + tdSql.execute("insert into ct_3 values(now, '\\x45');") + tdSql.query("select * from st where t1='';") + tdSql.checkRows(1) + tdSql.checkData(0, 2, b'') + + def stop(self): + tdSql.execute("drop database if exists %s" % self.db_name) + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/test_td28163.py b/tests/system-test/2-query/test_td28163.py new file mode 100644 index 0000000000..df727f6c5a --- /dev/null +++ b/tests/system-test/2-query/test_td28163.py @@ -0,0 +1,265 @@ +import random +import itertools +from util.log import * +from util.cases import * +from util.sql import * +from util.sqlset import * +from util import constant +from util.common import * + + +class TDTestCase: + """Verify the jira TD-28163 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def prepareData(self): + # db + tdSql.execute("create database if not exists db") + tdSql.execute("use db") + + # super table + tdSql.execute("create stable st(ts timestamp, c_ts_empty timestamp, c_int int, c_int_empty int, c_unsigned_int int unsigned, \ + c_unsigned_int_empty int unsigned, c_bigint bigint, c_bigint_empty bigint, c_unsigned_bigint bigint unsigned, \ + c_unsigned_bigint_empty bigint unsigned, c_float float, c_float_empty float, c_double double, c_double_empty double, \ + c_binary binary(16), c_binary_empty binary(16), c_smallint smallint, c_smallint_empty smallint, \ + c_smallint_unsigned smallint unsigned, c_smallint_unsigned_empty smallint unsigned, c_tinyint tinyint, \ + c_tinyint_empty tinyint, c_tinyint_unsigned tinyint unsigned, c_tinyint_unsigned_empty tinyint unsigned, \ + c_bool bool, c_bool_empty bool, c_nchar nchar(16), c_nchar_empty nchar(16), c_varchar varchar(16), \ + c_varchar_empty varchar(16), c_varbinary varbinary(16), c_varbinary_empty varbinary(16)) \ + tags(t_timestamp timestamp, t_timestamp_empty timestamp, t_int int, t_int_empty int, \ + t_unsigned_int int unsigned, t_unsigned_int_empty int unsigned, t_bigint bigint, t_bigint_empty bigint, \ + t_unsigned_bigint bigint unsigned, t_unsigned_bigint_empty bigint unsigned, t_float float, t_float_empty float, \ + t_double double, t_double_empty double, t_binary binary(16), t_binary_empty binary(16), t_smallint smallint, \ + t_smallint_empty smallint, t_smallint_unsigned smallint unsigned, t_smallint_unsigned_empty smallint unsigned, \ + t_tinyint tinyint, t_tinyint_empty tinyint, t_tinyint_unsigned tinyint unsigned, t_tinyint_unsigned_empty tinyint unsigned, \ + t_bool bool, t_bool_empty bool, t_nchar nchar(16), t_nchar_empty nchar(16), t_varchar varchar(16), \ + t_varchar_empty varchar(16), t_varbinary varbinary(16), t_varbinary_empty varbinary(16));") + + # child tables + start_ts = 1704085200000 + tags = [ + "'2024-01-01 13:00:01', null, 1, null, 1, null, 1111111111111111, null, 1111111111111111, null, 1.1, null, 1.11, null, 'aaaaaaaa', '', 1, null, 1, null, 1, null, 1, null, True, null, 'ncharaa', null, 'varcharaa', null, '0x7661726331', null", + "'2024-01-01 13:00:02', null, 2, null, 2, null, 2222222222222222, null, 2222222222222222, null, 2.2, null, 2.22, null, 'bbbbbbbb', '', 2, null, 2, null, 2, null, 2, null, False, null, 'ncharbb', null, 'varcharbb', null, '0x7661726332', null", + "'2024-01-01 13:00:03', null, 3, null, 3, null, 3333333333333333, null, 3333333333333333, null, 3.3, null, 3.33, null, 'cccccccc', '', 3, null, 3, null, 3, null, 3, null, True, null, 'ncharcc', null, 'varcharcc', null, '0x7661726333', null", + "'2024-01-01 13:00:04', null, 4, null, 4, null, 4444444444444444, null, 4444444444444444, null, 4.4, null, 4.44, null, 'dddddddd', '', 4, null, 4, null, 4, null, 4, null, False, null, 'nchardd', null, 'varchardd', null, '0x7661726334', null", + "'2024-01-01 13:00:05', null, 5, null, 5, null, 5555555555555555, null, 5555555555555555, null, 5.5, null, 5.55, null, 'eeeeeeee', '', 5, null, 5, null, 5, null, 5, null, True, null, 'ncharee', null, 'varcharee', null, '0x7661726335', null", + ] + for i in range(5): + tdSql.execute(f"create table ct{i+1} using st tags({tags[i]});") + + # insert data + data = "null, 1, null, 1, null, 1111111111111111, null, 1111111111111111, null, 1.1, null, 1.11, null, 'aaaaaaaa', null, 1, null, 1, null, 1, null, 1, null, True, null, 'ncharaa', null, 'varcharaa', null, '0x7661726331', null" + for round in range(100): + sql = f"insert into ct{i+1} values" + for j in range(100): + sql += f"({start_ts + (round * 100 + j + 1) * 1000}, {data})" + sql += ";" + tdSql.execute(sql) + tdLog.debug("Prepare data successfully") + + def test_query_with_filter(self): + # total row number + tdSql.query("select count(*) from st;") + total_rows = tdSql.queryResult[0][0] + tdLog.debug("Total row number is %s" % total_rows) + + # start_ts and end_ts + tdSql.query("select first(ts), last(ts) from st;") + start_ts = tdSql.queryResult[0][0] + end_ts = tdSql.queryResult[0][1] + tdLog.debug("start_ts is %s, end_ts is %s" % (start_ts, end_ts)) + + filter_dic = { + "all_filter_list": ["ts <= now", "t_timestamp <= now", f"ts between '{start_ts}' and '{end_ts}'", + f"t_timestamp between '{start_ts}' and '{end_ts}'", "c_ts_empty is null", + "t_timestamp_empty is null", "ts > '1970-01-01 00:00:00'", "t_int in (1, 2, 3, 4, 5)", + "c_int=1", "c_int_empty is null", "c_unsigned_int=1", "c_unsigned_int_empty is null", + "c_unsigned_int in (1, 2, 3, 4, 5)", "c_unsigned_int_empty is null", "c_bigint=1111111111111111", + "c_bigint_empty is null", "c_unsigned_bigint in (1111111111111111)", "c_unsigned_bigint_empty is null", + "c_float=1.1", "c_float_empty is null", "c_double=1.11", "c_double_empty is null", "c_binary='aaaaaaaa'", + "c_binary_empty is null", "c_smallint=1", "c_smallint_empty is null", "c_smallint_unsigned=1", + "c_smallint_unsigned_empty is null", "c_tinyint=1", "c_tinyint_empty is null", "c_tinyint_unsigned=1", + "c_tinyint_unsigned_empty is null", "c_bool=True", "c_bool_empty is null", "c_nchar='ncharaa'", + "c_nchar_empty is null", "c_varchar='varcharaa'", "c_varchar_empty is null", "c_varbinary='0x7661726331'", + "c_varbinary_empty is null"], + "empty_filter_list": ["ts > now", "t_timestamp > now", "c_ts_empty is not null","t_timestamp_empty is not null", + "ts <= '1970-01-01 00:00:00'", "c_ts_empty < '1970-01-01 00:00:00'", "c_int <> 1", "c_int_empty is not null", + "t_int in (10, 11)", "t_int_empty is not null"] + } + for filter in filter_dic["all_filter_list"]: + tdLog.debug("Execute query with filter '%s'" % filter) + tdSql.query(f"select * from st where {filter};") + tdSql.checkRows(total_rows) + + for filter in filter_dic["empty_filter_list"]: + tdLog.debug("Execute query with filter '%s'" % filter) + tdSql.query(f"select * from st where {filter};") + tdSql.checkRows(0) + + def test_query_with_groupby(self): + tdSql.query("select count(*) from st group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(c_unsigned_int_empty + c_int_empty * c_float_empty - c_double_empty + c_smallint_empty / c_tinyint_empty) from st where c_int_empty is null group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 0) + + tdSql.query("select sum(t_unsigned_int_empty + t_int_empty * t_float_empty - t_double_empty + t_smallint_empty / t_tinyint_empty) from st where t_int_empty is null group by tbname;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + + tdSql.query("select max(c_bigint_empty) from st group by tbname, t_bigint_empty, t_float_empty, t_double_empty;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + + tdSql.query("select min(t_double) as v from st where c_nchar like '%aa%' and t_double is not null group by tbname, t_bigint_empty, t_float_empty, t_double_empty order by v limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1.11) + + tdSql.query("select top(c_float, 1) as v from st where c_nchar like '%aa%' group by tbname order by v desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1.1) + + tdSql.query("select first(ts) from st where c_varchar is not null partition by tbname order by ts slimit 1;") + tdSql.checkRows(5) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + + tdSql.query("select first(c_nchar_empty) from st group by tbname;") + tdSql.checkRows(0) + + tdSql.query("select first(ts), first(c_nchar_empty) from st group by tbname, ts order by ts slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, None) + + tdSql.query("select first(c_nchar_empty) from st group by t_timestamp_empty order by t_timestamp;") + tdSql.checkRows(0) + + tdSql.query("select last(ts), last(c_nchar_empty) from st group by tbname, ts order by ts slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, None) + + tdSql.query("select elapsed(ts, 1s) t from st where c_int = 1 and c_nchar like '%aa%' group by tbname order by t desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9999) + + tdSql.query("select elapsed(ts, 1s) t from st where c_int_empty is not null and c_nchar like '%aa%' group by tbname order by t desc slimit 1 limit 1;") + tdSql.checkRows(0) + + def test_query_with_join(self): + tdSql.query("select count(*) from st as t1 join st as t2 on t1.ts = t2.ts and t1.c_float_empty is not null;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select count(t1.c_ts_empty) as v from st as t1 join st as t2 on t1.ts = t2.ts and t1.c_float_empty is null order by v desc;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select avg(t1.c_tinyint), sum(t2.c_bigint) from st t1, st t2 where t1.ts=t2.ts and t1.c_int > t2.c_int;") + tdSql.checkRows(0) + + tdSql.query("select avg(t1.c_tinyint), sum(t2.c_bigint) from st t1, st t2 where t1.ts=t2.ts and t1.c_int <= t2.c_int;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1076616672134475760) + + tdSql.query("select count(t1.c_float_empty) from st t1, st t2 where t1.ts=t2.ts and t1.c_int = t2.c_int and t1.t_int_empty=t2.t_int_empty;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + def test_query_with_window(self): + # time window + tdSql.query("select sum(c_int_empty) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m) fill(value, 10);") + tdSql.checkRows(841) + tdSql.checkData(0, 0, 10) + + tdSql.query("select _wstart, _wend, sum(c_int) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m);") + tdSql.checkRows(65) + + # status window + tdSql.error("select _wstart, count(*) from st state_window(t_bool);") + tdSql.query("select _wstart, count(*) from st partition by tbname state_window(c_bool);") + tdSql.checkRows(5) + tdSql.checkData(0, 1, 10000) + + # session window + tdSql.query("select _wstart, count(*) from st partition by tbname, t_int session(ts, 1m);") + tdSql.checkRows(5) + tdSql.checkData(0, 1, 10000) + + # event window + tdSql.query("select _wstart, _wend, count(*) from (select * from st order by ts, tbname) event_window start with t_bool=true end with t_bool=false;") + tdSql.checkRows(20000) + + def test_query_with_union(self): + tdSql.query("select count(ts) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(ts) from (select * from ct1 union all select * from ct2 union all select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 30000) + + tdSql.query("select count(*) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(c_ts_empty) from (select * from ct1 union select * from ct2 union select * from ct3);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select count(*) from (select ts from st union select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10001) + + tdSql.query("select count(*) from (select ts from st union all select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 100000) + + tdSql.query("select count(ts) from (select ts from st union select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(ts) from (select ts from st union all select c_ts_empty from st);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 50000) + + def test_nested_query(self): + tdSql.query("select elapsed(ts, 1s) from (select * from (select * from st where c_int = 1) where c_int_empty is null);") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9999) + + tdSql.query("select first(ts) as t, avg(c_int) as v from (select * from (select * from st where c_int = 1) where c_int_empty is null) group by t_timestamp order by t_timestamp desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '2024-01-01 13:00:01.000') + tdSql.checkData(0, 1, 1) + + tdSql.query("select max(c_tinyint) from (select c_tinyint, tbname from st where c_float_empty is null or t_int_empty is null) group by tbname order by c_tinyint desc slimit 1 limit 1;") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select top(c_int, 3) from (select c_int, tbname from st where t_int in (2, 3)) group by tbname slimit 3;") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 1) + + def run(self): + self.prepareData() + self.test_query_with_filter() + self.test_query_with_groupby() + self.test_query_with_join() + self.test_query_with_window() + self.test_query_with_union() + self.test_nested_query() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 015fb4d679d9f825d2f99ab7ff28bf57d105ad2d Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 11:15:39 +0800 Subject: [PATCH 07/45] feat: support uniq grant --- source/libs/catalog/inc/catalogInt.h | 16 ++++- source/libs/catalog/src/ctgCache.c | 99 +++++++++++++++------------- 2 files changed, 68 insertions(+), 47 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index b9a0752178..d86825c216 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -320,14 +320,19 @@ typedef struct SCtgUserAuth { uint64_t userCacheSize; } SCtgUserAuth; +typedef struct SCtgGrantCache { + SRWLatch lock; + SGrantHbRsp grantInfo; + uint64_t grantCacheSize; +} SCtgGrantCache; + typedef struct SCatalog { uint64_t clusterId; bool stopUpdate; SDynViewVersion dynViewVer; SHashObj* userCache; // key:user, value:SCtgUserAuth SHashObj* dbCache; // key:dbname, value:SCtgDBCache - SHashObj* grantCache; - SGrantHbRsp _grantCache; + SCtgGrantCache grantCache; SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; @@ -377,6 +382,7 @@ typedef struct SCtgJob { int32_t tbCfgNum; int32_t svrVerNum; int32_t viewNum; + int32_t grantNum; } SCtgJob; typedef struct SCtgMsgCtx { @@ -560,6 +566,10 @@ typedef struct SCtgUpdateGrantInfoMsg { SGrantHbRsp* pRsp; } SCtgUpdateGrantInfoMsg; +typedef struct SCtgDropGrantInfoMsg { + SCatalog* pCtg; +} SCtgDropGrantInfoMsg; + typedef struct SCtgCacheOperation { int32_t opId; void* data; @@ -922,6 +932,7 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action); int32_t ctgOpDropStbMeta(SCtgCacheOperation* action); int32_t ctgOpDropTbMeta(SCtgCacheOperation* action); int32_t ctgOpDropViewMeta(SCtgCacheOperation* action); +int32_t ctgOpDropGrantInfo(SCtgCacheOperation* action); int32_t ctgOpUpdateUser(SCtgCacheOperation* action); int32_t ctgOpUpdateEpset(SCtgCacheOperation* operation); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char* dbFName, SCtgDBCache** pCache); @@ -969,6 +980,7 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation* operation); int32_t ctgOpUpdateTbIndex(SCtgCacheOperation* operation); int32_t ctgOpClearCache(SCtgCacheOperation* operation); int32_t ctgOpUpdateViewMeta(SCtgCacheOperation *operation); +int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation); int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char* tableName, int32_t* tbType); int32_t ctgGetTbHashVgroupFromCache(SCatalog* pCtg, const SName* pTableName, SVgroupInfo** pVgroup); int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsCtx *ctx, int32_t dbIdx, diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 05078fabb4..92bda82bd3 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -160,6 +160,16 @@ int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } +int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); +} + +void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { + CTG_UNLOCK(CTG_READ, &dbCache->dbLock); + taosHashRelease(pCtg->dbCache, dbCache); +} + + void ctgReleaseVgInfoToCache(SCatalog *pCtg, SCtgDBCache *dbCache) { ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); @@ -1512,6 +1522,47 @@ int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } +int32_t ctgGetAddGrantCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, dbFName, &dbCache); + + if (dbCache) { + // TODO OPEN IT +#if 0 + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#else + if (0 == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbId && (dbCache->dbId == 0)) { + dbCache->dbId = dbId; + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#endif + CTG_ERR_RET(ctgRemoveDBFromCache(pCtg, dbCache, dbFName)); + } + + CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); + + ctgGetDBCache(pCtg, dbFName, &dbCache); + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { @@ -2488,8 +2539,6 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { SCtgUpdateGrantInfoMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; SGrantHbRsp *pRsp = msg->pRsp; - SCtgDBCache *dbCache = NULL; - SViewMeta *pMeta = NULL; taosMemoryFreeClear(msg); @@ -2523,56 +2572,16 @@ _return: CTG_RET(code); } -int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) { +int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgDropViewMetaMsg *msg = operation->data; + SCtgDropGrantInfoMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; - int32_t tblType = 0; if (pCtg->stopUpdate) { goto _return; } - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); - if (NULL == dbCache) { - goto _return; - } - - if ((0 != msg->dbId) && (dbCache->dbId != msg->dbId)) { - ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", dbFName:%s, viewName:%s", msg->dbId, dbCache->dbId, - msg->dbFName, msg->viewName); - goto _return; - } - - SCtgViewCache *pViewCache = taosHashGet(dbCache->viewCache, msg->viewName, strlen(msg->viewName)); - if (NULL == pViewCache) { - ctgDebug("view %s already not in cache", msg->viewName); - goto _return; - } - - int64_t viewId = pViewCache->pMeta->viewId; - if (0 != msg->viewId && viewId != msg->viewId) { - ctgDebug("viewId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", viewName:%s", msg->viewId, viewId, msg->viewName); - goto _return; - } - - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pViewCache->pMeta)); - ctgFreeViewCacheImpl(pViewCache, true); - - if (taosHashRemove(dbCache->viewCache, msg->viewName, strlen(msg->viewName))) { - ctgError("view %s not exist in cache, dbFName:%s", msg->viewName, msg->dbFName); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } else { - atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(SCtgViewCache) + strlen(msg->viewName)); - CTG_DB_NUM_DEC(CTG_CI_VIEW); - } - - ctgDebug("view %s removed from cache, dbFName:%s", msg->viewName, msg->dbFName); - - CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->viewRent, viewId, ctgViewVersionSortCompare, ctgViewVersionSearchCompare)); - - ctgDebug("view %s removed from rent, dbFName:%s, viewId:0x%" PRIx64, msg->viewName, msg->dbFName, viewId); + printf("prop:%s:%d grant %s removed from rent\n"); _return: From 3236ef7bae72d50aa526701784fd05d619447d77 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Feb 2024 11:26:47 +0800 Subject: [PATCH 08/45] feat(stream): return the stream source task exec delay, and do some internal refactor. --- include/libs/executor/storageapi.h | 8 +-- include/libs/stream/tstream.h | 2 +- source/dnode/vnode/inc/vnode.h | 4 +- source/dnode/vnode/src/inc/tq.h | 7 +- source/dnode/vnode/src/inc/tsdb.h | 3 +- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/sma/smaTimeRange.c | 4 +- source/dnode/vnode/src/tq/tq.c | 3 +- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/tq/tqSink.c | 8 +-- source/dnode/vnode/src/tq/tqUtil.c | 72 ++++++++++++++++++- source/dnode/vnode/src/tsdb/tsdbRead2.c | 6 +- source/dnode/vnode/src/vnd/vnodeInitApi.c | 6 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 5 ++ .../executor/src/streameventwindowoperator.c | 1 - source/libs/parser/src/parTranslater.c | 58 +++++++++++---- source/libs/qworker/src/qworker.c | 7 ++ source/libs/stream/src/streamCheckpoint.c | 10 +-- source/libs/stream/src/streamQueue.c | 4 +- source/libs/wal/src/walRead.c | 11 ++- source/libs/wal/src/walWrite.c | 2 +- 21 files changed, 165 insertions(+), 60 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 669340f9e5..9987dab166 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -211,6 +211,7 @@ typedef struct SStoreTqReader { bool (*tqNextBlockImpl)(); // todo remove it SSDataBlock* (*tqGetResultBlock)(); int64_t (*tqGetResultBlockTime)(); + int32_t (*tqGetStreamExecProgress)(); void (*tqReaderSetColIdList)(); int32_t (*tqReaderSetQueryTableList)(); @@ -266,16 +267,11 @@ typedef struct SStoreMeta { // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list); int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); - void* storeGetVersionRange; - void* storeGetLastTimestamp; - - int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema + int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols); void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, int64_t* numOfNormalTables); - int64_t (*getNumOfRowsInMem)(void* pVnode); - SMCtbCursor* (*openCtbCursor)(void* pVnode, tb_uid_t uid, int lock); int32_t (*resumeCtbCursor)(SMCtbCursor* pCtbCur, int8_t first); void (*pauseCtbCursor)(SMCtbCursor* pCtbCur); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9738be839d..dce8fffe11 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -313,7 +313,7 @@ typedef struct SCheckpointInfo { int64_t failedId; // record the latest failed checkpoint id int64_t checkpointingId; int32_t downstreamAlignNum; - int32_t checkpointNotReadyTasks; + int32_t numOfNotReady; bool dispatchCheckpointTrigger; int64_t msgVer; int32_t transId; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 97cf0ffebc..3c334be2f2 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -90,6 +90,8 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num); int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num); int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num); +int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); + void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeGetLoadLite(SVnode *pVnode, SVnodeLoadLite *pLoad); @@ -180,7 +182,6 @@ int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, SArray *pTableUids); void *tsdbCacherowsReaderClose(void *pReader); -int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); size_t tsdbCacheGetCapacity(SVnode *pVnode); @@ -233,6 +234,7 @@ int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, i bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); +int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, bool* fhFinished); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index cded4ddd7c..475a26aff5 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -97,7 +97,6 @@ typedef struct { struct STQ { SVnode* pVnode; char* path; - int64_t walLogLastVer; SRWLatch lock; SHashObj* pPushMgr; // subKey -> STqHandle SHashObj* pHandle; // subKey -> STqHandle @@ -153,14 +152,14 @@ char* tqOffsetBuildFName(const char* path, int32_t fVer); int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); // tq util -int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); +int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever); int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset); void tqUpdateNodeStage(STQ* pTq, bool isLeader); -int32_t setDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, - SSubmitTbData* pTableData, const char* id); +int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, + SSubmitTbData* pTableData, const char* id); int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id); SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 9d8d5013fa..cac3be9ee3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -279,6 +279,7 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); void tsdbUntakeReadSnap2(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); +int32_t tsdbGetTableSchema(SMeta* pMeta, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbMerge.c ============================================================================================== typedef struct { @@ -970,8 +971,6 @@ static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { return pIter->pRow; } -int32_t tRowInfoCmprFn(const void *p1, const void *p2); - typedef struct { int64_t suid; int64_t uid; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 138bcbb133..621651507e 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -1554,7 +1554,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA } _resume_delete: version = RSMA_EXEC_MSG_VER(msg); - if ((terrno = extractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version, + if ((terrno = tqExtractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version, &packData.pDataBlock, 1))) { taosFreeQitem(msg); goto _err; diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index f537ede8c1..767ea47e21 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -203,7 +203,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * int32_t *index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId)); if (index == NULL) { // no data yet, append it - code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, ""); + code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, ""); if (code != TSDB_CODE_SUCCESS) { continue; } @@ -213,7 +213,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * int32_t size = (int32_t)taosArrayGetSize(pReq->aSubmitTbData) - 1; taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); } else { - code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, ""); + code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, ""); if (code != TSDB_CODE_SUCCESS) { continue; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8689c30a55..bde6889ecd 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -66,7 +66,6 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { pTq->path = taosStrdup(path); pTq->pVnode = pVnode; - pTq->walLogLastVer = pVnode->pWal->vers.lastVer; pTq->pHandle = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); taosHashSetFreeFp(pTq->pHandle, tqDestroyTqHandle); @@ -1055,7 +1054,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { int32_t code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); // let's continue scan data in the wal files - if(code == 0 && pReq->reqType >= 0){ + if (code == 0 && (pReq->reqType >= 0 || pReq->reqType == STREAM_EXEC_T_RESUME_TASK)) { tqScanWalAsync(pTq, false); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 383a636f71..8392f4c479 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -344,7 +344,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con void* pBody = POINTER_SHIFT(pCont->body, sizeof(SMsgHead)); int32_t len = pCont->bodyLen - sizeof(SMsgHead); - code = extractDelDataBlock(pBody, len, ver, (void**)pItem, 0); + code = tqExtractDelDataBlock(pBody, len, ver, (void**)pItem, 0); if (code == TSDB_CODE_SUCCESS) { if (*pItem == NULL) { tqDebug("s-task:%s empty delete msg, discard it, len:%d, ver:%" PRId64, id, len, ver); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 7fcb86d84a..7050870c57 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -746,7 +746,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat return TDB_CODE_SUCCESS; } -int32_t setDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, +int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, SSubmitTbData* pTableData, const char* id) { int32_t numOfRows = pDataBlock->info.rows; @@ -821,7 +821,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { continue; } - code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id); + code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id); if (code != TSDB_CODE_SUCCESS) { continue; } @@ -868,7 +868,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { continue; } - code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id); + code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id); if (code != TSDB_CODE_SUCCESS) { continue; } @@ -878,7 +878,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); } else { - code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id); + code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id); if (code != TSDB_CODE_SUCCESS) { continue; } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index d18455d221..b9f578a74b 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -388,7 +388,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* return 0; } -int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type) { +int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type) { SDecoder* pCoder = &(SDecoder){0}; SDeleteRes* pRes = &(SDeleteRes){0}; @@ -449,3 +449,73 @@ int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** return TSDB_CODE_SUCCESS; } + +int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, bool* fhFinished) { + SStreamMeta* pMeta = pVnode->pTq->pStreamMeta; + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + int32_t code = TSDB_CODE_SUCCESS; + + if (pDelay != NULL) { + *pDelay = 0; + } + + *fhFinished = false; + + if (numOfTasks <= 0) { + return code; + } + + // extract the required source task for a given stream, identified by streamId + for (int32_t i = 0; i < numOfTasks; ++i) { + STaskId* pId = taosArrayGet(pMeta->pTaskList, i); + if (pId->streamId != streamId) { + continue; + } + + SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId)); + if (ppTask == NULL) { + tqError("vgId:%d failed to acquire task:0x%" PRIx64 " in retrieving progress", pMeta->vgId, pId->taskId); + continue; + } + + if ((*ppTask)->info.taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + // here we get the required stream source task + SStreamTask* pTask = *ppTask; + *fhFinished = !HAS_RELATED_FILLHISTORY_TASK(pTask); + + int64_t ver = walReaderGetCurrentVer(pTask->exec.pWalReader); + + SVersionRange verRange = {0}; + walReaderValidVersionRange(pTask->exec.pWalReader, &verRange.minVer, &verRange.maxVer); + + SWalReader* pReader = walOpenReader(pTask->exec.pWalReader->pWal, NULL, 0); + if (pReader == NULL) { + tqError("failed to open wal reader to extract exec progress, vgId:%d", pMeta->vgId); + continue; + } + + int64_t cur = 0; + int64_t latest = 0; + + code = walFetchHead(pReader, ver); + if (code != TSDB_CODE_SUCCESS) { + cur = pReader->pHead->head.ingestTs; + } + + code = walFetchHead(pReader, verRange.maxVer); + if (code != TSDB_CODE_SUCCESS) { + latest = pReader->pHead->head.ingestTs; + } + + if (pDelay != NULL) { // delay in ms + *pDelay = (latest - cur) / 1000; + } + + walCloseReader(pReader); + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 9d158668d2..d9b932a367 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4995,9 +4995,9 @@ int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) { return rows; } -int32_t tsdbGetTableSchema(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { +int32_t tsdbGetTableSchema(SMeta* pMeta, int64_t uid, STSchema** pSchema, int64_t* suid) { SMetaReader mr = {0}; - metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0); + metaReaderDoInit(&mr, pMeta, 0); int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; @@ -5027,7 +5027,7 @@ int32_t tsdbGetTableSchema(void* pVnode, int64_t uid, STSchema** pSchema, int64_ metaReaderClear(&mr); // get the newest table schema version - code = metaGetTbTSchemaEx(((SVnode*)pVnode)->pMeta, *suid, uid, -1, pSchema); + code = metaGetTbTSchemaEx(pMeta, *suid, uid, -1, pSchema); return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index c323a81093..2392716bbf 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -91,7 +91,7 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getTableTypeByName = metaGetTableTypeByName; pMeta->getTableNameByUid = metaGetTableNameByUid; - pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor + pMeta->getTableSchema = vnodeGetTableSchema; pMeta->storeGetTableList = vnodeGetTableList; pMeta->getCachedTableList = metaGetCachedTableUidList; @@ -135,7 +135,9 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; pTq->tqGetResultBlockTime = tqGetResultBlockTime; -} + + pTq->tqGetStreamExecProgress = tqGetStreamExecInfo; + } void initStateStoreAPI(SStateStore* pStore) { pStore->streamFileStateInit = streamFileStateInit; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index b6a9360afd..4fc7a88494 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -14,6 +14,7 @@ */ #include "vnd.h" +#include "tsdb.h" #define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \ do { \ @@ -703,3 +704,7 @@ void *vnodeGetIvtIdx(void *pVnode) { } return metaGetIvtIdx(((SVnode *)pVnode)->pMeta); } + +int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid) { + return tsdbGetTableSchema(((SVnode*)pVnode)->pMeta, uid, pSchema, suid); +} diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 8aca76597b..0602016268 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -725,7 +725,6 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys } if (pInfo->isHistoryOp) { - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pAllUpdated = tSimpleHashInit(64, hashFn); } else { pInfo->pAllUpdated = NULL; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index e221d2158c..0047fdb514 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8090,27 +8090,27 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta tstrncpy(col->tableAlias, pTable, tListLen(col->tableAlias)); tstrncpy(col->colName, pMeta->schema[0].name, tListLen(col->colName)); - SNodeList* pParamterList = nodesMakeList(); - if (NULL == pParamterList) { + SNodeList* pParameterList = nodesMakeList(); + if (NULL == pParameterList) { nodesDestroyNode((SNode*)col); return TSDB_CODE_OUT_OF_MEMORY; } - int32_t code = nodesListStrictAppend(pParamterList, (SNode*)col); + int32_t code = nodesListStrictAppend(pParameterList, (SNode*)col); if (code) { - nodesDestroyList(pParamterList); + nodesDestroyList(pParameterList); return code; } - SNode* pFunc = (SNode*)createFunction("last", pParamterList); + SNode* pFunc = (SNode*)createFunction("last", pParameterList); if (NULL == pFunc) { - nodesDestroyList(pParamterList); + nodesDestroyList(pParameterList); return TSDB_CODE_OUT_OF_MEMORY; } SNodeList* pProjectionList = nodesMakeList(); if (NULL == pProjectionList) { - nodesDestroyList(pParamterList); + nodesDestroyNode(pFunc); return TSDB_CODE_OUT_OF_MEMORY; } @@ -8122,7 +8122,7 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta SFunctionNode* pFunc1 = createFunction("_vgid", NULL); if (NULL == pFunc1) { - nodesDestroyList(pParamterList); + nodesDestroyList(pProjectionList); return TSDB_CODE_OUT_OF_MEMORY; } @@ -8135,7 +8135,7 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta SFunctionNode* pFunc2 = createFunction("_vgver", NULL); if (NULL == pFunc2) { - nodesDestroyList(pParamterList); + nodesDestroyList(pProjectionList); return TSDB_CODE_OUT_OF_MEMORY; } @@ -8152,24 +8152,54 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta return code; } - // todo add the group by statement SSelectStmt** pSelect1 = (SSelectStmt**)pQuery; (*pSelect1)->pGroupByList = nodesMakeList(); + if (NULL == (*pSelect1)->pGroupByList) { + return TSDB_CODE_OUT_OF_MEMORY; + } SGroupingSetNode* pNode1 = (SGroupingSetNode*)nodesMakeNode(QUERY_NODE_GROUPING_SET); + if (NULL == pNode1) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pNode1->groupingSetType = GP_TYPE_NORMAL; pNode1->pParameterList = nodesMakeList(); - nodesListAppend(pNode1->pParameterList, (SNode*)pFunc1); + if (NULL == pNode1->pParameterList) { + nodesDestroyNode((SNode*)pNode1); + return TSDB_CODE_OUT_OF_MEMORY; + } - nodesListAppend((*pSelect1)->pGroupByList, (SNode*)pNode1); + code = nodesListAppend(pNode1->pParameterList, (SNode*)pFunc1); + if (code) { + nodesDestroyNode((SNode*)pNode1); + return code; + } + + code = nodesListAppend((*pSelect1)->pGroupByList, (SNode*)pNode1); + if (code) { + return code; + } SGroupingSetNode* pNode2 = (SGroupingSetNode*)nodesMakeNode(QUERY_NODE_GROUPING_SET); + if (NULL == pNode2) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pNode2->groupingSetType = GP_TYPE_NORMAL; pNode2->pParameterList = nodesMakeList(); - nodesListAppend(pNode2->pParameterList, (SNode*)pFunc2); + if (NULL == pNode2->pParameterList) { + nodesDestroyNode((SNode*)pNode1); + return TSDB_CODE_OUT_OF_MEMORY; + } - nodesListAppend((*pSelect1)->pGroupByList, (SNode*)pNode2); + code = nodesListAppend(pNode2->pParameterList, (SNode*)pFunc2); + if (code) { + nodesDestroyNode((SNode*)pNode2); + return code; + } + code = nodesListAppend((*pSelect1)->pGroupByList, (SNode*)pNode2); return code; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 7376aa3a9c..93559745be 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -737,6 +737,13 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(code); } +#if 0 + SReadHandle* pReadHandle = qwMsg->node; + int64_t delay = 0; + bool fhFinish = false; + pReadHandle->api.tqReaderFn.tqGetStreamExecProgress(pReadHandle->vnode, 0, &delay, &fhFinish); +#endif + code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, sql, OPTR_EXEC_MODEL_BATCH); sql = NULL; if (code) { diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index f45904f036..b1783fb640 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -158,7 +158,7 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo pTask->chkInfo.transId = pReq->transId; pTask->chkInfo.checkpointingId = pReq->checkpointId; - pTask->chkInfo.checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); + pTask->chkInfo.numOfNotReady = streamTaskGetNumOfDownstream(pTask); pTask->chkInfo.startTs = taosGetTimestampMs(); pTask->execInfo.checkpoint += 1; @@ -214,7 +214,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); continueDispatchCheckpointBlock(pBlock, pTask); } else { // only one task exists, no need to dispatch downstream info - atomic_add_fetch_32(&pTask->chkInfo.checkpointNotReadyTasks, 1); + atomic_add_fetch_32(&pTask->chkInfo.numOfNotReady, 1); streamProcessCheckpointReadyMsg(pTask); streamFreeQitem((SStreamQueueItem*)pBlock); } @@ -249,7 +249,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc // set the needed checked downstream tasks, only when all downstream tasks do checkpoint complete, this task // can start local checkpoint procedure - pTask->chkInfo.checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); + pTask->chkInfo.numOfNotReady = streamTaskGetNumOfDownstream(pTask); // Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task // already. And then, dispatch check point msg to all downstream tasks @@ -268,7 +268,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE || pTask->info.taskLevel == TASK_LEVEL__AGG); // only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task - int32_t notReady = atomic_sub_fetch_32(&pTask->chkInfo.checkpointNotReadyTasks, 1); + int32_t notReady = atomic_sub_fetch_32(&pTask->chkInfo.numOfNotReady, 1); ASSERT(notReady >= 0); if (notReady == 0) { @@ -287,7 +287,7 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { pTask->chkInfo.checkpointingId = 0; // clear the checkpoint id pTask->chkInfo.failedId = 0; pTask->chkInfo.startTs = 0; // clear the recorded start time - pTask->chkInfo.checkpointNotReadyTasks = 0; + pTask->chkInfo.numOfNotReady = 0; pTask->chkInfo.transId = 0; pTask->chkInfo.dispatchCheckpointTrigger = false; diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 78929c365e..0936d410bf 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -155,14 +155,14 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu *blockSize = 0; // no available token in bucket for sink task, let's wait for a little bit - if (taskLevel == TASK_LEVEL__SINK && (!streamTaskExtractAvailableToken(pTask->outputInfo.pTokenBucket, pTask->id.idStr))) { + if (taskLevel == TASK_LEVEL__SINK && (!streamTaskExtractAvailableToken(pTask->outputInfo.pTokenBucket, id))) { stDebug("s-task:%s no available token in bucket for sink data, wait for 10ms", id); return TSDB_CODE_SUCCESS; } while (1) { if (streamTaskShouldPause(pTask) || streamTaskShouldStop(pTask)) { - stDebug("s-task:%s task should pause, extract input blocks:%d", pTask->id.idStr, *numOfBlocks); + stDebug("s-task:%s task should pause, extract input blocks:%d", id, *numOfBlocks); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 3854e90901..d491b00e73 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -70,10 +70,9 @@ int32_t walNextValidMsg(SWalReader *pReader) { int64_t committedVer = walGetCommittedVer(pReader->pWal); int64_t appliedVer = walGetAppliedVer(pReader->pWal); - wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 - ", applied index:%" PRId64, + wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64, pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer); - if (fetchVer > appliedVer){ + if (fetchVer > appliedVer) { terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; return -1; } @@ -86,10 +85,8 @@ int32_t walNextValidMsg(SWalReader *pReader) { int32_t type = pReader->pHead->head.msgType; if (type == TDMT_VND_SUBMIT || ((type == TDMT_VND_DELETE) && (pReader->cond.deleteMsg == 1)) || (IS_META_MSG(type) && pReader->cond.scanMeta)) { - if (walFetchBody(pReader) < 0) { - return -1; - } - return 0; + int32_t code = walFetchBody(pReader); + return (code == TSDB_CODE_SUCCESS)? 0:-1; } else { if (walSkipFetchBody(pReader) < 0) { return -1; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 341d989f8f..9783705bad 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -498,7 +498,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.head.version = index; pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.msgType = msgType; - pWal->writeHead.head.ingestTs = 0; + pWal->writeHead.head.ingestTs = taosGetTimestampUs(); // sync info for sync module pWal->writeHead.head.syncMeta = syncMeta; From a4524802497c61c170495bc102f8d9baeb09ecbf Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 12:28:51 +0800 Subject: [PATCH 09/45] feat: support uniq grant --- source/libs/catalog/src/ctgCache.c | 39 +++--------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 92bda82bd3..fc471d2513 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -161,14 +161,12 @@ int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) } int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { - CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); -} - -void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { - CTG_UNLOCK(CTG_READ, &dbCache->dbLock); - taosHashRelease(pCtg->dbCache, dbCache); + CTG_LOCK(CTG_READ, &pCtg->grantCache.lock); + *ppCache = &pCtg->grantCache; + CTG_CACHE_HIT_INC(CTG_CI_GRANT_INFO, 1); } +void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { CTG_UNLOCK(CTG_READ, &pCache->lock); } void ctgReleaseVgInfoToCache(SCatalog *pCtg, SCtgDBCache *dbCache) { ctgRUnlockVgInfo(dbCache); @@ -1527,36 +1525,7 @@ int32_t ctgGetAddGrantCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, dbFName, &dbCache); - if (dbCache) { - // TODO OPEN IT -#if 0 - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#else - if (0 == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - if (dbId && (dbCache->dbId == 0)) { - dbCache->dbId = dbId; - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#endif - CTG_ERR_RET(ctgRemoveDBFromCache(pCtg, dbCache, dbFName)); - } - - CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); - - ctgGetDBCache(pCtg, dbFName, &dbCache); *pCache = dbCache; From 3f061a8460c9c1e64b9d90a16deece217a440ac4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 13:44:33 +0800 Subject: [PATCH 10/45] feat: support uniq grant --- include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 1 + source/libs/catalog/src/ctgCache.c | 54 ++++++++++++++---------------- source/libs/catalog/src/ctgRent.c | 14 ++++++++ source/libs/catalog/src/ctgUtil.c | 19 +++++++++++ 5 files changed, 61 insertions(+), 28 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index dfbdc5a9f4..524f8864f3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1887,6 +1887,7 @@ int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); void tFreeSViewHbRsp(SViewHbRsp* pRsp); typedef struct { + int32_t version; uint32_t flags; } SGrantHbRsp; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 2e013413a3..ba59ab85e9 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -146,6 +146,7 @@ typedef struct SSTableVersion { } SSTableVersion; typedef struct SGrantVersion { + int32_t grantId; int32_t version; } SGrantVersion; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index fc471d2513..800e1edb80 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1520,18 +1520,6 @@ int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } -int32_t ctgGetAddGrantCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { - int32_t code = 0; - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, dbFName, &dbCache); - - - - *pCache = dbCache; - - return TSDB_CODE_SUCCESS; -} - int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { @@ -1743,6 +1731,30 @@ _return: CTG_RET(code); } +int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { + int32_t code = TSDB_CODE_SUCCESS; + SCtgGrantCache *pCache = &pCtg->grantCache; + + CTG_LOCK(CTG_WRITE, &pCache->lock); + pCache->grantInfo = *pRsp; + CTG_UNLOCK(CTG_WRITE, &pCache->lock); + + ctgDebug("grant info updated to cache, version"); + + CTG_ERR_RET(ctgUpdateRentViewVersion(pCtg, dbFName, viewName, dbCache->dbId, pMeta->viewId, pCache)); + + pMeta = NULL; + +_return: + + if (pMeta) { + ctgFreeSViewMeta(pMeta); + taosMemoryFree(pMeta); + } + + CTG_RET(code); +} + int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool syncReq) { STableMetaOutput *pOutput = NULL; int32_t code = 0; @@ -2508,6 +2520,7 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { SCtgUpdateGrantInfoMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; SGrantHbRsp *pRsp = msg->pRsp; + SCtgGrantCache *pGrantCache = NULL; taosMemoryFreeClear(msg); @@ -2515,29 +2528,14 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pRsp->dbFName, pRsp->dbId, &dbCache)); - if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, pRsp->dbFName, pRsp->dbId); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - pMeta = taosMemoryCalloc(1, sizeof(SViewMeta)); - if (NULL == pMeta) { - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - - CTG_ERR_JRET(dupViewMetaFromRsp(pRsp, pMeta)); + CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); code = ctgWriteViewMetaToCache(pCtg, dbCache, pRsp->dbFName, pRsp->name, pMeta); pMeta = NULL; _return: - tFreeSViewMetaRsp(pRsp); taosMemoryFree(pRsp); - ctgFreeSViewMeta(pMeta); - taosMemoryFree(pMeta); - CTG_RET(code); } diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 457285b147..799c76d5e5 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,6 +301,20 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } +int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, + SCtgGrantCache *pCache) { + SGrantVersion metaRent = {.grantId = grantId, .version = pCache->grantInfo.version}; + + + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), + ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); + + ctgDebug("grant %d version %d updated to grantRent", grantId, metaRent.version); + + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index cacdfcb8d6..e157028bee 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1328,6 +1328,16 @@ int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2) { } } +int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2) { + if (*(int32_t*)key1 == ((SGrantVersion*)key2)->grantId) { + return 0; + } else if (*(int32_t*)key1 < ((SGrantVersion*)key2)->grantId) { + return -1; + } else { + return 1; + } +} + int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) { @@ -1359,6 +1369,15 @@ int32_t ctgViewVersionSortCompare(const void* key1, const void* key2) { } } +int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2) { + if (((SGrantVersion*)key1)->grantId == ((SGrantVersion*)key2)->grantId) { + return 0; + } else if (((SGrantVersion*)key1)->grantId < ((SGrantVersion*)key2)->grantId) { + return -1; + } else { + return 1; + } +} int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { if (NULL == dbInfo) { From 3e7884a4f789efe75b121c9a30afc68dd90045cd Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 13:59:45 +0800 Subject: [PATCH 11/45] feat: support uniq grant --- include/libs/catalog/catalog.h | 2 +- source/libs/catalog/inc/catalogInt.h | 3 +++ source/libs/catalog/src/ctgCache.c | 12 +++++++----- source/libs/catalog/src/ctgRent.c | 9 +-------- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index ba59ab85e9..914ecb64ff 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -146,7 +146,7 @@ typedef struct SSTableVersion { } SSTableVersion; typedef struct SGrantVersion { - int32_t grantId; + int64_t grantId; int32_t version; } SGrantVersion; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index d86825c216..4d759d6bd7 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -568,6 +568,7 @@ typedef struct SCtgUpdateGrantInfoMsg { typedef struct SCtgDropGrantInfoMsg { SCatalog* pCtg; + int64_t grantId; } SCtgDropGrantInfoMsg; typedef struct SCtgCacheOperation { @@ -1040,10 +1041,12 @@ void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); int32_t ctgViewVersionSortCompare(const void* key1, const void* key2); +int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2); +int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 800e1edb80..0f93ba317a 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1739,7 +1739,7 @@ int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { pCache->grantInfo = *pRsp; CTG_UNLOCK(CTG_WRITE, &pCache->lock); - ctgDebug("grant info updated to cache, version"); + ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); CTG_ERR_RET(ctgUpdateRentViewVersion(pCtg, dbFName, viewName, dbCache->dbId, pMeta->viewId, pCache)); @@ -2540,15 +2540,18 @@ _return: } int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgDropGrantInfoMsg *msg = operation->data; - SCatalog *pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; if (pCtg->stopUpdate) { goto _return; } - printf("prop:%s:%d grant %s removed from rent\n"); + CTG_ERR_JRET( + ctgMetaRentRemove(&pCtg->grantRent, msg->grantId, ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); + + printf("prop:grant:0x%" PRIx64 "removed from rent", msg->grantId); _return: @@ -2557,7 +2560,6 @@ _return: CTG_RET(code); } - void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 799c76d5e5..9ef1cf5928 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,11 +301,9 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } -int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, - SCtgGrantCache *pCache) { +int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SCtgGrantCache *pCache) { SGrantVersion metaRent = {.grantId = grantId, .version = pCache->grantInfo.version}; - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); @@ -313,8 +311,3 @@ int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, return TSDB_CODE_SUCCESS; } - - - - - From 2ac374b2e01b300ef1b5a08106bf365ff31e14f3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 2 Feb 2024 14:02:13 +0800 Subject: [PATCH 12/45] max delay --- .../executor/src/streameventwindowoperator.c | 1 + .../executor/src/streamtimewindowoperator.c | 2 + tests/script/tsim/stream/windowClose.sim | 50 +++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 8aca76597b..d1138afb65 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -349,6 +349,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 02f8b90864..14fabaecea 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2083,6 +2083,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); @@ -3425,6 +3426,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index 67678963ea..e4b8fc7d84 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -290,6 +290,56 @@ if $data32 != $now32 then return -1 endi +print step 1 max delay 2s +sql create database test3 vgroups 4; +sql use test3; +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql create stream stream13 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 session(ts, 10s); + +sleep 1000 + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791223001,2,2,3,1.1); + +$loop_count = 0 + +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt13; + +if $rows != 2 then + print ======rows=$rows + goto loop2 +endi + +$now02 = $data02 +$now12 = $data12 + + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + print ======over system sh/exec.sh -n dnode1 -s stop -x SIGINT From 54f216b6c635453261d8d26c666c3bb34d6cb113 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 2 Feb 2024 15:19:58 +0800 Subject: [PATCH 13/45] add ci --- .../executor/src/streamtimewindowoperator.c | 4 + tests/script/tsim/stream/windowClose.sim | 161 +++++++++++++++++- 2 files changed, 160 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 14fabaecea..f26ff7156b 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2287,6 +2287,10 @@ int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { int32_t iter = 0; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { SResultWindowInfo* pWinInfo = pIte; + if (!pWinInfo->pStatePos->beUpdated) { + continue; + } + pWinInfo->pStatePos->beUpdated = false; saveResult(*pWinInfo, pStUpdated); } return TSDB_CODE_SUCCESS; diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index e4b8fc7d84..775ff81f51 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -290,17 +290,17 @@ if $data32 != $now32 then return -1 endi -print step 1 max delay 2s -sql create database test3 vgroups 4; -sql use test3; +print step 2 max delay 2s +sql create database test15 vgroups 4; +sql use test15; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream stream13 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 session(ts, 10s); +sql create stream stream15 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 session(ts, 10s); sleep 1000 sql insert into t1 values(1648791213000,1,2,3,1.0); -sql insert into t1 values(1648791223001,2,2,3,1.1); +sql insert into t1 values(1648791233001,2,2,3,1.1); $loop_count = 0 @@ -330,6 +330,157 @@ sleep 3000 sql select * from streamt13; +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +print step 3 max delay 2s +sql create database test16 vgroups 4; +sql use test16; +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql create stream stream16 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 state_window(a, 10s); + +sleep 1000 + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791233001,2,2,3,1.1); + +$loop_count = 0 + +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt13; + +if $rows != 2 then + print ======rows=$rows + goto loop2 +endi + +$now02 = $data02 +$now12 = $data12 + + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +print step 4 max delay 2s +sql create database test17 vgroups 4; +sql use test17; +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql create stream stream17 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 event_window start with a = 0 end with a = 9; + +sleep 1000 + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791213001,9,2,3,1.0); + +sql insert into t1 values(1648791233001,1,2,3,1.1); +sql insert into t1 values(1648791233009,9,2,3,1.1); + +$loop_count = 0 + +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt13; + +if $rows != 2 then + print ======rows=$rows + goto loop2 +endi + +$now02 = $data02 +$now12 = $data12 + + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + if $data02 != $now02 then print ======data02=$data02 return -1 From 092fc8d0110bb834968bb56ad703d2d50d8b27cf Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 3 Feb 2024 12:10:37 +0800 Subject: [PATCH 14/45] feat: support uniq grant --- include/util/tdef.h | 1 + source/common/src/systable.c | 16 ++++++++-------- source/common/src/tmsg.c | 2 ++ source/libs/catalog/inc/catalogInt.h | 4 +++- source/libs/catalog/src/ctgCache.c | 15 ++++----------- source/libs/catalog/src/ctgRent.c | 4 ++-- 6 files changed, 20 insertions(+), 22 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 4698d50e67..f136005026 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -287,6 +287,7 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_VALUE_LEN 256 #define TSDB_CLUSTER_VALUE_LEN 1000 +#define TSDB_GRANT_LOG_COL_LEN 15072 #define TSDB_ACTIVE_KEY_LEN 109 #define TSDB_CONN_ACTIVE_KEY_LEN 255 diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 77083d0425..47eac317ec 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -349,21 +349,21 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = { }; static const SSysDbTableSchema useGrantsFullSchema[] = { - {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema useGrantsLogsSchema[] = { - {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "machine", .bytes = 9088 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema useMachinesSchema[] = { {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "machine", .bytes = 6016 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysTableMeta infosMeta[] = { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index df62e8d8fc..72c17756b5 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9341,6 +9341,7 @@ int32_t tSerializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32v(&encoder, pRsp->version) < 0) return -1; if (tEncodeU32v(&encoder, pRsp->flags) < 0) return -1; tEndEncode(&encoder); @@ -9356,6 +9357,7 @@ int32_t tDeserializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32v(&decoder, &pRsp->version) < 0) return -1; if (tDecodeU32v(&decoder, &pRsp->flags) < 0) return -1; tEndDecode(&decoder); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 4d759d6bd7..b880c0a155 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -39,6 +39,7 @@ extern "C" { #define CTG_MAX_COMMAND_LEN 512 #define CTG_DEFAULT_CACHE_MON_MSEC 5000 #define CTG_CLEAR_CACHE_ROUND_TB_NUM 3000 +#define CGT_GRANT_ID 0 #define CTG_RENT_SLOT_SECOND 1.5 @@ -968,7 +969,8 @@ void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache); int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid, SCtgTbCache *pCache); int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, - SCtgViewCache *pCache); + SCtgViewCache *pCache); +int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 0f93ba317a..3b8a18e908 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -164,6 +164,7 @@ int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { CTG_LOCK(CTG_READ, &pCtg->grantCache.lock); *ppCache = &pCtg->grantCache; CTG_CACHE_HIT_INC(CTG_CI_GRANT_INFO, 1); + return TSDB_CODE_SUCCESS; } void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { CTG_UNLOCK(CTG_READ, &pCache->lock); } @@ -1741,17 +1742,10 @@ int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); - CTG_ERR_RET(ctgUpdateRentViewVersion(pCtg, dbFName, viewName, dbCache->dbId, pMeta->viewId, pCache)); - - pMeta = NULL; + CTG_ERR_RET(ctgUpdateRentGrantVersion(pCtg, CGT_GRANT_ID, pRsp)); _return: - if (pMeta) { - ctgFreeSViewMeta(pMeta); - taosMemoryFree(pMeta); - } - CTG_RET(code); } @@ -2528,10 +2522,9 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); + // CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); // TODO: inc/dec or cacheSize ? - code = ctgWriteViewMetaToCache(pCtg, dbCache, pRsp->dbFName, pRsp->name, pMeta); - pMeta = NULL; + code = ctgWriteGrantInfoToCache(pCtg, pRsp); _return: diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 9ef1cf5928..f757bd856f 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,8 +301,8 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } -int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SCtgGrantCache *pCache) { - SGrantVersion metaRent = {.grantId = grantId, .version = pCache->grantInfo.version}; +int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SGrantHbRsp *pGrant) { + SGrantVersion metaRent = {.grantId = grantId, .version = pGrant->version}; CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); From 3a0892631978a8126cf6b2531342503393b43511 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 3 Feb 2024 17:32:02 +0800 Subject: [PATCH 15/45] feat: support uniq grant --- include/common/tmsg.h | 2 +- include/libs/catalog/catalog.h | 3 +- source/client/src/clientHb.c | 54 ++++++++++++++++++++++-- source/dnode/mnode/impl/src/mndProfile.c | 4 +- source/libs/catalog/inc/catalogInt.h | 8 ++-- source/libs/catalog/src/catalog.c | 17 ++++++++ source/libs/catalog/src/ctgCache.c | 7 --- source/libs/catalog/src/ctgRent.c | 10 ----- source/libs/catalog/src/ctgUtil.c | 19 --------- 9 files changed, 76 insertions(+), 48 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 524f8864f3..96bf9677b7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -110,7 +110,7 @@ enum { HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_DYN_VIEW, HEARTBEAT_KEY_VIEWINFO, - HEARTBEAT_KEY_GRANT, + HEARTBEAT_KEY_GRANTINFO, }; typedef enum _mgmt_table { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 914ecb64ff..16deda205a 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -146,7 +146,6 @@ typedef struct SSTableVersion { } SSTableVersion; typedef struct SGrantVersion { - int64_t grantId; int32_t version; } SGrantVersion; @@ -343,6 +342,8 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion** stables, u int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* num, SDynViewVersion** dynViewVersion); +int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num); + int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* num); int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 755ab0eb91..6972ca3c38 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -342,7 +342,8 @@ static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCata return -1; } - tscInfo("hb to update grant info:%u", hbRsp->flags); + tscDebug("hb to update grant info, version:%d, flags:%u", hbRsp->version, hbRsp->flags); + catalogUpdateGrantInfo(pCatalog, hbRsp); return TSDB_CODE_SUCCESS; @@ -398,11 +399,12 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p hbProcessViewInfoRsp(kv->value, kv->valueLen, pCatalog); break; } - case HEARTBEAT_KEY_GRANT: { + case HEARTBEAT_KEY_GRANTINFO: { if (kv->valueLen <= 0 || NULL == kv->value) { tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); break; } + assert(0); hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); break; @@ -872,7 +874,7 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S view->version = htonl(view->version); } - tscDebug("hb got %d expired view, valueLen:%lu", viewNum, sizeof(SViewVersion) * viewNum); + tscDebug("hb got %u expired view, valueLen:%lu", viewNum, sizeof(SViewVersion) * viewNum); if (NULL == req->info) { req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); @@ -895,6 +897,47 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S return TSDB_CODE_SUCCESS; } +int32_t hbGetExpiredGrantInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SGrantVersion *grants = NULL; + uint32_t grantNum = 0; + int32_t code = 0; + + code = catalogGetExpiredGrants(pCatalog, &grants, &grantNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (grantNum <= 0) { + taosMemoryFree(grants); + return code; + } + + for (int32_t i = 0; i < grantNum; ++i) { + SGrantVersion *gv = &grants[i]; + gv->version = htonl(gv->version); + } + + tscDebug("hb got %d expired grant, valueLen:%u", grantNum, (int32_t)sizeof(SGrantVersion) * grantNum); + + if (!req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + if (!req->info) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(grants); + return code; + taosMemoryFree(grants); + } + } + + SKv kv = { + .key = HEARTBEAT_KEY_GRANTINFO, + .valueLen = sizeof(SGrantVersion) * grantNum, + .value = grants, + }; + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); @@ -958,6 +1001,11 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req if (TSDB_CODE_SUCCESS != code) { return code; } + + code = hbGetExpiredGrantInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } #endif } else { req->app.appId = 0; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d7e6d9609c..2d2d12a668 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -606,12 +606,12 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } - case HEARTBEAT_KEY_GRANT: { + case HEARTBEAT_KEY_GRANTINFO: { void *rspMsg = NULL; int32_t rspLen = 0; mndValidateGrant(pMnode, kv->value, &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { - SKv kv1 = {.key = HEARTBEAT_KEY_GRANT, .valueLen = rspLen, .value = rspMsg}; + SKv kv1 = {.key = HEARTBEAT_KEY_GRANTINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv1); } break; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index b880c0a155..89fd50657d 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -337,7 +337,6 @@ typedef struct SCatalog { SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; - SCtgRentMgmt grantRent; SCtgCacheStat cacheStat; } SCatalog; @@ -569,7 +568,6 @@ typedef struct SCtgUpdateGrantInfoMsg { typedef struct SCtgDropGrantInfoMsg { SCatalog* pCtg; - int64_t grantId; } SCtgDropGrantInfoMsg; typedef struct SCtgCacheOperation { @@ -970,7 +968,7 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uin SCtgTbCache *pCache); int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, SCtgViewCache *pCache); -int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); +// int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); @@ -1043,12 +1041,12 @@ void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); int32_t ctgViewVersionSortCompare(const void* key1, const void* key2); -int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); +// int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2); -int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); +// int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f52787a61e..c1bf641303 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1513,6 +1513,23 @@ int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* n CTG_API_LEAVE(ctgMetaRentGet(&pCtg->viewRent, (void**)views, num, sizeof(SViewVersion))); } +int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == grants || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + *num = 1; + *grants = taosMemoryCalloc(*num, sizeof(SGrantVersion)); + if (!(*grants)) { + ctgError("calloc %d grantVersion failed", *num); + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + (*grants)[0].version = pCtg->grantCache.grantInfo.version; + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 3b8a18e908..12c9907ffd 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1742,8 +1742,6 @@ int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); - CTG_ERR_RET(ctgUpdateRentGrantVersion(pCtg, CGT_GRANT_ID, pRsp)); - _return: CTG_RET(code); @@ -2541,11 +2539,6 @@ int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_JRET( - ctgMetaRentRemove(&pCtg->grantRent, msg->grantId, ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); - - printf("prop:grant:0x%" PRIx64 "removed from rent", msg->grantId); - _return: taosMemoryFreeClear(msg); diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index f757bd856f..0ac133b8e7 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,13 +301,3 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } -int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SGrantHbRsp *pGrant) { - SGrantVersion metaRent = {.grantId = grantId, .version = pGrant->version}; - - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), - ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); - - ctgDebug("grant %d version %d updated to grantRent", grantId, metaRent.version); - - return TSDB_CODE_SUCCESS; -} diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index e157028bee..cacdfcb8d6 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1328,16 +1328,6 @@ int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2) { } } -int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2) { - if (*(int32_t*)key1 == ((SGrantVersion*)key2)->grantId) { - return 0; - } else if (*(int32_t*)key1 < ((SGrantVersion*)key2)->grantId) { - return -1; - } else { - return 1; - } -} - int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) { @@ -1369,15 +1359,6 @@ int32_t ctgViewVersionSortCompare(const void* key1, const void* key2) { } } -int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2) { - if (((SGrantVersion*)key1)->grantId == ((SGrantVersion*)key2)->grantId) { - return 0; - } else if (((SGrantVersion*)key1)->grantId < ((SGrantVersion*)key2)->grantId) { - return -1; - } else { - return 1; - } -} int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { if (NULL == dbInfo) { From f53d0bce5d12669e21d75007b46a864a3ae84fca Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 00:24:42 +0800 Subject: [PATCH 16/45] feat: support uniq grant --- include/common/tgrant.h | 5 +++ include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 5 ++- include/util/taoserror.h | 2 -- source/dnode/vnode/src/vnd/vnodeSvr.c | 5 +++ source/libs/catalog/inc/catalogInt.h | 1 - source/libs/catalog/src/catalog.c | 47 ++++++++++++++++++++++++++ source/libs/catalog/src/ctgCache.c | 2 -- source/libs/parser/inc/parInsertUtil.h | 3 +- source/libs/parser/src/parInsertSml.c | 6 ++-- source/libs/parser/src/parInsertSql.c | 24 ++++++------- source/libs/parser/src/parInsertUtil.c | 9 ++--- source/util/src/terror.c | 2 -- 13 files changed, 84 insertions(+), 28 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 9f7572ed63..66408e9fa4 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -30,6 +30,10 @@ extern "C" { #define GRANT_HEART_BEAT_MIN 2 #define GRANT_ACTIVE_CODE "activeCode" +#define GRANT_ALL_FLAG (0x01) +#define GRANT_AUDIT_FLAG (0x02) +#define GRANT_CSV_FLAG (0x04) +#define GRANT_VIEW_FLAG (0x08) typedef enum { TSDB_GRANT_ALL, @@ -50,6 +54,7 @@ typedef enum { TSDB_GRANT_SUBSCRIPTION, TSDB_GRANT_AUDIT, TSDB_GRANT_CSV, + TSDB_GRANT_VIEW, TSDB_GRANT_MULTI_TIER, TSDB_GRANT_BACKUP_RESTORE, } EGrantType; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 96bf9677b7..899ce62a9d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3931,6 +3931,7 @@ int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq); #define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 +#define SUBMIT_REQ_FROM_FILE 0x4 typedef struct { int32_t flags; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 16deda205a..6ce1769f5f 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -25,6 +25,7 @@ extern "C" { #include "taosdef.h" #include "tarray.h" #include "tcommon.h" +#include "tgrant.h" #include "thash.h" #include "tmsg.h" #include "tname.h" @@ -364,7 +365,9 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes); -int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists); +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pAuth, SUserAuthRes* pRes, bool* exists); + +int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant); int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 94fe80b901..c0c20a0fde 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -575,8 +575,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0821) #define TSDB_CODE_GRANT_DUPLICATED_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x0822) #define TSDB_CODE_GRANT_VIEW_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0823) -#define TSDB_CODE_GRANT_CSV_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0824) -#define TSDB_CODE_GRANT_AUDIT_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0825) // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 3ec6adee41..e4d7b11176 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -238,6 +238,11 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int TSDB_CHECK_CODE(code, lino, _exit); } + if (submitTbData.flags & SUBMIT_REQ_FROM_FILE) { + code = grantCheck(TSDB_GRANT_CSV); + TSDB_CHECK_CODE(code, lino, _exit); + } + int64_t uid; if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { code = vnodePreprocessCreateTableReq(pVnode, pCoder, btimeMs, &uid); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 89fd50657d..0934aa0a27 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -324,7 +324,6 @@ typedef struct SCtgUserAuth { typedef struct SCtgGrantCache { SRWLatch lock; SGrantHbRsp grantInfo; - uint64_t grantCacheSize; } SCtgGrantCache; typedef struct SCatalog { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index c1bf641303..3d256f07d3 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -353,6 +353,39 @@ _return: CTG_RET(code); } +int32_t ctgChkGrant(SCatalog* pCtg, EGrantType grant) { + int32_t code = 0; + int32_t flag = 0; + SCtgGrantCache* pCache = &pCtg->grantCache; + + CTG_LOCK(CTG_READ, &pCache->lock); + switch (grant) { + case TSDB_GRANT_ALL: { + flag = pCache->grantInfo.flags & GRANT_ALL_FLAG; + break; + } + case TSDB_GRANT_AUDIT: { + flag = pCache->grantInfo.flags & GRANT_AUDIT_FLAG; + break; + } + case TSDB_GRANT_CSV: { + flag = pCache->grantInfo.flags & GRANT_CSV_FLAG; + break; + } + case TSDB_GRANT_VIEW: { + flag = pCache->grantInfo.flags & GRANT_VIEW_FLAG; + break; + } + } + CTG_UNLOCK(CTG_READ, &pCache->lock); + + if (flag) code = TSDB_CODE_GRANT_EXPIRED; + +_return: + + CTG_RET(code); +} + int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); @@ -1693,6 +1726,20 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant) { + CTG_API_ENTER(); + + if (NULL == pCtg) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgChkGrant(pCtg, grant)); + +_return: + + CTG_API_LEAVE(code); +} int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 12c9907ffd..ad48aa3fb1 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2520,8 +2520,6 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { goto _return; } - // CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); // TODO: inc/dec or cacheSize ? - code = ctgWriteGrantInfoToCache(pCtg, pRsp); _return: diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index b20587dd43..f1ba58f6de 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -48,7 +48,8 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); void insInitColValues(STableMeta* pTableMeta, SArray* aColValues); void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, - SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals); + SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals, + uint32_t insertType); int32_t initTableColSubmitData(STableDataCxt *pTableCxt); int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks, bool isRebuild); int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 13c4431b62..ffb4eacaea 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -169,7 +169,8 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { STableDataCxt* pTableCxt = NULL; SVCreateTbReq* pCreateTbReq = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false, + ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); if (ret != TSDB_CODE_SUCCESS) { return NULL; } @@ -313,7 +314,8 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc STableDataCxt* pTableCxt = NULL; ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false, + ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); goto end; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 8e32eca28e..b6df2e47ed 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1333,7 +1333,7 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), - pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false); + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false, pStmt->insertType); } char tbFName[TSDB_TABLE_FNAME_LEN]; @@ -1342,7 +1342,7 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS pStmt->pTableMeta->uid = 0; } return insGetTableDataCxt(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), pStmt->pTableMeta, - &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false); + &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false, pStmt->insertType); } static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { @@ -1931,7 +1931,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt } code = insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), - pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true); + pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true, pStmt->insertType); initTableColSubmitData(*ppTableDataCxt); if (code == TSDB_CODE_SUCCESS) { SRow** pRow = taosArrayReserve((*ppTableDataCxt)->pData->aRowP, 1); @@ -2139,14 +2139,20 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { + int32_t code = 0; + int32_t numOfRows = 0; + + if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { + return code; + } + // init only for file if (NULL == pStmt->pTableCxtHashObj) { pStmt->pTableCxtHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } - int32_t numOfRows = 0; - int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); + code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; @@ -2193,10 +2199,6 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); } - // if ((terrno = grantCheck(TSDB_GRANT_CSV)) < 0) { - // return buildInvalidOperationMsg(&pCxt->msg, terrstr()); - // } - NEXT_TOKEN(pStmt->pSql, *pToken); if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); @@ -2761,10 +2763,6 @@ static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpSt int32_t code = TSDB_CODE_SUCCESS; SRowsDataContext rowsDataCxt; - // if ((code = grantCheck(TSDB_GRANT_CSV)) < 0) { - // return code; - // } - if (!pStmt->stbSyntax) { STableDataCxt* pTableCxt = NULL; code = getTableDataCxt(pCxt, pStmt, &pTableCxt); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 6b655bfae6..5c7de2519c 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -208,7 +208,7 @@ void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { void insDestroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, - bool colMode, bool ignoreColVals) { + bool colMode, bool ignoreColVals, uint32_t insertType) { STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == pTableCxt) { return TSDB_CODE_OUT_OF_MEMORY; @@ -249,6 +249,7 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat } else { pTableCxt->pData->flags = (pCreateTbReq != NULL && NULL != *pCreateTbReq) ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; pTableCxt->pData->flags |= colMode ? SUBMIT_REQ_COLUMN_DATA_FORMAT : 0; + pTableCxt->pData->flags |= insertType == TSDB_QUERY_TYPE_FILE_INSERT ? SUBMIT_REQ_FROM_FILE : 0; pTableCxt->pData->suid = pTableMeta->suid; pTableCxt->pData->uid = pTableMeta->uid; pTableCxt->pData->sver = pTableMeta->sversion; @@ -330,7 +331,7 @@ static void resetColValues(SArray* pValues) { } int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, - SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals) { + SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals, uint32_t insertType) { STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); if (NULL != tmp) { *pTableCxt = *tmp; @@ -339,7 +340,7 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* } return TSDB_CODE_SUCCESS; } - int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals); + int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals, insertType); if (TSDB_CODE_SUCCESS == code) { void* pData = *pTableCxt; // deal scan coverity code = taosHashPut(pHash, id, idLen, &pData, POINTER_BYTES); @@ -645,7 +646,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false); + sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false, ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); if (ret != TSDB_CODE_SUCCESS) { uError("insGetTableDataCxt error"); goto end; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9fcca86744..7510b89736 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -462,8 +462,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MACHINES_MISMATCH, "Cluster machines mism TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE, "Expire time of optional grant item is too large") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUPLICATED_ACTIVE, "The active code can't be activated repeatedly") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED, "Number of view has reached the licensed upper limit") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CSV_LIMITED, "Csv has reached the licensed upper limit") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_AUDIT_LIMITED, "Audit has reached the licensed upper limit") // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") From a9316c37100a8d8aec51041d8a1ec7423eb34626 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 12:28:00 +0800 Subject: [PATCH 17/45] feat: support uniq grant --- include/libs/nodes/plannodes.h | 6 +++++- include/libs/nodes/querynodes.h | 2 ++ include/libs/parser/parser.h | 4 +++- include/libs/planner/planner.h | 2 ++ include/libs/qcom/query.h | 5 +++++ source/client/src/clientHb.c | 4 +--- source/client/src/clientImpl.c | 4 ++++ source/libs/nodes/src/nodesCloneFuncs.c | 2 ++ source/libs/nodes/src/nodesCodeFuncs.c | 12 ++++++++++++ source/libs/nodes/src/nodesMsgFuncs.c | 13 ++++++++++++- source/libs/parser/inc/parInsertUtil.h | 3 +-- source/libs/parser/inc/parTranslater.h | 2 ++ source/libs/parser/src/parInsertSml.c | 6 ++---- source/libs/parser/src/parInsertSql.c | 9 ++++++--- source/libs/parser/src/parInsertUtil.c | 9 ++++----- source/libs/parser/src/parUtil.c | 2 ++ source/libs/planner/src/planLogicCreater.c | 2 ++ source/libs/scheduler/src/schRemote.c | 3 +++ 18 files changed, 70 insertions(+), 20 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 8e375e9da8..3ff50ca2e8 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -92,6 +92,8 @@ typedef struct SScanLogicNode { STimeWindow scanRange; SName tableName; bool showRewrite; + bool isView; + bool isAudit; double ratio; SNodeList* pDynamicScanFuncs; int32_t dataRequired; @@ -713,8 +715,10 @@ typedef struct SSubplan { SNode* pTagCond; SNode* pTagIndexCond; bool showRewrite; - int32_t rowsThreshold; + bool isView; + bool isAudit; bool dynamicRowThreshold; + int32_t rowsThreshold; } SSubplan; typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 9647c0adac..97a20c2fc0 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -517,6 +517,8 @@ typedef struct SQuery { SArray* pTableList; SArray* pDbList; bool showRewrite; + // bool isView; + // bool isAudit; int32_t placeholderNum; SArray* pPlaceholderValues; SNode* pPrepareRoot; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 6a41f4607b..2ac2c3ccbd 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -86,8 +86,10 @@ typedef struct SParseContext { bool enableSysInfo; bool async; bool hasInvisibleCol; - const char* svrVer; + bool isView; + bool isAudit; bool nodeOffline; + const char* svrVer; SArray* pTableMetaPos; // sql table pos => catalog data pos SArray* pTableVgroupPos; // sql table pos => catalog data pos int64_t allocatorId; diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 1b523c0323..707d70b71b 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -32,6 +32,8 @@ typedef struct SPlanContext { bool streamQuery; bool rSmaQuery; bool showRewrite; + bool isView; + bool isAudit; int8_t triggerType; int64_t watermark; int64_t deleteMark; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index f79a0a0718..ad8de4707d 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -66,7 +66,12 @@ typedef enum { #define QUERY_RSP_POLICY_QUICK 1 #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) +#define QUERY_MSG_MASK_VIEW() (1 << 1) +#define QUERY_MSG_MASK_AUDIT() (1 << 2) #define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) +#define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) +#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_MASK()) != 0) + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6972ca3c38..e7529e7ed3 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -404,8 +404,6 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); break; } - assert(0); - hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); break; } @@ -917,7 +915,7 @@ int32_t hbGetExpiredGrantInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, gv->version = htonl(gv->version); } - tscDebug("hb got %d expired grant, valueLen:%u", grantNum, (int32_t)sizeof(SGrantVersion) * grantNum); + tscDebug("hb got %d expired grant, valueLen:%d", grantNum, (int32_t)(sizeof(SGrantVersion) * grantNum)); if (!req->info) { req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c1e1da617d..ddcf8ea105 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -458,6 +458,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .mgmtEpSet = getEpSet_s(&pAppInfo->mgmtEp), .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, + .isView = pQuery->isView; + .isAudit = pQuery->isAudit; .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, @@ -1154,6 +1156,8 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, + .isView = pWrapper->pParseCtx->isView, + .isAudit = pWrapper->pParseCtx->isAudit, .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index bc9839792c..b882271175 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -423,6 +423,8 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_OBJECT_FIELD(scanRange, sizeof(STimeWindow)); COPY_OBJECT_FIELD(tableName, sizeof(SName)); COPY_SCALAR_FIELD(showRewrite); + COPY_SCALAR_FIELD(isView); + COPY_SCALAR_FIELD(isAudit); COPY_SCALAR_FIELD(ratio); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); COPY_SCALAR_FIELD(dataRequired); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index cae6ffa5fa..b05c9b019e 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3332,6 +3332,12 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->showRewrite); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isAudit); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkSubplanRowsThreshold, pNode->rowsThreshold); } @@ -3379,6 +3385,12 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->showRewrite); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isAudit); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanRowsThreshold, &pNode->rowsThreshold); } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 3fd219b8d8..7d8cd72cdb 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -3930,6 +3930,12 @@ static int32_t subplanInlineToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueBool(pEncoder, pNode->dynamicRowThreshold); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->isAudit); + } return code; } @@ -3985,7 +3991,12 @@ static int32_t msgToSubplanInline(STlvDecoder* pDecoder, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueBool(pDecoder, &pNode->dynamicRowThreshold); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->isAudit); + } return code; } diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index f1ba58f6de..b20587dd43 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -48,8 +48,7 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); void insInitColValues(STableMeta* pTableMeta, SArray* aColValues); void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, - SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals, - uint32_t insertType); + SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals); int32_t initTableColSubmitData(STableDataCxt *pTableCxt); int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks, bool isRebuild); int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h index 55ea71a368..de885bb1be 100644 --- a/source/libs/parser/inc/parTranslater.h +++ b/source/libs/parser/inc/parTranslater.h @@ -43,6 +43,8 @@ typedef struct STranslateContext { bool createStream; bool stableQuery; bool showRewrite; + bool isView; + bool isAudit; SNode* pPrevRoot; SNode* pPostRoot; } STranslateContext; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index ffb4eacaea..13c4431b62 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -169,8 +169,7 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { STableDataCxt* pTableCxt = NULL; SVCreateTbReq* pCreateTbReq = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false, - ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); if (ret != TSDB_CODE_SUCCESS) { return NULL; } @@ -314,8 +313,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc STableDataCxt* pTableCxt = NULL; ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false, - ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); goto end; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index b6df2e47ed..83a38c4b15 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1333,7 +1333,7 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), - pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false, pStmt->insertType); + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false); } char tbFName[TSDB_TABLE_FNAME_LEN]; @@ -1342,7 +1342,7 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS pStmt->pTableMeta->uid = 0; } return insGetTableDataCxt(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), pStmt->pTableMeta, - &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false, pStmt->insertType); + &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false); } static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { @@ -1931,7 +1931,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt } code = insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), - pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true, pStmt->insertType); + pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true); initTableColSubmitData(*ppTableDataCxt); if (code == TSDB_CODE_SUCCESS) { SRow** pRow = taosArrayReserve((*ppTableDataCxt)->pData->aRowP, 1); @@ -2157,6 +2157,9 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT); + if (rowsDataCxt.pTableDataCxt && rowsDataCxt.pTableDataCxt->pData) { + rowsDataCxt.pTableDataCxt->pData->flags |= SUBMIT_REQ_FROM_FILE; + } if (!pStmt->fileProcessing) { taosCloseFile(&pStmt->fp); } else { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 5c7de2519c..6b655bfae6 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -208,7 +208,7 @@ void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { void insDestroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, - bool colMode, bool ignoreColVals, uint32_t insertType) { + bool colMode, bool ignoreColVals) { STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == pTableCxt) { return TSDB_CODE_OUT_OF_MEMORY; @@ -249,7 +249,6 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat } else { pTableCxt->pData->flags = (pCreateTbReq != NULL && NULL != *pCreateTbReq) ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; pTableCxt->pData->flags |= colMode ? SUBMIT_REQ_COLUMN_DATA_FORMAT : 0; - pTableCxt->pData->flags |= insertType == TSDB_QUERY_TYPE_FILE_INSERT ? SUBMIT_REQ_FROM_FILE : 0; pTableCxt->pData->suid = pTableMeta->suid; pTableCxt->pData->uid = pTableMeta->uid; pTableCxt->pData->sver = pTableMeta->sversion; @@ -331,7 +330,7 @@ static void resetColValues(SArray* pValues) { } int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, - SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals, uint32_t insertType) { + SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals) { STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); if (NULL != tmp) { *pTableCxt = *tmp; @@ -340,7 +339,7 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* } return TSDB_CODE_SUCCESS; } - int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals, insertType); + int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals); if (TSDB_CODE_SUCCESS == code) { void* pData = *pTableCxt; // deal scan coverity code = taosHashPut(pHash, id, idLen, &pData, POINTER_BYTES); @@ -646,7 +645,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false, ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); + sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false); if (ret != TSDB_CODE_SUCCESS) { uError("insGetTableDataCxt error"); goto end; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index dfe33ce55e..8aa37f5784 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -192,6 +192,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Out of memory"; case TSDB_CODE_PAR_ORDERBY_AMBIGUOUS: return "ORDER BY \"%s\" is ambiguous"; + case TSDB_CODE_GRANT_EXPIRED: + return "License expired"; default: return "Unknown error"; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 12b7360165..a5828af969 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -340,6 +340,8 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT strcpy(pScan->tableName.dbname, pRealTable->table.dbName); strcpy(pScan->tableName.tname, pRealTable->table.tableName); pScan->showRewrite = pCxt->pPlanCxt->showRewrite; + pScan->isView = pCxt->pPlanCxt->isView; + pScan->isAudit = pCxt->pPlanCxt->isAudit; pScan->ratio = pRealTable->ratio; pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; pScan->cacheLastMode = pRealTable->cacheLastMode; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 5c67c7974f..1a9c37ffe9 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1109,6 +1109,9 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.refId = pJob->refId; qMsg.execId = pTask->execId; qMsg.msgMask = (pTask->plan->showRewrite) ? QUERY_MSG_MASK_SHOW_REWRITE() : 0; + qMsg.msgMask |= (pTask->plan->isView) ? QUERY_MSG_MASK_VIEW() : 0; + qMsg.msgMask |= (pTask->plan->isAudit) ? QUERY_MSG_MASK_AUDIT() : 0; + qMsg.taskType = TASK_TYPE_TEMP; qMsg.explain = SCH_IS_EXPLAIN_JOB(pJob); qMsg.needFetch = SCH_TASK_NEED_FETCH(pTask); From b852993b8d9ce074f660a225dc657a31fd95b457 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 13:22:39 +0800 Subject: [PATCH 18/45] remove TD_TSZ macro define --- include/util/tcompression.h | 1 - source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 6 - source/util/src/tcompression.c | 24 +--- .../army/community/storage/oneStageComp.json | 66 ++++++++++ tests/army/community/storage/oneStageComp.py | 123 ++++++++++++++++++ tests/parallel_test/cases.task | 2 +- 6 files changed, 192 insertions(+), 30 deletions(-) create mode 100644 tests/army/community/storage/oneStageComp.json create mode 100644 tests/army/community/storage/oneStageComp.py diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 75ddbb12e7..79fe9b613b 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -55,7 +55,6 @@ extern "C" { #define HEAD_MODE(x) x % 2 #define HEAD_ALGO(x) x / 2 -#ifdef TD_TSZ extern bool lossyFloat; extern bool lossyDouble; int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals, diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 6cbf31b15f..2eab045251 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -19,10 +19,8 @@ #include "index.h" #include "qworker.h" #include "tstream.h" -#ifdef TD_TSZ #include "tcompression.h" #include "tglobal.h" -#endif static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); @@ -47,10 +45,8 @@ int32_t dmInitDnode(SDnode *pDnode) { goto _OVER; } -#ifdef TD_TSZ // compress module init tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor); -#endif pDnode->wrappers[DNODE].func = dmGetMgmtFunc(); pDnode->wrappers[MNODE].func = mmGetMgmtFunc(); @@ -119,10 +115,8 @@ void dmCleanupDnode(SDnode *pDnode) { indexCleanup(); taosConvDestroy(); -#ifdef TD_TSZ // compress destroy tsCompressExit(); -#endif dDebug("dnode is closed, ptr:%p", pDnode); } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 3cc00ddc7f..5cb92a4d47 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -54,9 +54,7 @@ #include "tlog.h" #include "ttypes.h" -#ifdef TD_TSZ #include "td_sz.h" -#endif static const int32_t TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) @@ -64,7 +62,6 @@ static const int32_t TEST_NUMBER = 1; #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) -#ifdef TD_TSZ bool lossyFloat = false; bool lossyDouble = false; @@ -83,7 +80,6 @@ int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, // exit call void tsCompressExit() { tdszExit(); } -#endif /* * Compress Integer (Simple8B). @@ -906,7 +902,6 @@ int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, c return nelements * FLOAT_BYTES; } -#ifdef TD_TSZ // // ---------- float double lossy ----------- // @@ -977,6 +972,7 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co } #endif +#ifdef BUILD_NO_CALL /************************************************************************* * STREAM COMPRESSION *************************************************************************/ @@ -2120,7 +2116,7 @@ int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) { return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData); } - +#endif /************************************************************************* * REGULAR COMPRESSION *************************************************************************/ @@ -2154,13 +2150,11 @@ int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, // Float ===================================================== int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ // lossy mode if (lossyFloat) { return tsCompressFloatLossyImp(pIn, nEle, pOut); // lossless mode } else { -#endif if (cmprAlg == ONE_STAGE_COMP) { return tsCompressFloatImp(pIn, nEle, pOut); } else if (cmprAlg == TWO_STAGE_COMP) { @@ -2170,19 +2164,15 @@ int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_ ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut); } else { -#endif // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { return tsDecompressFloatImp(pIn, nEle, pOut); @@ -2193,20 +2183,16 @@ int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3 ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } // Double ===================================================== int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ if (lossyDouble) { // lossy mode return tsCompressDoubleLossyImp(pIn, nEle, pOut); } else { -#endif // lossless mode if (cmprAlg == ONE_STAGE_COMP) { return tsCompressDoubleImp(pIn, nEle, pOut); @@ -2217,19 +2203,15 @@ int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32 ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { -#ifdef TD_TSZ if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut); } else { -#endif // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { return tsDecompressDoubleImp(pIn, nEle, pOut); @@ -2240,9 +2222,7 @@ int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int ASSERTS(0, "compress algo invalid"); return -1; } -#ifdef TD_TSZ } -#endif } // Binary ===================================================== diff --git a/tests/army/community/storage/oneStageComp.json b/tests/army/community/storage/oneStageComp.json new file mode 100644 index 0000000000..12fa51db83 --- /dev/null +++ b/tests/army/community/storage/oneStageComp.json @@ -0,0 +1,66 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 3000, + "prepared_rand": 3000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 3, + "wal_retention_period": 10, + "wal_retention_size": 100, + "keep": "60d,120d,365d", + "stt_trigger": 1, + "wal_level": 2, + "WAL_FSYNC_PERIOD": 3300, + "cachemode": "last_value", + "TABLE_PREFIX":1, + "comp": 1 + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 10, + "insert_rows": 100000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":"2023-01-01 00:00:00", + "columns": [ + { "type": "bool", "name": "bc","max": 1,"min": 1}, + { "type": "float", "name": "fc" ,"max": 101,"min": 101}, + { "type": "double", "name": "dc" ,"max": 102,"min": 102}, + { "type": "tinyint", "name": "ti" ,"max": 103,"min": 103}, + { "type": "smallint", "name": "si" ,"max": 104,"min": 104}, + { "type": "int", "name": "ic" ,"max": 105,"min": 105}, + { "type": "bigint", "name": "bi" ,"max": 106,"min": 106}, + { "type": "utinyint", "name": "uti","max": 107,"min": 107}, + { "type": "usmallint", "name": "usi","max": 108,"min": 108}, + { "type": "uint", "name": "ui" ,"max": 109,"min": 109}, + { "type": "ubigint", "name": "ubi","max": 110,"min": 110}, + { "type": "binary", "name": "bin", "len": 16}, + { "type": "nchar", "name": "nch", "len": 32} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 100,"min": 100}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/community/storage/oneStageComp.py b/tests/army/community/storage/oneStageComp.py new file mode 100644 index 0000000000..f3718bc716 --- /dev/null +++ b/tests/army/community/storage/oneStageComp.py @@ -0,0 +1,123 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import random + +import taos +import frame +import frame.etool + + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + updatecfgDict = { + "compressMsgSize" : "100", + } + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + jfile = etool.curFile(__file__, "oneStageComp.json") + etool.benchMark(json=jfile) + + tdSql.execute(f"use {self.db}") + # set insert data information + self.childtable_count = 10 + self.insert_rows = 100000 + self.timestamp_step = 1000 + + + + def checkColValueCorrect(self): + tdLog.info(f"do action.") + self.flushDb() + + # check all columns correct + cnt = self.insert_rows * self.childtable_count + sql = "select * from stb where bc!=1" + tdSql.checkRows(0) + sql = "select * from stb where fc=101" + tdSql.checkRows(cnt) + sql = "select * from stb where dc!=102" + tdSql.checkRows(0) + sql = "select * from stb where ti!=103" + tdSql.checkRows(0) + sql = "select * from stb where si!=104" + tdSql.checkRows(0) + sql = "select * from stb where ic!=105" + tdSql.checkRows(0) + sql = "select * from stb where b!i=106" + tdSql.checkRows(0) + sql = "select * from stb where uti!=107" + tdSql.checkRows(0) + sql = "select * from stb where usi!=108" + tdSql.checkRows(0) + sql = "select * from stb where ui!=109" + tdSql.checkRows(0) + sql = "select * from stb where ubi!=110" + tdSql.checkRows(0) + + def insertNull(self): + # insert 6 lines + sql = "insert into d0(ts) values(now) (now + 1s) (now + 2s) (now + 3s) (now + 4s) (now + 5s)" + tdSql.execute(sql) + + self.flushDb() + self.trimDb() + + # check all columns correct + cnt = self.insert_rows * self.childtable_count + sql = "select * from stb where bc!=1" + tdSql.checkRows(6) + sql = "select * from stb where bc=1" + tdSql.checkRows(cnt) + sql = "select * from stb where usi!=108" + tdSql.checkRows(6) + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.checkColValueCorrect() + + # check save agg result correct + self.checkAggCorrect() + + # insert null + self.insertNull() + + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 91a0ac46e5..9ea03b4e6b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -23,7 +23,7 @@ fi ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py - +,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1 # From 8559e7061ce55f1c12582bdd1659beec0870743e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 13:26:36 +0800 Subject: [PATCH 19/45] fix: , replace | with float,double --- tests/army/community/cluster/snapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 26bb6b9377..3f9a497f16 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -30,7 +30,7 @@ from frame import * class TDTestCase(TBase): updatecfgDict = { "countAlwaysReturnValue" : "0", - "lossyColumns" : "float|double", + "lossyColumns" : "float,double", "fPrecision" : "0.000000001", "dPrecision" : "0.00000000000000001", "ifAdtFse" : "1" From 656af515ceed48eea2e85beff44dcf549128672f Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 13:34:14 +0800 Subject: [PATCH 20/45] fix: build error --- include/util/tcompression.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 79fe9b613b..9948102494 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -91,7 +91,6 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); } -#endif /************************************************************************* * REGULAR COMPRESSION From 3ae02aaac24826c2c8514c73a1b02386c2ccc9cd Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 13:38:13 +0800 Subject: [PATCH 21/45] fix: build error1 --- source/util/src/tcompression.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 5cb92a4d47..f605c60ccd 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -970,7 +970,6 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co // decompressed with sz return tdszDecompress(SZ_DOUBLE, input + 1, compressedSize - 1, nelements, output); } -#endif #ifdef BUILD_NO_CALL /************************************************************************* From b4621bb37c8186c0a342cb9e27b0f4563a885005 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 13:57:16 +0800 Subject: [PATCH 22/45] fix: restore TD_TSZ macro define --- include/util/tcompression.h | 2 ++ source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 6 ++++++ source/util/src/tcompression.c | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 9948102494..75ddbb12e7 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -55,6 +55,7 @@ extern "C" { #define HEAD_MODE(x) x % 2 #define HEAD_ALGO(x) x / 2 +#ifdef TD_TSZ extern bool lossyFloat; extern bool lossyDouble; int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals, @@ -91,6 +92,7 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); } +#endif /************************************************************************* * REGULAR COMPRESSION diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 2eab045251..6cbf31b15f 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -19,8 +19,10 @@ #include "index.h" #include "qworker.h" #include "tstream.h" +#ifdef TD_TSZ #include "tcompression.h" #include "tglobal.h" +#endif static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); @@ -45,8 +47,10 @@ int32_t dmInitDnode(SDnode *pDnode) { goto _OVER; } +#ifdef TD_TSZ // compress module init tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor); +#endif pDnode->wrappers[DNODE].func = dmGetMgmtFunc(); pDnode->wrappers[MNODE].func = mmGetMgmtFunc(); @@ -115,8 +119,10 @@ void dmCleanupDnode(SDnode *pDnode) { indexCleanup(); taosConvDestroy(); +#ifdef TD_TSZ // compress destroy tsCompressExit(); +#endif dDebug("dnode is closed, ptr:%p", pDnode); } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index f605c60ccd..656e2706f2 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -54,7 +54,9 @@ #include "tlog.h" #include "ttypes.h" +#ifdef TD_TSZ #include "td_sz.h" +#endif static const int32_t TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) @@ -62,6 +64,7 @@ static const int32_t TEST_NUMBER = 1; #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) +#ifdef TD_TSZ bool lossyFloat = false; bool lossyDouble = false; @@ -80,6 +83,7 @@ int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, // exit call void tsCompressExit() { tdszExit(); } +#endif /* * Compress Integer (Simple8B). @@ -902,6 +906,7 @@ int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, c return nelements * FLOAT_BYTES; } +#ifdef TD_TSZ // // ---------- float double lossy ----------- // @@ -970,6 +975,7 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co // decompressed with sz return tdszDecompress(SZ_DOUBLE, input + 1, compressedSize - 1, nelements, output); } +#endif #ifdef BUILD_NO_CALL /************************************************************************* @@ -2149,11 +2155,13 @@ int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, // Float ===================================================== int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { +#ifdef TD_TSZ // lossy mode if (lossyFloat) { return tsCompressFloatLossyImp(pIn, nEle, pOut); // lossless mode } else { +#endif if (cmprAlg == ONE_STAGE_COMP) { return tsCompressFloatImp(pIn, nEle, pOut); } else if (cmprAlg == TWO_STAGE_COMP) { @@ -2163,15 +2171,19 @@ int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_ ASSERTS(0, "compress algo invalid"); return -1; } +#ifdef TD_TSZ } +#endif } int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { +#ifdef TD_TSZ if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut); } else { +#endif // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { return tsDecompressFloatImp(pIn, nEle, pOut); @@ -2182,16 +2194,20 @@ int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int3 ASSERTS(0, "compress algo invalid"); return -1; } +#ifdef TD_TSZ } +#endif } // Double ===================================================== int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { +#ifdef TD_TSZ if (lossyDouble) { // lossy mode return tsCompressDoubleLossyImp(pIn, nEle, pOut); } else { +#endif // lossless mode if (cmprAlg == ONE_STAGE_COMP) { return tsCompressDoubleImp(pIn, nEle, pOut); @@ -2202,15 +2218,19 @@ int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32 ASSERTS(0, "compress algo invalid"); return -1; } +#ifdef TD_TSZ } +#endif } int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf) { +#ifdef TD_TSZ if (HEAD_ALGO(((uint8_t *)pIn)[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut); } else { +#endif // decompress lossless if (cmprAlg == ONE_STAGE_COMP) { return tsDecompressDoubleImp(pIn, nEle, pOut); @@ -2221,7 +2241,9 @@ int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int ASSERTS(0, "compress algo invalid"); return -1; } +#ifdef TD_TSZ } +#endif } // Binary ===================================================== From 6736fd1615b946d0a50cad8a6278dc008d4c3c87 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 4 Feb 2024 14:11:17 +0800 Subject: [PATCH 23/45] fix(tsdb): check and return if the rows in stt are before the data rows in data files. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 126 +++++++++++++++--------- 1 file changed, 82 insertions(+), 44 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index d9b932a367..86f58717e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2628,6 +2628,58 @@ static bool moveToNextTableForPreFileSetMem(SReaderStatus* pStatus) { return (pStatus->pProcMemTableIter != NULL); } +static void buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo) { + SReaderStatus* pStatus = &pReader->status; + SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + + SDataBlockInfo* pInfo = &pResBlock->info; + blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt); + + pInfo->rows = pScanInfo->numOfRowsInStt; + pInfo->id.uid = pScanInfo->uid; + pInfo->dataLoad = 1; + pInfo->window = pScanInfo->sttWindow; + + setComposedBlockFlag(pReader, true); + + pScanInfo->sttKeyInfo.nextProcKey = asc ? pScanInfo->sttWindow.ekey + 1 : pScanInfo->sttWindow.skey - 1; + pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA; + pScanInfo->lastProcKey = asc ? pScanInfo->sttWindow.ekey : pScanInfo->sttWindow.skey; + pScanInfo->sttBlockReturned = true; + + pSttBlockReader->mergeTree.pIter = NULL; + + tsdbDebug("%p uid:%" PRId64 " return clean stt block as one, brange:%" PRId64 "-%" PRId64 " rows:%" PRId64 " %s", + pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, + pResBlock->info.rows, pReader->idStr); +} + +static void buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, + SFileDataBlockInfo* pBlockInfo, int32_t blockIndex) { + // whole block is required, return it directly + SReaderStatus* pStatus = &pReader->status; + SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + + pInfo->rows = pBlockInfo->numRow; + pInfo->id.uid = pScanInfo->uid; + pInfo->dataLoad = 0; + pInfo->version = pReader->info.verRange.maxVer; + pInfo->window = (STimeWindow){.skey = pBlockInfo->firstKey, .ekey = pBlockInfo->lastKey}; + setComposedBlockFlag(pReader, false); + setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); + + // update the last key for the corresponding table + pScanInfo->lastProcKey = asc ? pInfo->window.ekey : pInfo->window.skey; + tsdbDebug("%p uid:%" PRIu64 " clean file block retrieved from file, global index:%d, " + "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s", + pReader, pScanInfo->uid, blockIndex, pBlockInfo->tbBlockIdx, pBlockInfo->numRow, pBlockInfo->firstKey, + pBlockInfo->lastKey, pReader->idStr); +} + static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader; @@ -2680,28 +2732,7 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { // if only require the total rows, no need to load data from stt file if it is clean stt blocks if (pReader->info.execMode == READER_EXEC_ROWS && pScanInfo->cleanSttBlocks) { - bool asc = ASCENDING_TRAVERSE(pReader->info.order); - - SDataBlockInfo* pInfo = &pResBlock->info; - blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt); - - pInfo->rows = pScanInfo->numOfRowsInStt; - pInfo->id.uid = pScanInfo->uid; - pInfo->dataLoad = 1; - pInfo->window = pScanInfo->sttWindow; - - setComposedBlockFlag(pReader, true); - - pScanInfo->sttKeyInfo.nextProcKey = asc ? pScanInfo->sttWindow.ekey + 1 : pScanInfo->sttWindow.skey - 1; - pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA; - pScanInfo->lastProcKey = asc ? pScanInfo->sttWindow.ekey : pScanInfo->sttWindow.skey; - pScanInfo->sttBlockReturned = true; - - pSttBlockReader->mergeTree.pIter = NULL; - - tsdbDebug("%p uid:%" PRId64 " return clean stt block as one, brange:%" PRId64 "-%" PRId64 " rows:%" PRId64 " %s", - pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, - pResBlock->info.rows, pReader->idStr); + buildCleanBlockFromSttFiles(pReader, pScanInfo); return TSDB_CODE_SUCCESS; } @@ -2741,10 +2772,11 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { } } -static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, bool asc) { +// current active data block not overlap with the stt-files/stt-blocks +static bool notOverlapWithFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, bool asc) { ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT); - if (pScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) { + if ((!hasDataInSttBlock(pScanInfo)) || (pScanInfo->cleanSttBlocks == true)) { return true; } else { int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; @@ -2794,24 +2826,32 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pScanInfo, pReader->info.order); code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { - if (notOverlapWithSttFiles(pBlockInfo, pScanInfo, asc)) { - // whole block is required, return it directly - SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; - pInfo->rows = pBlockInfo->numRow; - pInfo->id.uid = pScanInfo->uid; - pInfo->dataLoad = 0; - pInfo->version = pReader->info.verRange.maxVer; - pInfo->window = (STimeWindow){.skey = pBlockInfo->firstKey, .ekey = pBlockInfo->lastKey}; - setComposedBlockFlag(pReader, false); - setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order); + if (notOverlapWithFiles(pBlockInfo, pScanInfo, asc)) { + int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; - // update the last key for the corresponding table - pScanInfo->lastProcKey = asc ? pInfo->window.ekey : pInfo->window.skey; - tsdbDebug("%p uid:%" PRIu64 - " clean file block retrieved from file, global index:%d, " - "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s", - pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->numRow, - pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr); + if ((!hasDataInSttBlock(pScanInfo)) || (asc && pBlockInfo->lastKey < keyInStt) || + (!asc && pBlockInfo->firstKey > keyInStt)) { + if (pScanInfo->cleanSttBlocks && hasDataInSttBlock(pScanInfo)) { + if (asc) { // file block is located before the stt block + ASSERT(pScanInfo->sttWindow.skey > pBlockInfo->lastKey); + } else { // stt block is before the file block + ASSERT(pScanInfo->sttWindow.ekey < pBlockInfo->firstKey); + } + } + + buildCleanBlockFromDataFiles(pReader, pScanInfo, pBlockInfo, pBlockIter->index); + } else { // clean stt block + if (asc) { + ASSERT(pScanInfo->sttWindow.ekey < pBlockInfo->firstKey); + } else { + ASSERT(pScanInfo->sttWindow.skey > pBlockInfo->lastKey); + } + + // return the stt file block + ASSERT(pReader->info.execMode == READER_EXEC_ROWS && pSttBlockReader->mergeTree.pIter == NULL); + buildCleanBlockFromSttFiles(pReader, pScanInfo); + return TSDB_CODE_SUCCESS; + } } else { SBlockData* pBData = &pReader->status.fileBlockData; tBlockDataReset(pBData); @@ -2822,7 +2862,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { int64_t st = taosGetTimestampUs(); // let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files - pScanInfo->cleanSttBlocks = false; initSttBlockReader(pSttBlockReader, pScanInfo, pReader); // no data in stt block, no need to proceed. @@ -2840,8 +2879,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // data in stt now overlaps with current active file data block, need to composed with file data block. int64_t lastKeyInStt = getCurrentKeyInSttBlock(pSttBlockReader); - if ((lastKeyInStt >= pBlockInfo->firstKey && asc) || - (lastKeyInStt <= pBlockInfo->lastKey && (!asc))) { + if ((lastKeyInStt >= pBlockInfo->firstKey && asc) || (lastKeyInStt <= pBlockInfo->lastKey && (!asc))) { tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader, lastKeyInStt, pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr); break; From 0029c348c5e049ce90d8ef23e8fe644e8e7dfb6e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 14:56:24 +0800 Subject: [PATCH 24/45] fix: float and double tsz check --- tests/army/community/cluster/snapshot.json | 4 ++-- tests/army/community/cluster/snapshot.py | 17 +++++++++++++++++ utils/TSZ/sz/src/sz_double.c | 8 +++++--- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/tests/army/community/cluster/snapshot.json b/tests/army/community/cluster/snapshot.json index d4f6f00d37..f664278a72 100644 --- a/tests/army/community/cluster/snapshot.json +++ b/tests/army/community/cluster/snapshot.json @@ -35,8 +35,8 @@ "start_timestamp":"now-12d", "columns": [ { "type": "bool", "name": "bc"}, - { "type": "float", "name": "fc" }, - { "type": "double", "name": "dc"}, + { "type": "float", "name": "fc", "min": 100, "min": 100}, + { "type": "double", "name": "dc", "min": 200, "min": 200}, { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si" }, { "type": "int", "name": "ic" }, diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 3f9a497f16..6648efa027 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -52,6 +52,16 @@ class TDTestCase(TBase): sql = f"create table {self.db}.ta(ts timestamp, age int) tags(area int)" tdSql.execute(sql) + def checkFloatDouble(self): + sql = f"select count(*) from {self.db}.{self.stb} where fc!=100" + tdSql.checkFirstValue(sql, 0) + sql = f"select count(*) from {self.db}.{self.stb} where dc!=200" + tdSql.checkFirstValue(sql, 0) + sql = f"select avg(fc) from {self.db}.{self.stb}" + tdSql.checkFirstValue(sql, 100) + sql = f"select avg(dc) from {self.db}.{self.stb}" + tdSql.checkFirstValue(sql, 200) + def doAction(self): tdLog.info(f"do action.") self.flushDb() @@ -89,6 +99,9 @@ class TDTestCase(TBase): # check insert data correct self.checkInsertCorrect() + # check float double value ok + tdSql.checkFloatDouble() + # save self.snapshotAgg() @@ -101,6 +114,10 @@ class TDTestCase(TBase): # check insert correct again self.checkInsertCorrect() + # check float double value ok + tdSql.checkFloatDouble() + + tdLog.success(f"{__file__} successfully executed") diff --git a/utils/TSZ/sz/src/sz_double.c b/utils/TSZ/sz/src/sz_double.c index 1adfdf3b56..0510fc612d 100644 --- a/utils/TSZ/sz/src/sz_double.c +++ b/utils/TSZ/sz/src/sz_double.c @@ -385,9 +385,11 @@ unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength totalSampleSize++; pred_value = data_pos[-1]; pred_err = fabs(pred_value - *data_pos); - radiusIndex = (unsigned long)((pred_err/realPrecision+1)/2); - if(radiusIndex>=confparams_cpr->maxRangeRadius) - radiusIndex = confparams_cpr->maxRangeRadius - 1; + double dbri = (unsigned long)((pred_err/realPrecision+1)/2); + if(dbri >= (double)confparams_cpr->maxRangeRadius) + radiusIndex = confparams_cpr->maxRangeRadius - 1; + else + radiusIndex = dbri; intervals[radiusIndex]++; data_pos += confparams_cpr->sampleDistance; From 6cf7809e8d793de9ebf76300528dee1057c4c027 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 14:59:47 +0800 Subject: [PATCH 25/45] feat: support uniq grant --- include/common/tglobal.h | 2 +- include/libs/nodes/plannodes.h | 2 -- include/libs/nodes/querynodes.h | 2 -- include/libs/qcom/query.h | 2 +- source/client/src/clientImpl.c | 2 -- source/common/src/tglobal.c | 15 ++++++++++++++- source/libs/parser/inc/parTranslater.h | 4 ++-- source/libs/planner/src/planPhysiCreater.c | 2 ++ source/libs/qworker/src/qwMsg.c | 22 ++++++++++++++++++---- 9 files changed, 38 insertions(+), 15 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index f23bb4d51b..04e9b5a380 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -232,7 +232,7 @@ struct SConfig *taosGetCfg(); void taosSetAllDebugFlag(int32_t flag); void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); -int8_t taosGranted(); +int8_t taosGranted(int8_t type); #ifdef __cplusplus } diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 3ff50ca2e8..84934b4f4f 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -92,8 +92,6 @@ typedef struct SScanLogicNode { STimeWindow scanRange; SName tableName; bool showRewrite; - bool isView; - bool isAudit; double ratio; SNodeList* pDynamicScanFuncs; int32_t dataRequired; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 97a20c2fc0..9647c0adac 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -517,8 +517,6 @@ typedef struct SQuery { SArray* pTableList; SArray* pDbList; bool showRewrite; - // bool isView; - // bool isAudit; int32_t placeholderNum; SArray* pPlaceholderValues; SNode* pPrepareRoot; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index ad8de4707d..b63d3d5db4 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -70,7 +70,7 @@ typedef enum { #define QUERY_MSG_MASK_AUDIT() (1 << 2) #define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) #define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) -#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_MASK()) != 0) +#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_AUDIT()) != 0) typedef struct STableComInfo { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ddcf8ea105..d3d8ee1dc1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -458,8 +458,6 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .mgmtEpSet = getEpSet_s(&pAppInfo->mgmtEp), .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, - .isView = pQuery->isView; - .isAudit = pQuery->isAudit; .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 46c4d613fb..d46b91276b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1801,4 +1801,17 @@ void taosSetAllDebugFlag(int32_t flag) { if (terrno == TSDB_CODE_CFG_NOT_FOUND) terrno = TSDB_CODE_SUCCESS; // ignore not exist } -int8_t taosGranted() { return atomic_load_8(&tsGrant); } +int8_t taosGranted(int8_t type) { + switch (type) { + case TSDB_GRANT_ALL: + return atomic_load_8(&tsGrant) & GRANT_ALL_FLAG; + case TSDB_GRANT_AUDIT: + return atomic_load_8(&tsGrant) & GRANT_AUDIT_FLAG; + case TSDB_GRANT_VIEW: + return atomic_load_8(&tsGrant) & GRANT_VIEW_FLAG; + default: + ASSERTS(0, "undefined grant type:%" PRIi8, type); + break; + } + return 0; +} \ No newline at end of file diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h index de885bb1be..00f37cb002 100644 --- a/source/libs/parser/inc/parTranslater.h +++ b/source/libs/parser/inc/parTranslater.h @@ -43,8 +43,8 @@ typedef struct STranslateContext { bool createStream; bool stableQuery; bool showRewrite; - bool isView; - bool isAudit; + // bool isView; + // bool isAudit; SNode* pPrevRoot; SNode* pPostRoot; } STranslateContext; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 21c637116f..4b0024d098 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2164,6 +2164,8 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl pSubplan->level = pLogicSubplan->level; pSubplan->rowsThreshold = 4096; pSubplan->dynamicRowThreshold = false; + pSubplan->isView = pCxt->pPlanCxt->isView; + pSubplan->isAudit = pCxt->pPlanCxt->isAudit; if (NULL != pCxt->pPlanCxt->pUser) { snprintf(pSubplan->user, sizeof(pSubplan->user), "%s", pCxt->pPlanCxt->pUser); } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 9a1c309ab0..c9d3cb51e3 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -360,10 +360,24 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - if (chkGrant && (!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && !taosGranted()) { - QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + if (chkGrant) { + if ((!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_ALL)) { + QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + + if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) { + QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + + if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) { + QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } } uint64_t sId = msg.sId; From d51e651c17ef19b74b9325b3ebf95164e10601e3 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 15:00:44 +0800 Subject: [PATCH 26/45] fix: case right --- tests/army/community/cluster/snapshot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 6648efa027..6fd2218344 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -100,7 +100,7 @@ class TDTestCase(TBase): self.checkInsertCorrect() # check float double value ok - tdSql.checkFloatDouble() + self.checkFloatDouble() # save self.snapshotAgg() @@ -115,7 +115,7 @@ class TDTestCase(TBase): self.checkInsertCorrect() # check float double value ok - tdSql.checkFloatDouble() + self.checkFloatDouble() tdLog.success(f"{__file__} successfully executed") From d63448be72b9ebeadc8876e33cc07371b62d2e82 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Sun, 4 Feb 2024 15:44:49 +0800 Subject: [PATCH 27/45] reset group id for event window --- source/libs/executor/src/eventwindowoperator.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index 3cfd0ab582..2cba6e3241 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -220,7 +220,6 @@ static int32_t setSingleOutputTupleBufv1(SResultRowInfo* pResultRowInfo, STimeWi (*pResult)->win = *win; - clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs); setResultRowInitCtx(*pResult, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset); return TSDB_CODE_SUCCESS; } @@ -262,6 +261,7 @@ int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* p } else if (pInfo->groupId != gid) { // this is a new group, reset the info pInfo->inWindow = false; + pInfo->groupId = gid; } SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; @@ -319,6 +319,9 @@ int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* p doKeepNewWindowStartInfo(pRowSup, tsList, rowIndex, gid); pInfo->inWindow = true; startIndex = rowIndex; + if (pInfo->pRow != NULL) { + clearResultRowInitFlag(pSup->pCtx, pSup->numOfExprs); + } break; } } From 3ebc26e1fb35dd0cb589ce859949905496b3b48a Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 16:33:43 +0800 Subject: [PATCH 28/45] feat: support uniq grant --- include/common/systable.h | 5 +++++ include/libs/qcom/query.h | 5 +++++ source/dnode/mnode/impl/src/mndCluster.c | 2 +- source/libs/nodes/src/nodesCloneFuncs.c | 2 -- source/libs/parser/src/parInsertSql.c | 6 +++--- source/libs/parser/src/parTranslater.c | 16 +++++++++++++++- source/libs/planner/src/planLogicCreater.c | 2 -- 7 files changed, 29 insertions(+), 9 deletions(-) diff --git a/include/common/systable.h b/include/common/systable.h index 37593144d8..f0f8ac8cf6 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -65,6 +65,11 @@ extern "C" { #define TSDB_PERFS_TABLE_TRANS "perf_trans" #define TSDB_PERFS_TABLE_APPS "perf_apps" +#define TSDB_AUDIT_DB "audit" +#define TSDB_AUDIT_STB_OPERATION "operations" +#define TSDB_AUDIT_CTB_OPERATION "t_operations_" +#define TSDB_AUDIT_CTB_OPERATION_LEN 13 + typedef struct SSysDbTableSchema { const char* name; const int32_t type; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index b63d3d5db4..4c33c36d1b 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -343,6 +343,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define IS_SYS_DBNAME(_dbname) (IS_INFORMATION_SCHEMA_DB(_dbname) || IS_PERFORMANCE_SCHEMA_DB(_dbname)) +#define IS_AUDIT_DBNAME(_dbname) ((*(_dbname) == 'a') && (0 == strcmp(_dbname, TSDB_AUDIT_DB))) +#define IS_AUDIT_STB_NAME(_stbname) ((*(_stbname) == 'o') && (0 == strcmp(_stbname, TSDB_AUDIT_STB_OPERATION))) +#define IS_AUDIT_CTB_NAME(_ctbname) \ + ((*(_ctbname) == 't') && (0 == strncmp(_ctbname, TSDB_AUDIT_CTB_OPERATION, TSDB_AUDIT_CTB_OPERATION_LEN))) + #define qFatal(...) \ do { \ if (qDebugFlag & DEBUG_FATAL) { \ diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 1a55a161bf..36b079d347 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -409,7 +409,7 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) { } { // audit - auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, cfgReq.sqlLen); + auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, MIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1)); } _exit: tFreeSMCfgClusterReq(&cfgReq); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index b882271175..bc9839792c 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -423,8 +423,6 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_OBJECT_FIELD(scanRange, sizeof(STimeWindow)); COPY_OBJECT_FIELD(tableName, sizeof(SName)); COPY_SCALAR_FIELD(showRewrite); - COPY_SCALAR_FIELD(isView); - COPY_SCALAR_FIELD(isAudit); COPY_SCALAR_FIELD(ratio); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); COPY_SCALAR_FIELD(dataRequired); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 83a38c4b15..a229d2658e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2142,9 +2142,9 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt int32_t code = 0; int32_t numOfRows = 0; - if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { - return code; - } + // if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { + // return code; + // } // init only for file if (NULL == pStmt->pTableCxtHashObj) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7dabae0c3a..de667b2f20 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3165,6 +3165,19 @@ static int32_t checkJoinTable(STranslateContext* pCxt, SJoinTableNode* pJoinTabl return TSDB_CODE_SUCCESS; } +static int32_t translateAudit(STranslateContext* pCxt, SRealTableNode* pRealTable, SName* pName) { + if (pRealTable->pMeta->tableType == TSDB_SUPER_TABLE) { + if (IS_AUDIT_DBNAME(pName->dbname) && IS_AUDIT_STB_NAME(pName->tname)) { + pCxt->pParseCxt->isAudit = true; + } + } else if (pRealTable->pMeta->tableType == TSDB_CHILD_TABLE) { + if (IS_AUDIT_DBNAME(pName->dbname) && IS_AUDIT_CTB_NAME(pName->tname)) { + pCxt->pParseCxt->isAudit = true; + } + } + return 0; +} + int32_t translateTable(STranslateContext* pCxt, SNode** pTable) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(*pTable)) { @@ -3184,7 +3197,8 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable) { if (TSDB_VIEW_TABLE == pRealTable->pMeta->tableType) { return translateView(pCxt, pTable, &name); } -#endif + translateAudit(pCxt, pRealTable, &name); +#endif code = setTableVgroupList(pCxt, &name, pRealTable); if (TSDB_CODE_SUCCESS == code) { code = setTableIndex(pCxt, &name, pRealTable); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index a5828af969..12b7360165 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -340,8 +340,6 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT strcpy(pScan->tableName.dbname, pRealTable->table.dbName); strcpy(pScan->tableName.tname, pRealTable->table.tableName); pScan->showRewrite = pCxt->pPlanCxt->showRewrite; - pScan->isView = pCxt->pPlanCxt->isView; - pScan->isAudit = pCxt->pPlanCxt->isAudit; pScan->ratio = pRealTable->ratio; pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; pScan->cacheLastMode = pRealTable->cacheLastMode; From d281bc70b0f0ed2618e23cda284439f9ae122109 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 16:46:24 +0800 Subject: [PATCH 29/45] feat: remove hb of grantInfo --- include/common/tmsg.h | 9 --- include/libs/catalog/catalog.h | 5 -- source/client/src/clientHb.c | 77 ------------------- source/common/src/tmsg.c | 31 -------- source/dnode/mnode/impl/inc/mndGrant.h | 1 - source/dnode/mnode/impl/src/mndProfile.c | 10 --- source/libs/catalog/inc/catalogInt.h | 27 ------- source/libs/catalog/src/catalog.c | 82 --------------------- source/libs/catalog/src/ctgCache.c | 94 ------------------------ source/libs/parser/src/parInsertSql.c | 4 - 10 files changed, 340 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 899ce62a9d..9cd9acaee4 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -110,7 +110,6 @@ enum { HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_DYN_VIEW, HEARTBEAT_KEY_VIEWINFO, - HEARTBEAT_KEY_GRANTINFO, }; typedef enum _mgmt_table { @@ -1886,14 +1885,6 @@ int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); void tFreeSViewHbRsp(SViewHbRsp* pRsp); -typedef struct { - int32_t version; - uint32_t flags; -} SGrantHbRsp; - -int32_t tSerializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); -int32_t tDeserializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); - typedef struct { int32_t numOfTables; int32_t numOfVgroup; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 6ce1769f5f..96a05ccbf9 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -343,8 +343,6 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion** stables, u int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* num, SDynViewVersion** dynViewVersion); -int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num); - int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* num); int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num); @@ -367,8 +365,6 @@ int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *p int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pAuth, SUserAuthRes* pRes, bool* exists); -int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant); - int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet* epSet); @@ -388,7 +384,6 @@ int32_t catalogRemoveViewMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId int32_t catalogUpdateDynViewVer(SCatalog* pCtg, SDynViewVersion* pVer); int32_t catalogUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); -int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg); int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index e7529e7ed3..53a64cec86 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,28 +327,6 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } -static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { - int32_t code = 0; - - SGrantHbRsp *hbRsp = taosMemoryCalloc(1, sizeof(SGrantHbRsp)); - if (!hbRsp) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (tDeserializeSGrantHbRsp(value, valueLen, hbRsp) != 0) { - taosMemoryFree(hbRsp); - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - tscDebug("hb to update grant info, version:%d, flags:%u", hbRsp->version, hbRsp->flags); - - catalogUpdateGrantInfo(pCatalog, hbRsp); - - return TSDB_CODE_SUCCESS; -} - static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pKvs, i); @@ -399,14 +377,6 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p hbProcessViewInfoRsp(kv->value, kv->valueLen, pCatalog); break; } - case HEARTBEAT_KEY_GRANTINFO: { - if (kv->valueLen <= 0 || NULL == kv->value) { - tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); - break; - } - hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); - break; - } #endif default: tscError("invalid hb key type:%d", kv->key); @@ -895,48 +865,6 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S return TSDB_CODE_SUCCESS; } -int32_t hbGetExpiredGrantInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { - SGrantVersion *grants = NULL; - uint32_t grantNum = 0; - int32_t code = 0; - - code = catalogGetExpiredGrants(pCatalog, &grants, &grantNum); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - - if (grantNum <= 0) { - taosMemoryFree(grants); - return code; - } - - for (int32_t i = 0; i < grantNum; ++i) { - SGrantVersion *gv = &grants[i]; - gv->version = htonl(gv->version); - } - - tscDebug("hb got %d expired grant, valueLen:%d", grantNum, (int32_t)(sizeof(SGrantVersion) * grantNum)); - - if (!req->info) { - req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); - if (!req->info) { - code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(grants); - return code; - taosMemoryFree(grants); - } - } - - SKv kv = { - .key = HEARTBEAT_KEY_GRANTINFO, - .valueLen = sizeof(SGrantVersion) * grantNum, - .value = grants, - }; - taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); - - return TSDB_CODE_SUCCESS; -} - int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL != pApp) { @@ -999,11 +927,6 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req if (TSDB_CODE_SUCCESS != code) { return code; } - - code = hbGetExpiredGrantInfo(connKey, pCatalog, req); - if (TSDB_CODE_SUCCESS != code) { - return code; - } #endif } else { req->app.appId = 0; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 72c17756b5..785cb1878e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9333,35 +9333,4 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) { } taosArrayDestroy(pRsp->pViewRsp); -} - -int32_t tSerializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - - if (tEncodeI32v(&encoder, pRsp->version) < 0) return -1; - if (tEncodeU32v(&encoder, pRsp->flags) < 0) return -1; - - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - - if (tDecodeI32v(&decoder, &pRsp->version) < 0) return -1; - if (tDecodeU32v(&decoder, &pRsp->flags) < 0) return -1; - - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index a012d7a8f5..bda5cebf01 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -43,7 +43,6 @@ int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); - int32_t mndValidateGrant(SMnode * pMnode, SGrantVersion * pGrantVersion, void **ppRsp, int32_t *pRspLen); int32_t mndGrantGetLastState(SMnode * pMnode, SGrantState * pState); SGrantLogObj *mndAcquireGrant(SMnode * pMnode, void **ppIter); void mndReleaseGrant(SMnode * pMnode, SGrantLogObj * pGrant, void *pIter); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 2d2d12a668..f42cdfa4f3 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -606,16 +606,6 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } - case HEARTBEAT_KEY_GRANTINFO: { - void *rspMsg = NULL; - int32_t rspLen = 0; - mndValidateGrant(pMnode, kv->value, &rspMsg, &rspLen); - if (rspMsg && rspLen > 0) { - SKv kv1 = {.key = HEARTBEAT_KEY_GRANTINFO, .valueLen = rspLen, .value = rspMsg}; - taosArrayPush(hbRsp.info, &kv1); - } - break; - } #endif default: mError("invalid kv key:%d", kv->key); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 0934aa0a27..a35eec0ec5 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -39,7 +39,6 @@ extern "C" { #define CTG_MAX_COMMAND_LEN 512 #define CTG_DEFAULT_CACHE_MON_MSEC 5000 #define CTG_CLEAR_CACHE_ROUND_TB_NUM 3000 -#define CGT_GRANT_ID 0 #define CTG_RENT_SLOT_SECOND 1.5 @@ -70,7 +69,6 @@ typedef enum { CTG_CI_UDF, CTG_CI_SVR_VER, CTG_CI_VIEW, - CTG_CI_GRANT_INFO, CTG_CI_MAX_VALUE, } CTG_CACHE_ITEM; @@ -103,8 +101,6 @@ enum { CTG_OP_DROP_TB_INDEX, CTG_OP_UPDATE_VIEW_META, CTG_OP_DROP_VIEW_META, - CTG_OP_UPDATE_GRANT_INFO, - CTG_OP_DROP_GRANT_INFO, CTG_OP_CLEAR_CACHE, CTG_OP_MAX }; @@ -127,7 +123,6 @@ typedef enum { CTG_TASK_GET_TB_HASH_BATCH, CTG_TASK_GET_TB_TAG, CTG_TASK_GET_VIEW, - CTG_TASK_GET_GRANT_INFO, } CTG_TASK_TYPE; typedef enum { @@ -321,18 +316,12 @@ typedef struct SCtgUserAuth { uint64_t userCacheSize; } SCtgUserAuth; -typedef struct SCtgGrantCache { - SRWLatch lock; - SGrantHbRsp grantInfo; -} SCtgGrantCache; - typedef struct SCatalog { uint64_t clusterId; bool stopUpdate; SDynViewVersion dynViewVer; SHashObj* userCache; // key:user, value:SCtgUserAuth SHashObj* dbCache; // key:dbname, value:SCtgDBCache - SCtgGrantCache grantCache; SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; @@ -381,7 +370,6 @@ typedef struct SCtgJob { int32_t tbCfgNum; int32_t svrVerNum; int32_t viewNum; - int32_t grantNum; } SCtgJob; typedef struct SCtgMsgCtx { @@ -560,15 +548,6 @@ typedef struct SCtgDropViewMetaMsg { uint64_t viewId; } SCtgDropViewMetaMsg; -typedef struct SCtgUpdateGrantInfoMsg { - SCatalog* pCtg; - SGrantHbRsp* pRsp; -} SCtgUpdateGrantInfoMsg; - -typedef struct SCtgDropGrantInfoMsg { - SCatalog* pCtg; -} SCtgDropGrantInfoMsg; - typedef struct SCtgCacheOperation { int32_t opId; void* data; @@ -931,7 +910,6 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action); int32_t ctgOpDropStbMeta(SCtgCacheOperation* action); int32_t ctgOpDropTbMeta(SCtgCacheOperation* action); int32_t ctgOpDropViewMeta(SCtgCacheOperation* action); -int32_t ctgOpDropGrantInfo(SCtgCacheOperation* action); int32_t ctgOpUpdateUser(SCtgCacheOperation* action); int32_t ctgOpUpdateEpset(SCtgCacheOperation* operation); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char* dbFName, SCtgDBCache** pCache); @@ -967,10 +945,8 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uin SCtgTbCache *pCache); int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, SCtgViewCache *pCache); -// int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); -int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask); void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache* dbCache); @@ -980,7 +956,6 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation* operation); int32_t ctgOpUpdateTbIndex(SCtgCacheOperation* operation); int32_t ctgOpClearCache(SCtgCacheOperation* operation); int32_t ctgOpUpdateViewMeta(SCtgCacheOperation *operation); -int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation); int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char* tableName, int32_t* tbType); int32_t ctgGetTbHashVgroupFromCache(SCatalog* pCtg, const SName* pTableName, SVgroupInfo** pVgroup); int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsCtx *ctx, int32_t dbIdx, @@ -1040,12 +1015,10 @@ void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); int32_t ctgViewVersionSortCompare(const void* key1, const void* key2); -// int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2); -// int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3d256f07d3..5a53f167c4 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -353,39 +353,6 @@ _return: CTG_RET(code); } -int32_t ctgChkGrant(SCatalog* pCtg, EGrantType grant) { - int32_t code = 0; - int32_t flag = 0; - SCtgGrantCache* pCache = &pCtg->grantCache; - - CTG_LOCK(CTG_READ, &pCache->lock); - switch (grant) { - case TSDB_GRANT_ALL: { - flag = pCache->grantInfo.flags & GRANT_ALL_FLAG; - break; - } - case TSDB_GRANT_AUDIT: { - flag = pCache->grantInfo.flags & GRANT_AUDIT_FLAG; - break; - } - case TSDB_GRANT_CSV: { - flag = pCache->grantInfo.flags & GRANT_CSV_FLAG; - break; - } - case TSDB_GRANT_VIEW: { - flag = pCache->grantInfo.flags & GRANT_VIEW_FLAG; - break; - } - } - CTG_UNLOCK(CTG_READ, &pCache->lock); - - if (flag) code = TSDB_CODE_GRANT_EXPIRED; - -_return: - - CTG_RET(code); -} - int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); @@ -1546,24 +1513,6 @@ int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* n CTG_API_LEAVE(ctgMetaRentGet(&pCtg->viewRent, (void**)views, num, sizeof(SViewVersion))); } -int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num) { - CTG_API_ENTER(); - - if (NULL == pCtg || NULL == grants || NULL == num) { - CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); - } - - *num = 1; - *grants = taosMemoryCalloc(*num, sizeof(SGrantVersion)); - if (!(*grants)) { - ctgError("calloc %d grantVersion failed", *num); - CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); - } - (*grants)[0].version = pCtg->grantCache.grantInfo.version; - - CTG_API_LEAVE(TSDB_CODE_SUCCESS); -} - int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { CTG_API_ENTER(); @@ -1726,21 +1675,6 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant) { - CTG_API_ENTER(); - - if (NULL == pCtg) { - CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); - } - - int32_t code = 0; - CTG_ERR_JRET(ctgChkGrant(pCtg, grant)); - -_return: - - CTG_API_LEAVE(code); -} - int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) { CTG_API_ENTER(); @@ -1811,22 +1745,6 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg) { - CTG_API_ENTER(); - - if (NULL == pCtg || NULL == pMsg) { - CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); - } - - int32_t code = 0; - CTG_ERR_JRET(ctgUpdateGrantInfoToCache(pCtg, pMsg, true)); - -_return: - - CTG_API_LEAVE(code); -} - - int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index ad48aa3fb1..08a8f348e9 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -32,8 +32,6 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update v {CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex}, {CTG_OP_UPDATE_VIEW_META, "update viewMeta", ctgOpUpdateViewMeta}, {CTG_OP_DROP_VIEW_META, "drop viewMeta", ctgOpDropViewMeta}, - {CTG_OP_UPDATE_GRANT_INFO, "update grantInfo", ctgOpUpdateGrantInfo}, - {CTG_OP_DROP_GRANT_INFO, "drop grantInfo", ctgOpDropGrantInfo}, {CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}}; SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = { @@ -160,15 +158,6 @@ int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } -int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { - CTG_LOCK(CTG_READ, &pCtg->grantCache.lock); - *ppCache = &pCtg->grantCache; - CTG_CACHE_HIT_INC(CTG_CI_GRANT_INFO, 1); - return TSDB_CODE_SUCCESS; -} - -void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { CTG_UNLOCK(CTG_READ, &pCache->lock); } - void ctgReleaseVgInfoToCache(SCatalog *pCtg, SCtgDBCache *dbCache) { ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); @@ -1334,33 +1323,6 @@ _return: CTG_RET(code); } -int32_t ctgUpdateGrantInfoEnqueue(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncOp) { - int32_t code = 0; - SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); - op->opId = CTG_OP_UPDATE_GRANT_INFO; - op->syncOp = syncOp; - - SCtgUpdateGrantInfoMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateGrantInfoMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateGrantInfoMsg)); - taosMemoryFree(op); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - - msg->pCtg = pCtg; - msg->pRsp = pRsp; - - op->data = msg; - - CTG_ERR_RET(ctgEnqueue(pCtg, op)); - - return TSDB_CODE_SUCCESS; -_return: - - taosMemoryFree(pRsp); - CTG_RET(code); -} - int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *viewName, uint64_t viewId, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); @@ -1732,21 +1694,6 @@ _return: CTG_RET(code); } -int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { - int32_t code = TSDB_CODE_SUCCESS; - SCtgGrantCache *pCache = &pCtg->grantCache; - - CTG_LOCK(CTG_WRITE, &pCache->lock); - pCache->grantInfo = *pRsp; - CTG_UNLOCK(CTG_WRITE, &pCache->lock); - - ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); - -_return: - - CTG_RET(code); -} - int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool syncReq) { STableMetaOutput *pOutput = NULL; int32_t code = 0; @@ -1768,10 +1715,6 @@ int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncRe CTG_RET(ctgUpdateViewMetaEnqueue(pCtg, pRsp, syncReq)); } -int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq) { - CTG_RET(ctgUpdateGrantInfoEnqueue(pCtg, pRsp, syncReq)); -} - void ctgClearAllHandles(void) { SCatalog *pCtg = NULL; @@ -2507,43 +2450,6 @@ _return: CTG_RET(code); } -int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { - int32_t code = 0; - SCtgUpdateGrantInfoMsg *msg = operation->data; - SCatalog *pCtg = msg->pCtg; - SGrantHbRsp *pRsp = msg->pRsp; - SCtgGrantCache *pGrantCache = NULL; - - taosMemoryFreeClear(msg); - - if (pCtg->stopUpdate) { - goto _return; - } - - code = ctgWriteGrantInfoToCache(pCtg, pRsp); - -_return: - - taosMemoryFree(pRsp); - CTG_RET(code); -} - -int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { - int32_t code = 0; - SCtgDropGrantInfoMsg *msg = operation->data; - SCatalog *pCtg = msg->pCtg; - - if (pCtg->stopUpdate) { - goto _return; - } - -_return: - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index a229d2658e..4ac8e0e541 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2142,10 +2142,6 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt int32_t code = 0; int32_t numOfRows = 0; - // if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { - // return code; - // } - // init only for file if (NULL == pStmt->pTableCxtHashObj) { pStmt->pTableCxtHashObj = From c5f5aca4ce19f3bb0128b46a20e1d0f69303a7dd Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 17:35:34 +0800 Subject: [PATCH 30/45] fix: fix case passed --- tests/army/community/cluster/snapshot.json | 4 ++-- tests/army/community/cluster/snapshot.py | 8 +++++--- tests/army/community/storage/oneStageComp.json | 4 ++-- tests/army/community/storage/oneStageComp.py | 11 +++++++++++ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/army/community/cluster/snapshot.json b/tests/army/community/cluster/snapshot.json index f664278a72..4855c23260 100644 --- a/tests/army/community/cluster/snapshot.json +++ b/tests/army/community/cluster/snapshot.json @@ -35,8 +35,8 @@ "start_timestamp":"now-12d", "columns": [ { "type": "bool", "name": "bc"}, - { "type": "float", "name": "fc", "min": 100, "min": 100}, - { "type": "double", "name": "dc", "min": 200, "min": 200}, + { "type": "float", "name": "fc", "min": 100, "max": 100}, + { "type": "double", "name": "dc", "min": 200, "max": 200}, { "type": "tinyint", "name": "ti"}, { "type": "smallint", "name": "si" }, { "type": "int", "name": "ic" }, diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py index 6fd2218344..b4c4d3c4c8 100644 --- a/tests/army/community/cluster/snapshot.py +++ b/tests/army/community/cluster/snapshot.py @@ -53,10 +53,12 @@ class TDTestCase(TBase): tdSql.execute(sql) def checkFloatDouble(self): - sql = f"select count(*) from {self.db}.{self.stb} where fc!=100" - tdSql.checkFirstValue(sql, 0) + sql = f"select * from {self.db}.{self.stb} where fc!=100" + tdSql.query(sql) + tdSql.checkRows(0) sql = f"select count(*) from {self.db}.{self.stb} where dc!=200" - tdSql.checkFirstValue(sql, 0) + tdSql.query(sql) + tdSql.checkRows(0) sql = f"select avg(fc) from {self.db}.{self.stb}" tdSql.checkFirstValue(sql, 100) sql = f"select avg(dc) from {self.db}.{self.stb}" diff --git a/tests/army/community/storage/oneStageComp.json b/tests/army/community/storage/oneStageComp.json index 12fa51db83..f64fda3824 100644 --- a/tests/army/community/storage/oneStageComp.json +++ b/tests/army/community/storage/oneStageComp.json @@ -24,7 +24,7 @@ "stt_trigger": 1, "wal_level": 2, "WAL_FSYNC_PERIOD": 3300, - "cachemode": "last_value", + "cachemodel": "'last_value'", "TABLE_PREFIX":1, "comp": 1 }, @@ -37,7 +37,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"2023-01-01 00:00:00", + "start_timestamp":"now-360d", "columns": [ { "type": "bool", "name": "bc","max": 1,"min": 1}, { "type": "float", "name": "fc" ,"max": 101,"min": 101}, diff --git a/tests/army/community/storage/oneStageComp.py b/tests/army/community/storage/oneStageComp.py index f3718bc716..35a7717449 100644 --- a/tests/army/community/storage/oneStageComp.py +++ b/tests/army/community/storage/oneStageComp.py @@ -53,26 +53,37 @@ class TDTestCase(TBase): # check all columns correct cnt = self.insert_rows * self.childtable_count sql = "select * from stb where bc!=1" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where fc=101" + tdSql.query(sql) tdSql.checkRows(cnt) sql = "select * from stb where dc!=102" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where ti!=103" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where si!=104" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where ic!=105" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where b!i=106" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where uti!=107" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where usi!=108" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where ui!=109" + tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where ubi!=110" + tdSql.query(sql) tdSql.checkRows(0) def insertNull(self): From cb4cb936d32a09ef8390dce3c84dbb27c825e447 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 17:48:24 +0800 Subject: [PATCH 31/45] fix: case passed --- tests/army/community/storage/oneStageComp.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/army/community/storage/oneStageComp.py b/tests/army/community/storage/oneStageComp.py index 35a7717449..8860220673 100644 --- a/tests/army/community/storage/oneStageComp.py +++ b/tests/army/community/storage/oneStageComp.py @@ -70,7 +70,7 @@ class TDTestCase(TBase): sql = "select * from stb where ic!=105" tdSql.query(sql) tdSql.checkRows(0) - sql = "select * from stb where b!i=106" + sql = "select * from stb where bi!=106" tdSql.query(sql) tdSql.checkRows(0) sql = "select * from stb where uti!=107" @@ -97,10 +97,16 @@ class TDTestCase(TBase): # check all columns correct cnt = self.insert_rows * self.childtable_count sql = "select * from stb where bc!=1" + tdSql.query(sql) + tdSql.checkRows(0) + sql = "select * from stb where bc is null" + tdSql.query(sql) tdSql.checkRows(6) sql = "select * from stb where bc=1" + tdSql.query(sql) tdSql.checkRows(cnt) sql = "select * from stb where usi!=108" + tdSql.query(sql) tdSql.checkRows(6) # run From 60985381620e4e3ba028850724254a1195855f2a Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 18:00:02 +0800 Subject: [PATCH 32/45] feat: support uniq grant --- include/common/tgrant.h | 8 +++--- include/libs/catalog/catalog.h | 7 +----- source/client/src/clientHb.c | 2 ++ source/common/src/tglobal.c | 6 ++--- source/common/src/tmsg.c | 2 +- source/dnode/mnode/impl/inc/mndGrant.h | 5 ++-- source/dnode/mnode/impl/src/mndProfile.c | 1 - source/libs/catalog/inc/catalogInt.h | 5 ++-- source/libs/catalog/src/catalog.c | 3 +++ source/libs/catalog/src/ctgCache.c | 1 + source/libs/catalog/src/ctgRent.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 10 +++++--- source/libs/parser/inc/parTranslater.h | 2 -- source/libs/parser/src/parInsertSql.c | 11 +++----- source/libs/parser/src/parUtil.c | 2 -- source/libs/qworker/src/qwMsg.c | 32 ++++++++++++------------ 16 files changed, 48 insertions(+), 50 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 66408e9fa4..03c68ef59b 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -30,10 +30,10 @@ extern "C" { #define GRANT_HEART_BEAT_MIN 2 #define GRANT_ACTIVE_CODE "activeCode" -#define GRANT_ALL_FLAG (0x01) -#define GRANT_AUDIT_FLAG (0x02) -#define GRANT_CSV_FLAG (0x04) -#define GRANT_VIEW_FLAG (0x08) +#define GRANT_FLAG_ALL (0x01) +#define GRANT_FLAG_AUDIT (0x02) +#define GRANT_FLAG_CSV (0x04) +#define GRANT_FLAG_VIEW (0x08) typedef enum { TSDB_GRANT_ALL, diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 96a05ccbf9..1957bcbb24 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -25,7 +25,6 @@ extern "C" { #include "taosdef.h" #include "tarray.h" #include "tcommon.h" -#include "tgrant.h" #include "thash.h" #include "tmsg.h" #include "tname.h" @@ -146,10 +145,6 @@ typedef struct SSTableVersion { int32_t smaVer; } SSTableVersion; -typedef struct SGrantVersion { - int32_t version; -} SGrantVersion; - typedef struct SDbCacheInfo { char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; @@ -363,7 +358,7 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes); -int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pAuth, SUserAuthRes* pRes, bool* exists); +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists); int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 53a64cec86..a146712cab 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,6 +327,7 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } + static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pKvs, i); @@ -865,6 +866,7 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S return TSDB_CODE_SUCCESS; } + int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL != pApp) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d46b91276b..444a4c0ccc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1804,11 +1804,11 @@ void taosSetAllDebugFlag(int32_t flag) { int8_t taosGranted(int8_t type) { switch (type) { case TSDB_GRANT_ALL: - return atomic_load_8(&tsGrant) & GRANT_ALL_FLAG; + return atomic_load_8(&tsGrant) & GRANT_FLAG_ALL; case TSDB_GRANT_AUDIT: - return atomic_load_8(&tsGrant) & GRANT_AUDIT_FLAG; + return atomic_load_8(&tsGrant) & GRANT_FLAG_AUDIT; case TSDB_GRANT_VIEW: - return atomic_load_8(&tsGrant) & GRANT_VIEW_FLAG; + return atomic_load_8(&tsGrant) & GRANT_FLAG_VIEW; default: ASSERTS(0, "undefined grant type:%" PRIi8, type); break; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 785cb1878e..af1c1c4923 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9333,4 +9333,4 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) { } taosArrayDestroy(pRsp->pViewRsp); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index bda5cebf01..82b3260860 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -41,8 +41,9 @@ char **mergeActive); #endif - int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); - int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); + int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); + int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); + int32_t mndGrantGetLastState(SMnode * pMnode, SGrantState * pState); SGrantLogObj *mndAcquireGrant(SMnode * pMnode, void **ppIter); void mndReleaseGrant(SMnode * pMnode, SGrantLogObj * pGrant, void *pIter); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index f42cdfa4f3..e0559b4c48 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -18,7 +18,6 @@ #include "audit.h" #include "mndDb.h" #include "mndDnode.h" -#include "mndGrant.h" #include "mndMnode.h" #include "mndPrivilege.h" #include "mndQnode.h" diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a35eec0ec5..7474b3bf91 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -548,6 +548,7 @@ typedef struct SCtgDropViewMetaMsg { uint64_t viewId; } SCtgDropViewMetaMsg; + typedef struct SCtgCacheOperation { int32_t opId; void* data; @@ -943,8 +944,8 @@ void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache); void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache); int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid, SCtgTbCache *pCache); -int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, - SCtgViewCache *pCache); +int32_t ctgUpdateRentViewVersion(SCatalog* pCtg, char* dbFName, char* viewName, uint64_t dbId, uint64_t viewId, + SCtgViewCache* pCache); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 5a53f167c4..c2d88e5ce3 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1513,6 +1513,7 @@ int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* n CTG_API_LEAVE(ctgMetaRentGet(&pCtg->viewRent, (void**)views, num, sizeof(SViewVersion))); } + int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { CTG_API_ENTER(); @@ -1675,6 +1676,7 @@ _return: CTG_API_LEAVE(code); } + int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) { CTG_API_ENTER(); @@ -1745,6 +1747,7 @@ _return: CTG_API_LEAVE(code); } + int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 08a8f348e9..1b693b4e07 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2450,6 +2450,7 @@ _return: CTG_RET(code); } + void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 0ac133b8e7..f5016e9fef 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,3 +301,4 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } + diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index b05c9b019e..7533d00704 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3294,6 +3294,8 @@ static const char* jkSubplanTagIndexCond = "TagIndexCond"; static const char* jkSubplanShowRewrite = "ShowRewrite"; static const char* jkSubplanRowsThreshold = "RowThreshold"; static const char* jkSubplanDynamicRowsThreshold = "DyRowThreshold"; +static const char* jkSubplanIsView = "IsView"; +static const char* jkSubplanIsAudit = "IsAudit"; static int32_t subplanToJson(const void* pObj, SJson* pJson) { const SSubplan* pNode = (const SSubplan*)pObj; @@ -3333,10 +3335,10 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) { code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->showRewrite); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isView); + code = tjsonAddBoolToObject(pJson, jkSubplanIsView, pNode->isView); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isAudit); + code = tjsonAddBoolToObject(pJson, jkSubplanIsAudit, pNode->isAudit); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkSubplanRowsThreshold, pNode->rowsThreshold); @@ -3386,10 +3388,10 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->showRewrite); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isView); + code = tjsonGetBoolValue(pJson, jkSubplanIsView, &pNode->isView); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isAudit); + code = tjsonGetBoolValue(pJson, jkSubplanIsAudit, &pNode->isAudit); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanRowsThreshold, &pNode->rowsThreshold); diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h index 00f37cb002..55ea71a368 100644 --- a/source/libs/parser/inc/parTranslater.h +++ b/source/libs/parser/inc/parTranslater.h @@ -43,8 +43,6 @@ typedef struct STranslateContext { bool createStream; bool stableQuery; bool showRewrite; - // bool isView; - // bool isAudit; SNode* pPrevRoot; SNode* pPostRoot; } STranslateContext; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 4ac8e0e541..7d10d1f2df 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2139,16 +2139,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { - int32_t code = 0; - int32_t numOfRows = 0; - // init only for file if (NULL == pStmt->pTableCxtHashObj) { pStmt->pTableCxtHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } - - code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); + int32_t numOfRows = 0; + int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; @@ -2759,11 +2756,11 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOp } static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SRowsDataContext rowsDataCxt; if (!pStmt->stbSyntax) { - STableDataCxt* pTableCxt = NULL; + STableDataCxt* pTableCxt = NULL; code = getTableDataCxt(pCxt, pStmt, &pTableCxt); rowsDataCxt.pTableDataCxt = pTableCxt; } else { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 8aa37f5784..dfe33ce55e 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -192,8 +192,6 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Out of memory"; case TSDB_CODE_PAR_ORDERBY_AMBIGUOUS: return "ORDER BY \"%s\" is ambiguous"; - case TSDB_CODE_GRANT_EXPIRED: - return "License expired"; default: return "Unknown error"; } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index c9d3cb51e3..66ec460861 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -361,22 +361,22 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran } if (chkGrant) { - if ((!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_ALL)) { - QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); - } - - if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) { - QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); - } - - if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) { - QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + if ((!TEST_SHOW_REWRITE_MASK(msg.msgMask))) { + if (!taosGranted(TSDB_GRANT_ALL)) { + QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) { + QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) { + QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } } } From 8360d6a12b126b4a82152f50898ed288fc87f5c5 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sun, 4 Feb 2024 18:01:27 +0800 Subject: [PATCH 33/45] fix: check null value --- tests/army/community/storage/oneStageComp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/storage/oneStageComp.py b/tests/army/community/storage/oneStageComp.py index 8860220673..9a2c7cfcd6 100644 --- a/tests/army/community/storage/oneStageComp.py +++ b/tests/army/community/storage/oneStageComp.py @@ -105,7 +105,7 @@ class TDTestCase(TBase): sql = "select * from stb where bc=1" tdSql.query(sql) tdSql.checkRows(cnt) - sql = "select * from stb where usi!=108" + sql = "select * from stb where usi is null" tdSql.query(sql) tdSql.checkRows(6) From b05141d3983facbf496c521a778e900100bc6cbf Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Sun, 4 Feb 2024 18:12:04 +0800 Subject: [PATCH 34/45] fix app name on windows --- source/os/src/osSemaphore.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index dda4b14901..7d1cc746ff 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -56,6 +56,8 @@ int32_t taosGetAppName(char* name, int32_t* len) { char* end = strrchr(filepath, TD_DIRSEP[0]); if (end == NULL) { end = filepath; + } else { + end += 1; } tstrncpy(name, end, TSDB_APP_NAME_LEN); From 021247c45bd7b88395184ab9b98b55ea4c9d6282 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 06:51:44 +0800 Subject: [PATCH 35/45] feat: support uniq grant --- include/common/tgrant.h | 3 +-- include/libs/qcom/query.h | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 03c68ef59b..2fa6dde8f6 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -32,8 +32,7 @@ extern "C" { #define GRANT_ACTIVE_CODE "activeCode" #define GRANT_FLAG_ALL (0x01) #define GRANT_FLAG_AUDIT (0x02) -#define GRANT_FLAG_CSV (0x04) -#define GRANT_FLAG_VIEW (0x08) +#define GRANT_FLAG_VIEW (0x04) typedef enum { TSDB_GRANT_ALL, diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 4c33c36d1b..e80268bb5f 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -66,8 +66,8 @@ typedef enum { #define QUERY_RSP_POLICY_QUICK 1 #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) -#define QUERY_MSG_MASK_VIEW() (1 << 1) -#define QUERY_MSG_MASK_AUDIT() (1 << 2) +#define QUERY_MSG_MASK_AUDIT() (1 << 1) +#define QUERY_MSG_MASK_VIEW() (1 << 2) #define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) #define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) #define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_AUDIT()) != 0) From 205525dd652dbf89de91f47d63e780e75f0ee2b6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 07:03:51 +0800 Subject: [PATCH 36/45] feat: support uniq grant --- include/libs/qcom/query.h | 7 +++---- source/libs/catalog/inc/catalogInt.h | 4 ++-- source/libs/catalog/src/ctgRent.c | 2 -- source/libs/scheduler/src/schRemote.c | 1 - 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index e80268bb5f..8d6cd6a3c0 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -68,10 +68,9 @@ typedef enum { #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) #define QUERY_MSG_MASK_AUDIT() (1 << 1) #define QUERY_MSG_MASK_VIEW() (1 << 2) -#define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) -#define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) -#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_AUDIT()) != 0) - +#define TEST_SHOW_REWRITE_MASK(m) (((m) & QUERY_MSG_MASK_SHOW_REWRITE()) != 0) +#define TEST_AUDIT_MASK(m) (((m) & QUERY_MSG_MASK_AUDIT()) != 0) +#define TEST_VIEW_MASK(m) (((m) & QUERY_MSG_MASK_VIEW()) != 0) typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 7474b3bf91..f9f4ee7dfc 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -944,8 +944,8 @@ void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache); void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache); int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid, SCtgTbCache *pCache); -int32_t ctgUpdateRentViewVersion(SCatalog* pCtg, char* dbFName, char* viewName, uint64_t dbId, uint64_t viewId, - SCtgViewCache* pCache); +int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, + SCtgViewCache *pCache); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index f5016e9fef..a19eb19c02 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -300,5 +300,3 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } - - diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1a9c37ffe9..1c0b31109e 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1111,7 +1111,6 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.msgMask = (pTask->plan->showRewrite) ? QUERY_MSG_MASK_SHOW_REWRITE() : 0; qMsg.msgMask |= (pTask->plan->isView) ? QUERY_MSG_MASK_VIEW() : 0; qMsg.msgMask |= (pTask->plan->isAudit) ? QUERY_MSG_MASK_AUDIT() : 0; - qMsg.taskType = TASK_TYPE_TEMP; qMsg.explain = SCH_IS_EXPLAIN_JOB(pJob); qMsg.needFetch = SCH_TASK_NEED_FETCH(pTask); From 739485a41970ba5e88eed18f00a13b6b61250e27 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 5 Feb 2024 09:10:04 +0800 Subject: [PATCH 37/45] test(stream): add unit test case for the mnode stream module. --- include/libs/stream/streamState.h | 6 ++--- include/libs/stream/tstream.h | 6 ++--- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/inc/mndStream.h | 1 + source/dnode/mnode/impl/src/mndStream.c | 26 ++----------------- source/dnode/mnode/impl/src/mndStreamHb.c | 12 ++++++--- source/dnode/mnode/impl/src/mndStreamUtil.c | 24 +++++++++++++++++ source/dnode/mnode/impl/test/CMakeLists.txt | 2 +- .../mnode/impl/test/stream/CMakeLists.txt | 13 ++++++++++ 9 files changed, 56 insertions(+), 36 deletions(-) create mode 100644 source/dnode/mnode/impl/test/stream/CMakeLists.txt diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 24222677a4..c2f7c6de2f 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -13,6 +13,9 @@ * along with this program. If not, see . */ +#ifndef _STREAM_STATE_H_ +#define _STREAM_STATE_H_ + #include "tdatablock.h" #include "rocksdb/c.h" @@ -20,9 +23,6 @@ #include "tsimplehash.h" #include "tstreamFileState.h" -#ifndef _STREAM_STATE_H_ -#define _STREAM_STATE_H_ - #ifdef __cplusplus extern "C" { #endif diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index dce8fffe11..2135bb706b 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -13,6 +13,9 @@ * along with this program. If not, see . */ +#ifndef _STREAM_H_ +#define _STREAM_H_ + #include "os.h" #include "streamState.h" #include "tdatablock.h" @@ -26,9 +29,6 @@ extern "C" { #endif -#ifndef _STREAM_H_ -#define _STREAM_H_ - #define ONE_MiB_F (1048576.0) #define ONE_KiB_F (1024.0) #define SIZE_IN_MiB(_v) ((_v) / ONE_MiB_F) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index b056d561c7..f2b94aa1d4 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -553,7 +553,7 @@ typedef struct { } SMqConsumerObj; SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]); -void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer, bool delete); +void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer, bool isDeleted); int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer); void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer, int8_t sver); diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 372612274f..4d1125a340 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -124,6 +124,7 @@ SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream); void destroyStreamTaskIter(SStreamTaskIter *pIter); bool streamTaskIterNextTask(SStreamTaskIter *pIter); SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter); +void mndInitExecInfo(); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 18cecddbdb..d51a740b70 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -62,8 +62,6 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP static void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot); static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len); -static void freeCheckpointCandEntry(void *); -static void freeTaskList(void *param); static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); SSdbRaw *mndStreamSeqActionEncode(SStreamObj *pStream); @@ -121,17 +119,7 @@ int32_t mndInitStream(SMnode *pMnode) { mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask); - taosThreadMutexInit(&execInfo.lock, NULL); - _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); - - execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId)); - execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK); - execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK); - execInfo.transMgmt.pWaitingList = taosHashInit(32, fn, true, HASH_NO_LOCK); - execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK); - - taosHashSetFreeFp(execInfo.transMgmt.pWaitingList, freeCheckpointCandEntry); - taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); + mndInitExecInfo(); if (sdbSetTable(pMnode->pSdb, table) != 0) { return -1; @@ -2118,16 +2106,6 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); } -void freeCheckpointCandEntry(void *param) { - SCheckpointCandEntry *pEntry = param; - taosMemoryFreeClear(pEntry->pName); -} - -void freeTaskList(void* param) { - SArray** pList = (SArray **)param; - taosArrayDestroy(*pList); -} - static void doAddTaskId(SArray* pList, int32_t taskId, int64_t uid, int32_t numOfTotal) { int32_t num = taosArrayGetSize(pList); for(int32_t i = 0; i < num; ++i) { @@ -2203,4 +2181,4 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { taosThreadMutexUnlock(&execInfo.lock); return 0; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 4426ab0672..49a931bba8 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -316,16 +316,20 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { // current checkpoint is failed, rollback from the checkpoint trans // kill the checkpoint trans and then set all tasks status to be normal if (taosArrayGetSize(pFailedTasks) > 0) { - bool allReady = true; - SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady); - taosArrayDestroy(p); + bool allReady = true; + if (pMnode != NULL) { + SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady); + taosArrayDestroy(p); + } else { + allReady = false; + } if (allReady || snodeChanged) { // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal for(int32_t i = 0; i < taosArrayGetSize(pFailedTasks); ++i) { SFailedCheckpointInfo *pInfo = taosArrayGet(pFailedTasks, i); mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status", - pInfo->checkpointId, pInfo->transId); + pInfo->checkpointId, pInfo->transId); mndResetStatusFromCheckpoint(pMnode, pInfo->streamUid, pInfo->transId); } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 235c604b27..3cabce2201 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -543,3 +543,27 @@ int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj * taosWUnLockLatch(&pStream->lock); return 0; } + +static void freeCheckpointCandEntry(void *param) { + SCheckpointCandEntry *pEntry = param; + taosMemoryFreeClear(pEntry->pName); +} + +static void freeTaskList(void* param) { + SArray** pList = (SArray **)param; + taosArrayDestroy(*pList); +} + +void mndInitExecInfo() { + taosThreadMutexInit(&execInfo.lock, NULL); + _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); + + execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId)); + execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK); + execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK); + execInfo.transMgmt.pWaitingList = taosHashInit(32, fn, true, HASH_NO_LOCK); + execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK); + + taosHashSetFreeFp(execInfo.transMgmt.pWaitingList, freeCheckpointCandEntry); + taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); +} diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index a002b20bde..bc5b5125f1 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -4,7 +4,7 @@ add_subdirectory(acct) #add_subdirectory(db) #add_subdirectory(dnode) add_subdirectory(func) -#add_subdirectory(mnode) +add_subdirectory(stream) add_subdirectory(profile) add_subdirectory(qnode) add_subdirectory(sdb) diff --git a/source/dnode/mnode/impl/test/stream/CMakeLists.txt b/source/dnode/mnode/impl/test/stream/CMakeLists.txt new file mode 100644 index 0000000000..b1bb62735f --- /dev/null +++ b/source/dnode/mnode/impl/test/stream/CMakeLists.txt @@ -0,0 +1,13 @@ +SET(CMAKE_CXX_STANDARD 11) + +aux_source_directory(. MNODE_STREAM_TEST_SRC) +add_executable(streamTest ${MNODE_STREAM_TEST_SRC}) +target_link_libraries( + streamTest + PRIVATE dnode gtest +) + +add_test( + NAME streamTest + COMMAND streamTest +) From 3111a01a1202fc1e76809d044741335391675e5a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 5 Feb 2024 09:10:43 +0800 Subject: [PATCH 38/45] test: add unit test files. --- .../dnode/mnode/impl/test/stream/stream.cpp | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 source/dnode/mnode/impl/test/stream/stream.cpp diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp new file mode 100644 index 0000000000..31ba89f3ca --- /dev/null +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include +#include +#include "../../inc/mndStream.h" + +namespace { +SRpcMsg buildHbReq() { + SStreamHbMsg msg = {0}; + msg.vgId = 1; + msg.numOfTasks = 5; + msg.pTaskStatus = taosArrayInit(4, sizeof(STaskStatusEntry)); + + for (int32_t i = 0; i < 4; ++i) { + STaskStatusEntry entry = {0}; + entry.nodeId = i + 1; + entry.stage = 1; + entry.id.taskId = i + 1; + entry.id.streamId = 999; + + if (i == 0) { + entry.stage = 4; + } + + taosArrayPush(msg.pTaskStatus, &entry); + } + + // (p->checkpointId != 0) && p->checkpointFailed + // add failed checkpoint info + { + STaskStatusEntry entry = {0}; + entry.nodeId = 5; + entry.stage = 1; + + entry.id.taskId = 5; + entry.id.streamId = 999; + + entry.checkpointId = 1; + entry.checkpointFailed = true; + + taosArrayPush(msg.pTaskStatus, &entry); + } + + int32_t tlen = 0; + int32_t code = 0; + SEncoder encoder; + void* buf = NULL; + SRpcMsg msg1 = {0}; + msg1.info.noResp = 1; + + tEncodeSize(tEncodeStreamHbMsg, &msg, tlen, code); + if (code < 0) { + goto _end; + } + + buf = rpcMallocCont(tlen); + if (buf == NULL) { + goto _end; + } + + tEncoderInit(&encoder, (uint8_t*)buf, tlen); + if ((code = tEncodeStreamHbMsg(&encoder, &msg)) < 0) { + rpcFreeCont(buf); + goto _end; + } + tEncoderClear(&encoder); + + initRpcMsg(&msg1, TDMT_MND_STREAM_HEARTBEAT, buf, tlen); + + taosArrayDestroy(msg.pTaskStatus); + return msg1; + +_end: + return msg1; +} + +void setTask(SStreamTask* pTask, int32_t nodeId, int64_t streamId, int32_t taskId) { + SStreamExecInfo* pExecNode = &execInfo; + + pTask->id.streamId = streamId; + pTask->id.taskId = taskId; + pTask->info.nodeId = nodeId; + + STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; + STaskStatusEntry entry = {0}; + streamTaskStatusInit(&entry, pTask); + + entry.stage = 1; + entry.status = TASK_STATUS__READY; + + taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry)); + taosArrayPush(pExecNode->pTaskList, &id); +} +void initStreamExecInfo() { + SStreamExecInfo* pExecNode = &execInfo; + + SStreamTask task = {0}; + setTask(&task, 1, 999, 1); + setTask(&task, 1, 999, 2); + setTask(&task, 1, 999, 3); + setTask(&task, 1, 999, 4); + setTask(&task, 2, 999, 5); +} + +void initNodeInfo() { + execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry)); + SNodeEntry entry = {0}; + entry.nodeId = 2; + entry.stageUpdated = true; + taosArrayPush(execInfo.pNodeList, &entry); +} +} // namespace + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(mndHbTest, handle_error_in_hb) { + mndInitExecInfo(); + initStreamExecInfo(); + initNodeInfo(); + + SRpcMsg msg = buildHbReq(); + int32_t code = mndProcessStreamHb(&msg); + + rpcFreeCont(msg.pCont); +} + +#pragma GCC diagnostic pop \ No newline at end of file From 7e5dc8a02425a7826ea7195a77c0a812ce509e30 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 09:21:11 +0800 Subject: [PATCH 39/45] feat: support uniq grant --- source/dnode/mnode/impl/src/mndCluster.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 36b079d347..f2b279276e 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -409,7 +409,7 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) { } { // audit - auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, MIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1)); + auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, TMIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1)); } _exit: tFreeSMCfgClusterReq(&cfgReq); From 196256d9af4b2de3effba0c1bafd7bf6b025b5d0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 09:23:19 +0800 Subject: [PATCH 40/45] feat: support uniq grant --- source/dnode/mnode/impl/src/mndGrant.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index 0b85f8fd5a..2ec6e09d12 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -79,11 +79,6 @@ char *tGetMachineId() { return NULL; }; int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } int32_t mndProcessConfigGrantReq(SMnode *pMnode, SRpcMsg *pReq, SMCfgClusterReq *pCfg) { return 0; } -#else -#ifndef TD_UNIQ_GRANT -char *tGetMachineId() { return NULL; }; -int32_t mndProcessConfigGrantReq(SMnode *pMnode, SRpcMsg *pReq, SMCfgClusterReq *pCfg) { return 0; } -#endif #endif void mndGenerateMachineCode() { grantParseParameter(); } \ No newline at end of file From f6fea32eebb542d84c0bd3cf6de4ed82e9ae329c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 5 Feb 2024 09:27:19 +0800 Subject: [PATCH 41/45] fix: fix some errors. --- source/libs/parser/src/parTranslater.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4ffb5bfae8..51d37e0a35 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8209,7 +8209,7 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta return code; } - code = nodesListAppend((*pSelect1)->pGroupByList, (SNode*)pNode1); + code = nodesListAppend((*pSelect1)->pGroupByList, nodesCloneNode((const SNode*)pNode1)); if (code) { return code; } @@ -8222,18 +8222,17 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta pNode2->groupingSetType = GP_TYPE_NORMAL; pNode2->pParameterList = nodesMakeList(); if (NULL == pNode2->pParameterList) { - nodesDestroyNode((SNode*)pNode1); + nodesDestroyNode((SNode*)pNode2); return TSDB_CODE_OUT_OF_MEMORY; } - code = nodesListAppend(pNode2->pParameterList, (SNode*)pFunc2); + code = nodesListAppend(pNode2->pParameterList, nodesCloneNode((const SNode*)pFunc2)); if (code) { nodesDestroyNode((SNode*)pNode2); return code; } - code = nodesListAppend((*pSelect1)->pGroupByList, (SNode*)pNode2); - return code; + return nodesListAppend((*pSelect1)->pGroupByList, (SNode*)pNode2); } static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { From c56cebdd49f9c89cd0363e2fbc5d344d6471f9e8 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 5 Feb 2024 09:34:18 +0800 Subject: [PATCH 42/45] opt ci --- tests/script/tsim/stream/windowClose.sim | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index 775ff81f51..ce5c57572e 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -304,7 +304,7 @@ sql insert into t1 values(1648791233001,2,2,3,1.1); $loop_count = 0 -loop2: +loop4: sleep 1000 @@ -317,7 +317,7 @@ sql select * from streamt13; if $rows != 2 then print ======rows=$rows - goto loop2 + goto loop4 endi $now02 = $data02 @@ -356,12 +356,14 @@ if $data12 != $now12 then return -1 endi +print session max delay over + print step 3 max delay 2s sql create database test16 vgroups 4; sql use test16; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream stream16 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 state_window(a, 10s); +sql create stream stream16 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 state_window(a); sleep 1000 @@ -370,7 +372,7 @@ sql insert into t1 values(1648791233001,2,2,3,1.1); $loop_count = 0 -loop2: +loop5: sleep 1000 @@ -383,7 +385,7 @@ sql select * from streamt13; if $rows != 2 then print ======rows=$rows - goto loop2 + goto loop5 endi $now02 = $data02 @@ -422,12 +424,14 @@ if $data12 != $now12 then return -1 endi +print state max delay over + print step 4 max delay 2s sql create database test17 vgroups 4; sql use test17; sql create table t1(ts timestamp, a int, b int , c int, d double); -sql create stream stream17 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 event_window start with a = 0 end with a = 9; +sql create stream stream17 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 event_window start with a = 1 end with a = 9; sleep 1000 @@ -439,7 +443,7 @@ sql insert into t1 values(1648791233009,9,2,3,1.1); $loop_count = 0 -loop2: +loop6: sleep 1000 @@ -452,7 +456,7 @@ sql select * from streamt13; if $rows != 2 then print ======rows=$rows - goto loop2 + goto loop6 endi $now02 = $data02 @@ -491,6 +495,8 @@ if $data12 != $now12 then return -1 endi +print event max delay over + print ======over system sh/exec.sh -n dnode1 -s stop -x SIGINT From fc218a1f69ecc497a5d03360b072836f7da646db Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 5 Feb 2024 11:09:19 +0800 Subject: [PATCH 43/45] fix(test): fix syntax errors on windows. --- source/dnode/mnode/impl/test/stream/stream.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index 31ba89f3ca..a3babad80c 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -103,8 +103,11 @@ void setTask(SStreamTask* pTask, int32_t nodeId, int64_t streamId, int32_t taskI pTask->id.taskId = taskId; pTask->info.nodeId = nodeId; - STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - STaskStatusEntry entry = {0}; + STaskId id; + id.streamId = pTask->id.streamId; + id.taskId = pTask->id.taskId; + + STaskStatusEntry entry; streamTaskStatusInit(&entry, pTask); entry.stage = 1; From 08cb46ac3e925c724edfb134ff1ad767ae8117ca Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 5 Feb 2024 15:10:03 +0800 Subject: [PATCH 44/45] stream count window check point --- .../executor/src/streamcountwindowoperator.c | 2 +- source/libs/stream/src/streamSessionState.c | 36 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 4a1a6aa2ad..1045480e7e 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -220,7 +220,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; - blockDataEnsureCapacity(pAggSup->pScanBlock, rows); + blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); SStreamStateCur* pCur = NULL; COUNT_TYPE slidingRows = 0; diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 9d5a594f46..69a4a6b5a3 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -103,7 +103,12 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); pNewPos->needFree = true; pNewPos->beFlushed = true; - memcpy(pNewPos->pRowBuff, p, *pVLen); + if(p) { + memcpy(pNewPos->pRowBuff, p, *pVLen); + } else { + int32_t len = getRowStateRowSize(pFileState); + memset(pNewPos->pRowBuff, 0, len); + } taosMemoryFree(p); return pNewPos; } @@ -727,21 +732,30 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C int32_t size = taosArrayGetSize(pWinStates); if (size == 0) { - void* pFileStore = getStateFileStore(pFileState); - void* p = NULL; - + void* pFileStore = getStateFileStore(pFileState); + void* pRockVal = NULL; SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileStore, pKey->groupId); - int32_t code_file = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &p, pVLen); - if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { - (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); - code = code_file; - qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); + code = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &pRockVal, pVLen); + if (code == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { + qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code); + if (code == TSDB_CODE_SUCCESS) { + int32_t valSize = *pVLen; + COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)( (char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE)) ); + if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) { + (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); + streamStateFreeCur(pCur); + goto _end; + } + } + pWinKey->win.skey = startTs; + pWinKey->win.ekey = endTs; + (*pVal) = createSessionWinBuff(pFileState, pWinKey, NULL, NULL); + taosMemoryFree(pRockVal); + streamStateFreeCur(pCur); } else { (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); code = TSDB_CODE_FAILED; - taosMemoryFree(p); } - streamStateFreeCur(pCur); goto _end; } From 891249191e1cd20aa63ba0a32b4dde54da97c57e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 5 Feb 2024 16:01:46 +0800 Subject: [PATCH 45/45] stream count window --- source/libs/executor/src/countwindowoperator.c | 2 +- source/libs/executor/src/operator.c | 2 ++ source/libs/executor/src/scanoperator.c | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c index e91afb2e38..ffed6a7788 100644 --- a/source/libs/executor/src/countwindowoperator.c +++ b/source/libs/executor/src/countwindowoperator.c @@ -66,7 +66,7 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pBlock->info.rows; i++) { - // 1.如果group id发生变化，获取新group id上一次的window的缓存，并把旧group id的信息存入缓存。 + // todo(liuyao) 1.如果group id发生变化，获取新group id上一次的window的缓存，并把旧group id的信息存入缓存。 // 2.计算 当前需要合并的行数 // 3.做聚集计算。 // 4.达到行数，将结果存入pInfo->res中。 diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 4c4e83fced..bf5d1b2019 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -541,6 +541,8 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR pOptr = createDynQueryCtrlOperatorInfo(ops, size, (SDynQueryCtrlPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT == type) { pOptr = createStreamCountAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT == type) { + pOptr = createCountwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else { terrno = TSDB_CODE_INVALID_PARA; pTaskInfo->code = terrno; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c056f33504..8747082c08 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1284,7 +1284,7 @@ static bool isSlidingWindow(SStreamScanInfo* pInfo) { } static bool isCountSlidingWindow(SStreamScanInfo* pInfo) { - return pInfo->windowSup.pStreamAggSup->windowCount != pInfo->windowSup.pStreamAggSup->windowSliding; + return pInfo->windowSup.pStreamAggSup && (pInfo->windowSup.pStreamAggSup->windowCount != pInfo->windowSup.pStreamAggSup->windowSliding); } static bool isCountWindow(SStreamScanInfo* pInfo) {