From 6e43da8cf5ef69465e792aa7ff1cf3d6af5d2060 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 30 May 2022 18:56:11 +0800 Subject: [PATCH 01/78] test:add test case for tail function --- tests/system-test/2-query/tail.py | 436 ++++++++++++++++++++++++++++++ 1 file changed, 436 insertions(+) create mode 100644 tests/system-test/2-query/tail.py diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py new file mode 100644 index 0000000000..30d9870316 --- /dev/null +++ b/tests/system-test/2-query/tail.py @@ -0,0 +1,436 @@ +from math import floor +from random import randint, random +from numpy import equal +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def test_errors(self): + error_sql_lists = [ + "select tail from t1", + "select tail(123--123)==1 from t1", + "select tail(123,123) from t1", + "select tail(c1,ts) from t1", + "select tail(c1,c1,ts) from t1", + "select tail(c1) as 'd1' from t1", + "select tail(c1 ,c2 ) from t1", + "select tail(c1 ,NULL) from t1", + "select tail(,) from t1;", + "select tail(floor(c1) ab from t1)", + "select tail(c1) as int from t1", + "select tail('c1') from t1", + "select tail(NULL) from t1", + "select tail('') from t1", + "select tail(c%) from t1", + "select tail(t1) from t1", + "select tail(True) from t1", + "select tail(c1,1) , count(c1) from t1", + "select tail(c1,1) , avg(c1) from t1", + "select tail(c1,1) , min(c1) from t1", + "select tail(c1,1) , spread(c1) from t1", + "select tail(c1,1) , diff(c1) from t1", + "select tail(c1,1) , abs(c1) from t1", + "select tail(c1,1) , c1 from t1", + "select tail from stb1 partition by tbname", + "select tail(123--123)==1 from stb1 partition by tbname", + "select tail(123,123) from stb1 partition by tbname", + "select tail(c1,ts) from stb1 partition by tbname", + "select tail(c1,c1,ts) from stb1 partition by tbname", + "select tail(c1) as 'd1' from stb1 partition by tbname", + "select tail(c1 ,c2 ) from stb1 partition by tbname", + "select tail(c1 ,NULL) from stb1 partition by tbname", + "select tail(,) from stb1 partition by tbname;", + "select tail(floor(c1) ab from stb1 partition by tbname)", + "select tail(c1) as int from stb1 partition by tbname", + "select tail('c1') from stb1 partition by tbname", + "select tail(NULL) from stb1 partition by tbname", + "select tail('') from stb1 partition by tbname", + "select tail(c%) from stb1 partition by tbname", + "select tail(t1) from stb1 partition by tbname", + "select tail(True) from stb1 partition by tbname", + "select tail(c1,1) , count(c1) from stb1 partition by tbname", + "select tail(c1,1) , avg(c1) from stb1 partition by tbname", + "select tail(c1,1) , min(c1) from stb1 partition by tbname", + "select tail(c1,1) , spread(c1) from stb1 partition by tbname", + "select tail(c1,1) , diff(c1) from stb1 partition by tbname", + "select tail(c1,1) , abs(c1) from stb1 partition by tbname", + "select tail(c1,1) , c1 from stb1 partition by tbname" + + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + other_no_value_types = [ + "select tail(ts,1) from t1" , + "select tail(c7,1) from t1", + "select tail(c8,1) from t1", + "select tail(c9,1) from t1", + "select tail(ts,1) from ct1" , + "select tail(c7,1) from ct1", + "select tail(c8,1) from ct1", + "select tail(c9,1) from ct1", + "select tail(ts,1) from ct3" , + "select tail(c7,1) from ct3", + "select tail(c8,1) from ct3", + "select tail(c9,1) from ct3", + "select tail(ts,1) from ct4" , + "select tail(c7,1) from ct4", + "select tail(c8,1) from ct4", + "select tail(c9,1) from ct4", + "select tail(ts,1) from stb1 partition by tbname" , + "select tail(c7,1) from stb1 partition by tbname", + "select tail(c8,1) from stb1 partition by tbname", + "select tail(c9,1) from stb1 partition by tbname" + ] + + for type_sql in other_no_value_types: + tdSql.query(type_sql) + + type_sql_lists = [ + "select tail(c1,1) from t1", + "select tail(c2,1) from t1", + "select tail(c3,1) from t1", + "select tail(c4,1) from t1", + "select tail(c5,1) from t1", + "select tail(c6,1) from t1", + + "select tail(c1,1) from ct1", + "select tail(c2,1) from ct1", + "select tail(c3,1) from ct1", + "select tail(c4,1) from ct1", + "select tail(c5,1) from ct1", + "select tail(c6,1) from ct1", + + "select tail(c1,1) from ct3", + "select tail(c2,1) from ct3", + "select tail(c3,1) from ct3", + "select tail(c4,1) from ct3", + "select tail(c5,1) from ct3", + "select tail(c6,1) from ct3", + + "select tail(c1,1) from stb1 partition by tbname", + "select tail(c2,1) from stb1 partition by tbname", + "select tail(c3,1) from stb1 partition by tbname", + "select tail(c4,1) from stb1 partition by tbname", + "select tail(c5,1) from stb1 partition by tbname", + "select tail(c6,1) from stb1 partition by tbname", + + "select tail(c6,1) as alisb from stb1 partition by tbname", + "select tail(c6,1) alisb from stb1 partition by tbname", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def check_tail_table(self , tbname , col_name , tail_rows , offset): + tail_sql = f"select tail({col_name} , {tail_rows} , {offset}) from {tbname}" + equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts" + tdSql.query(tail_sql) + tail_result = tdSql.queryResult + + tdSql.query(equal_sql) + + equal_result = tdSql.queryResult + + if tail_result == equal_result: + tdLog.info(" tail query check pass , tail sql is: %s" %tail_sql) + else: + tdLog.exit(" tail query check fail , tail sql is: %s " %tail_sql) + + def basic_tail_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select tail(c1,1) from ct3") + tdSql.checkRows(0) + tdSql.query("select tail(c2,1) from ct3") + tdSql.checkRows(0) + tdSql.query("select tail(c3,1) from ct3") + tdSql.checkRows(0) + tdSql.query("select tail(c4,1) from ct3") + tdSql.checkRows(0) + tdSql.query("select tail(c5,1) from ct3") + tdSql.checkRows(0) + tdSql.query("select tail(c6,1) from ct3") + + # auto check for t1 table + # used for regular table + tdSql.query("select tail(c1,1) from t1") + + tdSql.query("desc t1") + col_lists_rows = tdSql.queryResult + col_lists = [] + for col_name in col_lists_rows: + col_lists.append(col_name[0]) + + for col in col_lists: + for loop in range(100): + limit = randint(1,100) + offset = randint(0,100) + self.check_tail_table("t1" , col , limit , offset) + + # tail for invalid params + + tdSql.error("select tail(c1,-10,10) from ct1") + tdSql.error("select tail(c1,10,10000) from ct1") + tdSql.error("select tail(c1,10,-100) from ct1") + tdSql.error("select tail(c1,100/2,10) from ct1") + tdSql.error("select tail(c1,5,10*2) from ct1") + tdSql.query("select tail(c1,100,100) from ct1") + tdSql.checkRows(0) + tdSql.query("select tail(c1,10,100) from ct1") + tdSql.checkRows(0) + tdSql.error("select tail(c1,10,101) from ct1") + tdSql.query("select tail(c1,10,0) from ct1") + tdSql.query("select tail(c1,100,10) from ct1") + tdSql.checkRows(3) + + # tail with super tags + + tdSql.query("select tail(c1,10,10) from ct1") + tdSql.checkRows(3) + + tdSql.error("select tail(c1,10,10),tbname from ct1") + tdSql.error("select tail(c1,10,10),t1 from ct1") + + # tail with common col + tdSql.error("select tail(c1,10,10) ,ts from ct1") + tdSql.error("select tail(c1,10,10) ,c1 from ct1") + + # tail with scalar function + tdSql.error("select tail(c1,10,10) ,abs(c1) from ct1") + tdSql.error("select tail(c1,10,10) , tail(c2,10,10) from ct1") + tdSql.error("select tail(c1,10,10) , abs(c2)+2 from ct1") + + # bug need fix for scalar value or compute again + # tdSql.error(" select tail(c1,10,10) , 123 from ct1") + # tdSql.error(" select abs(tail(c1,10,10)) from ct1") + # tdSql.error(" select abs(tail(c1,10,10)) + 2 from ct1") + + # tail with aggregate function + tdSql.error("select tail(c1,10,10) ,sum(c1) from ct1") + tdSql.error("select tail(c1,10,10) ,max(c1) from ct1") + tdSql.error("select tail(c1,10,10) ,csum(c1) from ct1") + tdSql.error("select tail(c1,10,10) ,count(c1) from ct1") + + # tail with filter where + tdSql.query("select tail(c1,3,1) from ct4 where c1 is null") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + + tdSql.query("select tail(c1,3,2) from ct4 where c1 >2 ") + tdSql.checkData(0, 0, 7) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 5) + + tdSql.query("select tail(c1,2,1) from ct4 where c2 between 0 and 99999") + tdSql.checkData(0, 0, 2) + tdSql.checkData(1, 0, 1) + + # tail with union all + tdSql.query("select tail(c1,2,1) from ct4 union all select c1 from ct1") + tdSql.checkRows(15) + tdSql.query("select tail(c1,2,1) from ct4 union all select c1 from ct2") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 0) + tdSql.query("select tail(c2,2,1) from ct4 union all select abs(c2)/2 from ct4") + tdSql.checkRows(14) + + # tail with join + # prepare join datas with same ts + + tdSql.execute(" use db ") + tdSql.execute(" create stable st1 (ts timestamp , num int) tags(ind int)") + tdSql.execute(" create table tb1 using st1 tags(1)") + tdSql.execute(" create table tb2 using st1 tags(2)") + + tdSql.execute(" create stable st2 (ts timestamp , num int) tags(ind int)") + tdSql.execute(" create table ttb1 using st2 tags(1)") + tdSql.execute(" create table ttb2 using st2 tags(2)") + + start_ts = 1622369635000 # 2021-05-30 18:13:55 + + for i in range(10): + ts_value = start_ts+i*1000 + tdSql.execute(f" insert into tb1 values({ts_value} , {i})") + tdSql.execute(f" insert into tb2 values({ts_value} , {i})") + + tdSql.execute(f" insert into ttb1 values({ts_value} , {i})") + tdSql.execute(f" insert into ttb2 values({ts_value} , {i})") + + tdSql.query("select tail(tb2.num,3,2) from tb1, tb2 where tb1.ts=tb2.ts ") + tdSql.checkRows(3) + tdSql.checkData(0,0,5) + tdSql.checkData(1,0,6) + tdSql.checkData(2,0,7) + + # nest query + # tdSql.query("select tail(c1,2) from (select c1 from ct1)") + tdSql.query("select c1 from (select tail(c1,2) c1 from ct4)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, None) + + tdSql.query("select sum(c1) from (select tail(c1,2) c1 from ct1)") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 18) + + tdSql.query("select abs(c1) from (select tail(c1,2) c1 from ct1)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 9) + + #partition by tbname + tdSql.query(" select tail(c1,5) from stb1 partition by tbname ") + tdSql.checkRows(10) + + tdSql.query(" select tail(c1,3) from stb1 partition by tbname ") + tdSql.checkRows(6) + + # group by + tdSql.error("select tail(c1,2) from ct1 group by c1") + tdSql.error("select tail(c1,2) from ct1 group by tbname") + + # super table + + + + + def check_boundary_values(self): + + tdSql.execute("drop database if exists bound_test") + tdSql.execute("create database if not exists bound_test") + tdSql.execute("use bound_test") + tdSql.execute( + "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + ) + tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute( + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), 2147483643, 9223372036854775803, 32763, 123, 3.39E+38, 1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.execute( + f"insert into sub1_bound values ( now(), -2147483643, -9223372036854775803, -32763, -123, -3.39E+38, -1.69e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.error( + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + + tdSql.query("select tail(c2,2) from sub1_bound") + tdSql.checkRows(2) + tdSql.checkData(0,0,9223372036854775803) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + self.prepare_datas() + + tdLog.printNoPrefix("==========step2:test errors ==============") + + self.test_errors() + + tdLog.printNoPrefix("==========step3:support types ============") + + self.support_types() + + tdLog.printNoPrefix("==========step4: floor basic query ============") + + self.basic_tail_function() + + tdLog.printNoPrefix("==========step5: floor boundary query ============") + + self.check_boundary_values() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 635d3b2921a76f2557dd64d0dc765433fcdb4a8f Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 1 Jun 2022 10:20:56 +0800 Subject: [PATCH 02/78] update case --- tests/system-test/2-query/tail.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index 30d9870316..6039f3effa 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -1,4 +1,3 @@ -from math import floor from random import randint, random from numpy import equal import taos @@ -80,7 +79,7 @@ class TDTestCase: "select tail(c1 ,c2 ) from t1", "select tail(c1 ,NULL) from t1", "select tail(,) from t1;", - "select tail(floor(c1) ab from t1)", + "select tail(tail(c1) ab from t1)", "select tail(c1) as int from t1", "select tail('c1') from t1", "select tail(NULL) from t1", @@ -104,7 +103,7 @@ class TDTestCase: "select tail(c1 ,c2 ) from stb1 partition by tbname", "select tail(c1 ,NULL) from stb1 partition by tbname", "select tail(,) from stb1 partition by tbname;", - "select tail(floor(c1) ab from stb1 partition by tbname)", + "select tail(tail(c1) ab from stb1 partition by tbname)", "select tail(c1) as int from stb1 partition by tbname", "select tail('c1') from stb1 partition by tbname", "select tail(NULL) from stb1 partition by tbname", @@ -194,6 +193,7 @@ class TDTestCase: tail_result = tdSql.queryResult tdSql.query(equal_sql) + print(equal_sql) equal_result = tdSql.queryResult @@ -233,6 +233,9 @@ class TDTestCase: col_lists_rows = tdSql.queryResult col_lists = [] for col_name in col_lists_rows: + if col_name[0] =="ts": + continue + col_lists.append(col_name[0]) for col in col_lists: @@ -419,11 +422,11 @@ class TDTestCase: self.support_types() - tdLog.printNoPrefix("==========step4: floor basic query ============") + tdLog.printNoPrefix("==========step4: tail basic query ============") self.basic_tail_function() - tdLog.printNoPrefix("==========step5: floor boundary query ============") + tdLog.printNoPrefix("==========step5: tail boundary query ============") self.check_boundary_values() From 31f3bed34748a2a0172eb28b15fabfbf339b3787 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 11 Jun 2022 20:39:16 +0800 Subject: [PATCH 03/78] support index cache --- include/common/tmsg.h | 1 + source/common/src/tmsg.c | 2 + source/libs/catalog/inc/catalogInt.h | 60 +- source/libs/catalog/src/catalog.c | 38 +- source/libs/catalog/src/ctgAsync.c | 44 +- source/libs/catalog/src/ctgCache.c | 810 +++++++++++++--------- source/libs/catalog/src/ctgDbg.c | 8 +- source/libs/catalog/src/ctgRemote.c | 5 +- source/libs/catalog/src/ctgUtil.c | 88 ++- source/libs/catalog/test/catalogTests.cpp | 4 +- source/libs/qcom/src/querymsg.c | 10 +- source/libs/qworker/src/qworker.c | 1 - source/libs/scalar/src/filter.c | 5 + source/util/src/tarray.c | 3 + 14 files changed, 657 insertions(+), 422 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 48a677a686..7c37484ef6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2513,6 +2513,7 @@ typedef struct { } STableIndexInfo; typedef struct { + int32_t version; SArray* pIndex; } STableIndexRsp; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a7d4903696..3805a8b5b4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2437,6 +2437,7 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; if (num > 0) { @@ -2471,6 +2472,7 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->version) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; if (num > 0) { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 1d56b6d5d4..ebd4d56aaf 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -57,6 +57,8 @@ enum { CTG_OP_DROP_TB_META, CTG_OP_UPDATE_USER, CTG_OP_UPDATE_VG_EPSET, + CTG_OP_UPDATE_TB_INDEX, + CTG_OP_DROP_TB_INDEX, CTG_OP_MAX }; @@ -128,19 +130,27 @@ typedef struct SCtgUserCtx { SUserAuthInfo user; } SCtgUserCtx; -typedef struct SCtgTbMetaCache { - SRWLatch stbLock; - SRWLatch metaLock; // RC between cache destroy and all other operations - SHashObj *metaCache; //key:tbname, value:STableMeta - SHashObj *stbCache; //key:suid, value:STableMeta* -} SCtgTbMetaCache; +typedef STableIndexRsp STableIndex; + +typedef struct SCtgTbCache { + SRWLatch metaLock; + STableMeta *pMeta; + SRWLatch indexLock; + STableIndex *pIndex; +} SCtgTbCache; + +typedef struct SCtgVgCache { + SRWLatch vgLock; + SDBVgInfo *vgInfo; +} SCtgVgCache; typedef struct SCtgDBCache { - SRWLatch vgLock; + SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads uint64_t dbId; int8_t deleted; - SDBVgInfo *vgInfo; - SCtgTbMetaCache tbCache; + SCtgVgCache vgCache; + SHashObj *tbCache; // key:tbname, value:SCtgTbCache + SHashObj *stbCache; // key:suid, value:STableMeta* } SCtgDBCache; typedef struct SCtgRentSlot { @@ -245,8 +255,10 @@ typedef struct SCtgCacheStat { uint64_t userNum; uint64_t vgHitNum; uint64_t vgMissNum; - uint64_t tblHitNum; - uint64_t tblMissNum; + uint64_t tbMetaHitNum; + uint64_t tbMetaMissNum; + uint64_t tbIndexHitNum; + uint64_t tbIndexMissNum; uint64_t userHitNum; uint64_t userMissNum; } SCtgCacheStat; @@ -268,10 +280,10 @@ typedef struct SCtgUpdateVgMsg { SDBVgInfo* dbInfo; } SCtgUpdateVgMsg; -typedef struct SCtgUpdateTblMsg { - SCatalog* pCtg; - STableMetaOutput* output; -} SCtgUpdateTblMsg; +typedef struct SCtgUpdateTbMetaMsg { + SCatalog* pCtg; + STableMetaOutput* pMeta; +} SCtgUpdateTbMetaMsg; typedef struct SCtgDropDBMsg { SCatalog* pCtg; @@ -305,6 +317,19 @@ typedef struct SCtgUpdateUserMsg { SGetUserAuthRsp userAuth; } SCtgUpdateUserMsg; +typedef struct SCtgUpdateTbIndexMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; + STableIndex* pIndex; +} SCtgUpdateTbIndexMsg; + +typedef struct SCtgDropTbIndexMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; +} SCtgDropTbIndexMsg; + typedef struct SCtgUpdateEpsetMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; @@ -465,8 +490,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *action); int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache); -void ctgReleaseVgInfo(SCtgDBCache *dbCache); -int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); +void ctgRUnlockVgInfo(SCtgDBCache *dbCache); int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist); int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); @@ -479,6 +503,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet); +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); @@ -521,6 +546,7 @@ void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); char *ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask); +int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f1a81d1112..bac13bf476 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -97,8 +97,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* if (NULL != dbCache) { input.dbId = dbCache->dbId; - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } input.vgVersion = CTG_DEFAULT_INVALID_VERSION; @@ -383,6 +382,18 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SArray** pRes) { + CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); + if (*pRes) { + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_RET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pVgList) { STableMeta *tbMeta = NULL; int32_t code = 0; @@ -443,7 +454,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTabl _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -633,8 +644,7 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers *dbId = dbCache->dbId; *tableNum = dbCache->vgInfo->numOfTable; - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version); @@ -659,7 +669,7 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* SDBVgInfo *vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo)); if (dbCache) { - vgHash = dbCache->vgInfo->vgHash; + vgHash = dbCache->vgCache.vgInfo->vgHash; } else { vgHash = vgInfo->vgHash; } @@ -672,7 +682,7 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -940,7 +950,7 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -1143,7 +1153,17 @@ int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SNam CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(*pRes, &pInfo)); + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, pInfo, false)); + +_return: + + CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index a96e4cec30..69eddd7ba6 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -344,6 +344,11 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, SCtgJob *pJob, const SCatalogReq* p } } + for (int32_t i = 0; i < pJob->tbIndexNum; ++i) { + SName* name = taosArrayGet(pReq->pTableIndex, i); + ctgDropTbIndexEnqueue(pCtg, name, true); + } + return TSDB_CODE_SUCCESS; } @@ -524,7 +529,7 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableHash, &res); + taosArrayPush(pJob->jobRes.pTableIndex, &res); return TSDB_CODE_SUCCESS; } @@ -687,8 +692,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * ctx->vgId = vgInfo.vgId; CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } else { SBuildUseDBInput input = {0}; @@ -786,7 +790,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -866,9 +870,16 @@ _return: int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - TSWAP(pTask->res, pTask->msgCtx.out); + STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(pOut->pIndex, &pInfo)); + pTask->res = pInfo; + pTask->msgCtx.out = NULL; + + SCtgTbIndexCtx* ctx = pTask->taskCtx; + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, ctx->pName, pOut, false)); _return: @@ -1024,8 +1035,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1070,8 +1080,7 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1105,8 +1114,7 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1117,6 +1125,15 @@ int32_t ctgLaunchGetTbIndexTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbIndexCtx* pCtx = (SCtgTbIndexCtx*)pTask->taskCtx; + SArray* pRes = NULL; + + CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pCtx->pName, &pRes)); + if (pRes) { + pTask->res = pRes; + + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } CTG_ERR_RET(ctgGetTbIndexFromMnode(pCtg, pConn, pCtx->pName, NULL, pTask)); return TSDB_CODE_SUCCESS; @@ -1167,8 +1184,7 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index add44d6a16..2b56eb6b49 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -59,6 +59,11 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { CTG_OP_UPDATE_VG_EPSET, "update epset", ctgOpUpdateEpset + }, + { + CTG_OP_UPDATE_TB_INDEX, + "update tbIndex", + ctgOpUpdateTbIndex } }; @@ -66,11 +71,11 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { -int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { - CTG_LOCK(CTG_READ, &dbCache->vgLock); +int32_t ctgRLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { + CTG_LOCK(CTG_READ, &dbCache->vgCache.vgLock); if (dbCache->deleted) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); @@ -79,8 +84,8 @@ int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { } - if (NULL == dbCache->vgInfo) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + if (NULL == dbCache->vgCache.vgInfo) { + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); *inCache = false; ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); @@ -92,50 +97,47 @@ int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { return TSDB_CODE_SUCCESS; } -int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); +int32_t ctgWLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { + CTG_LOCK(CTG_WRITE, &dbCache->vgCache.vgLock); if (dbCache->deleted) { ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } return TSDB_CODE_SUCCESS; } +void ctgRUnlockVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); +} + +void ctgWUnlockVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); +} + void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { - taosHashRelease(pCtg->dbCache, dbCache); + CTG_UNLOCK(CTG_READ, &dbCache->dbLock); } -void ctgReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); -} - -void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); -} - - int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { char *p = strchr(dbFName, '.'); if (p && CTG_IS_SYS_DBNAME(p + 1)) { dbFName = p + 1; } - SCtgDBCache *dbCache = NULL; - if (acquire) { - dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); - } else { - dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); - } - + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); if (NULL == dbCache) { *pCache = NULL; ctgDebug("db not in cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } + if (acquire) { + CTG_LOCK(CTG_READ, &dbCache->dbLock); + } + if (dbCache->deleted) { if (acquire) { ctgReleaseDBCache(pCtg, dbCache); @@ -159,15 +161,35 @@ int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } +void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache) { + ctgRUnlockVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); +} + +void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* pCache) { + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } +} + +void ctgReleaseTbIndexToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* pCache) { + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->indexLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } +} int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { SCtgDBCache *dbCache = NULL; - - if (NULL == pCtg->dbCache) { - ctgDebug("empty db cache, dbFName:%s", dbFName); - goto _return; - } - ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); @@ -175,7 +197,7 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac } bool inCache = false; - ctgAcquireVgInfo(pCtg, dbCache, &inCache); + ctgRLockVgInfo(pCtg, dbCache, &inCache); if (!inCache) { ctgDebug("vgInfo of db %s not in cache", dbFName); goto _return; @@ -202,54 +224,107 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { - if (NULL == pCtg->dbCache) { - *exist = 0; - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; - } - +int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - *exist = 0; - return TSDB_CODE_SUCCESS; + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + SCtgTbCache* pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); + goto _return; } - size_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - - *exist = 0; - ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("tb %s meta not in cache, dbFName:%s", tbName, dbFName); + goto _return; } - *exist = 1; + *pDb = dbCache; + *pTb = pCache; - ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("tb %s meta got in cache, dbFName:%s", tbName, dbFName); - ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); + CTG_CACHE_STAT_INC(tbMetaHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbMetaMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + SCtgTbCache* pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); + goto _return; + } + + CTG_LOCK(CTG_READ, &pCache->indexLock); + if (NULL == pCache->pIndex) { + ctgDebug("tb %s index not in cache, dbFName:%s", tbName, dbFName); + goto _return; + } + + *pDb = dbCache; + *pTb = pCache; + + ctgDebug("tb %s index got in cache, dbFName:%s", tbName, dbFName); + + CTG_CACHE_STAT_INC(tbIndexHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbIndexToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbIndexMissNum, 1); return TSDB_CODE_SUCCESS; } +int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { + SCtgDBCache *dbCache = NULL; + ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + *exist = 1; + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - + SCtgTbCache *tbCache = NULL; *pTableMeta = NULL; - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", ctx->pName->tname); - return TSDB_CODE_SUCCESS; - } - char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { strcpy(dbFName, ctx->pName->dbname); @@ -257,78 +332,53 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** tNameGetFullDbName(ctx->pName, dbFName); } - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %d.%s not in cache", ctx->pName->acctId, ctx->pName->dbname); - return TSDB_CODE_SUCCESS; - } - - int32_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - taosHashGetDup_m(dbCache->tbCache.metaCache, ctx->pName->tname, strlen(ctx->pName->tname), (void **)pTableMeta, &sz); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == *pTableMeta) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + ctgAcquireTbMetaFromCache(pCtg, dbFName, ctx->pName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - STableMeta* tbMeta = *pTableMeta; + STableMeta* tbMeta = tbCache->pMeta; ctx->tbInfo.inCache = true; ctx->tbInfo.dbId = dbCache->dbId; ctx->tbInfo.suid = tbMeta->suid; ctx->tbInfo.tbType = tbMeta->tableType; if (tbMeta->tableType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, ctx->pName->tname); - - CTG_CACHE_STAT_INC(tblHitNum, 1); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", ctx->pName->tname, tbMeta->tableType, dbFName); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); + STableMeta **stbMeta = taosHashGet(dbCache->stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); goto _return; } if ((*stbMeta)->suid != tbMeta->suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%"PRIx64 , (*stbMeta)->suid, tbMeta->suid); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = CTG_META_SIZE(*stbMeta); *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgError("realloc size[%d] failed", metaSize); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgReleaseDBCache(pCtg, dbCache); - - CTG_CACHE_STAT_INC(tblHitNum, 1); - - ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + ctgDebug("Got tb %s meta from cache, dbFName:%s", ctx->pName->tname, dbFName); return TSDB_CODE_SUCCESS; _return: - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); taosMemoryFreeClear(*pTableMeta); - - CTG_CACHE_STAT_INC(tblMissNum, 1); CTG_RET(code); } @@ -338,62 +388,42 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t * *sver = -1; *tver = -1; - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); - return TSDB_CODE_SUCCESS; - } - SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, dbFName); - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %s not in cache", pTableName->tname); + ctgAcquireTbMetaFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); - if (tbMeta) { - *tbType = tbMeta->tableType; - *suid = tbMeta->suid; - if (*tbType != TSDB_CHILD_TABLE) { - *sver = tbMeta->sversion; - *tver = tbMeta->tversion; - } - } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; - } + STableMeta* tbMeta = tbCache->pMeta; + *tbType = tbMeta->tableType; + *suid = tbMeta->suid; if (*tbType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got sver %d tver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tver, *tbType, dbFName, pTableName->tname); + *sver = tbMeta->sversion; + *tver = tbMeta->tversion; + ctgDebug("Got tb %s ver from cache, dbFName:%s, tbType:%d, sver:%d, tver:%d, suid:%" PRIx64, + pTableName->tname, dbFName, *tbType, *sver, *tver, *suid); + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, *suid); - - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, suid, sizeof(*suid)); + STableMeta **stbMeta = taosHashGet(dbCache->stbCache, suid, sizeof(*suid)); if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid); return TSDB_CODE_SUCCESS; } if ((*stbMeta)->suid != *suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, *suid, - (*stbMeta)->suid); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%" PRIx64 , (*stbMeta)->suid, *suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -406,52 +436,55 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t * *sver = (*stbMeta)->sversion; *tver = (*stbMeta)->tversion; - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got sver %d tver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tver, *tbType, dbFName, pTableName->tname); + ctgDebug("Got tb %s sver %d tver %d from cache, type:%d, dbFName:%s", pTableName->tname, *sver, *tver, *tbType, dbFName); return TSDB_CODE_SUCCESS; } int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { - if (NULL == pCtg->dbCache) { - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); - return TSDB_CODE_SUCCESS; - } - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { + SCtgTbCache *tbCache = NULL; + ctgAcquireTbMetaFromCache(pCtg, dbFName, tableName, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, tableName, strlen(tableName)); + *tbType = tbCache->pMeta->tableType; + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - if (NULL == pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); - ctgReleaseDBCache(pCtg, dbCache); - - return TSDB_CODE_SUCCESS; - } - - *tbType = atomic_load_8(&pTableMeta->tableType); - - taosHashRelease(dbCache->tbCache.metaCache, pTableMeta); - - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); + ctgDebug("Got tb %s tbType %d from cache, dbFName:%s", tableName, *tbType, dbFName); return TSDB_CODE_SUCCESS; } +int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, const SName* pTableName, SArray** pRes) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + + *pRes = NULL; + + ctgAcquireTbIndexFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbIndexToCache(pCtg, dbCache, tbCache); + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_JRET(ctgCloneTableIndex(tbCache->pIndex->pIndex, pRes)); + +_return: + + ctgReleaseTbIndexToCache(pCtg, dbCache, tbCache); + + CTG_RET(code); +} + int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { if (NULL == pCtg->userCache) { ctgDebug("empty user auth cache, user:%s", user); @@ -718,9 +751,9 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy op->opId = CTG_OP_UPDATE_TB_META; op->syncOp = syncOp; - SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbMetaMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -730,7 +763,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy } msg->pCtg = pCtg; - msg->output = output; + msg->pMeta = output; op->data = msg; @@ -805,6 +838,68 @@ _return: CTG_RET(code); } +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_UPDATE_TB_INDEX; + op->syncOp = syncOp; + + SCtgUpdateTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbIndexMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbIndexMsg)); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->pIndex = pIndex; + tNameGetFullDbName(pName, msg->dbFName); + strcpy(msg->tbName, pName->tname); + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_DROP_TB_INDEX; + op->syncOp = syncOp; + + SCtgDropTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTbIndexMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbIndexMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + tNameGetFullDbName(pName, msg->dbFName); + strcpy(msg->tbName, pName->tname); + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; @@ -1006,14 +1101,14 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { SCtgDBCache newDBCache = {0}; newDBCache.dbId = dbId; - newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.metaCache) { + newDBCache.tbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache) { ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.stbCache) { + newDBCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.stbCache) { ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1050,22 +1145,22 @@ _return: } -void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - void *pIter = taosHashIterate(cache->stbCache, NULL); - while (pIter) { - uint64_t *suid = NULL; - suid = taosHashGetKey(pIter, NULL); - - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { - ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); - } - - pIter = taosHashIterate(cache->stbCache, pIter); - } +void ctgRemoveStbRent(SCatalog* pCtg, SCtgDBCache *dbCache) { + if (NULL == dbCache->stbCache) { + return; + } + + void *pIter = taosHashIterate(dbCache->stbCache, NULL); + while (pIter) { + uint64_t *suid = NULL; + suid = taosHashGetKey(pIter, NULL); + + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { + ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); + } + + pIter = taosHashIterate(dbCache->stbCache, pIter); } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); } @@ -1074,15 +1169,16 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + CTG_LOCK(CTG_WRITE, &dbCache->dbLock); + atomic_store_8(&dbCache->deleted, 1); - - ctgRemoveStbRent(pCtg, &dbCache->tbCache); - + ctgRemoveStbRent(pCtg, &dbCache); ctgFreeDbCache(dbCache); - CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); + + CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); @@ -1090,7 +1186,6 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d } CTG_CACHE_STAT_DEC(dbNum, 1); - ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); return TSDB_CODE_SUCCESS; @@ -1138,120 +1233,54 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } - -int32_t ctgWriteDBVgInfoToCache(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { - int32_t code = 0; - SDBVgInfo* dbInfo = *pDbInfo; - - if (NULL == dbInfo->vgHash) { - return TSDB_CODE_SUCCESS; - } - - if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", - dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; - - SCtgDBCache *dbCache = NULL; - CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); - if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - SDBVgInfo *vgInfo = NULL; - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); - - if (dbCache->vgInfo) { - if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) { - ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } - - if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) { - ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } - - ctgFreeVgInfo(dbCache->vgInfo); - } - - dbCache->vgInfo = dbInfo; - - *pDbInfo = NULL; - - ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); - - ctgWReleaseVgInfo(dbCache); - - dbCache = NULL; - - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - CTG_RET(code); -} - - int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { - SCtgTbMetaCache *tbCache = &dbCache->tbCache; - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { + taosMemoryFree(meta); ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } + bool isStb = meta->tableType == TSDB_SUPER_TABLE; + SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + STableMeta *orig = (pCache ? pCache->pMeta : NULL); int8_t origType = 0; uint64_t origSuid = 0; - bool isStb = meta->tableType == TSDB_SUPER_TABLE; - STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (orig) { origType = orig->tableType; if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosMemoryFree(meta); + ctgDebug("ignore table %s meta update", tbName); return TSDB_CODE_SUCCESS; } if (origType == TSDB_SUPER_TABLE) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + if (taosHashRemove(dbCache->stbCache, &orig->suid, sizeof(orig->suid))) { ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } else { CTG_CACHE_STAT_DEC(stblNum, 1); + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); - origSuid = orig->suid; } } - if (isStb) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - } - - if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { - if (isStb) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + if (NULL == pCache) { + SCtgTbCache cache = {0}; + cache.pMeta = meta; + if (taosHashPut(dbCache->tbCache, tbName, strlen(tbName), &cache, sizeof(SCtgTbCache)) != 0) { + taosMemoryFree(meta); + ctgError("taosHashPut new tbCache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + } else { + taosMemoryFree(pCache->pMeta); + pCache->pMeta = meta; } if (NULL == orig) { @@ -1262,23 +1291,15 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgdShowTableMeta(pCtg, tbName, meta); if (!isStb) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; } - STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); - if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); + if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), &pCache, POINTER_BYTES) != 0) { + ctgError("taosHashPut to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_INC(stblNum, 1); - - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); @@ -1290,6 +1311,43 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam return TSDB_CODE_SUCCESS; } +int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbName, STableIndex **index) { + if (NULL == dbCache->tbCache) { + taosMemoryFreeClear(*index); + ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + STableIndex* pIndex = *index; + SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + SCtgTbCache cache = {0}; + cache.pIndex = pIndex; + + if (taosHashPut(table->tbCache, tbName, strlen(tbName), &cache, sizeof(cache)) != 0) { + taosMemoryFreeClear(*index); + ctgError("taosHashPut new tbCache failed, tbName:%s", tbName); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + *index = NULL; + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + return TSDB_CODE_SUCCESS; + } + + if (pCache->pIndex) { + taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pCache->pIndex); + } + + pCache->pIndex = pIndex; + *index = NULL; + + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq) { STableMetaOutput* pOutput = NULL; int32_t code = 0; @@ -1305,12 +1363,66 @@ _return: CTG_RET(code); } - int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; + SDBVgInfo* dbInfo = msg->dbInfo; + char* dbFName = msg->dbFName; - CTG_ERR_JRET(ctgWriteDBVgInfoToCache(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); + if (NULL == dbInfo->vgHash) { + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", + dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + bool newAdded = false; + SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; + + SCtgDBCache *dbCache = NULL; + CTG_ERR_RET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, msg->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SCtgVgCache *vgCache = &dbCache->vgCache; + CTG_ERR_RET(ctgWLockVgInfo(msg->pCtg, dbCache)); + + if (vgCache->vgInfo) { + SDBVgInfo *vgInfo = vgCache->vgInfo; + + if (dbInfo->vgVersion < vgInfo->vgVersion) { + ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); + ctgWUnlockVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) { + ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); + ctgWUnlockVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + ctgFreeVgInfo(vgInfo); + } + + vgCache->vgInfo = dbInfo; + msg->dbInfo = NULL; + + ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + + ctgWUnlockVgInfo(dbCache); + + dbCache = NULL; + + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_RET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); _return: @@ -1356,14 +1468,14 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + CTG_ERR_RET(ctgWLockVgInfo(pCtg, dbCache)); - ctgFreeVgInfo(dbCache->vgInfo); - dbCache->vgInfo = NULL; + ctgFreeVgInfo(dbCache->vgCache.vgInfo); + dbCache->vgCache.vgInfo = NULL; ctgDebug("db vgInfo removed, dbFName:%s", msg->dbFName); - ctgWReleaseVgInfo(dbCache); + ctgWUnlockVgInfo(dbCache); _return: @@ -1375,42 +1487,47 @@ _return: int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgUpdateTblMsg *msg = operation->data; + SCtgUpdateTbMetaMsg *msg = operation->data; SCatalog* pCtg = msg->pCtg; - STableMetaOutput* output = msg->output; + STableMetaOutput* pMeta = msg->pMeta; SCtgDBCache *dbCache = NULL; - if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { - ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + if ((!CTG_IS_META_CTABLE(pMeta->metaType)) && NULL == pMeta->tbMeta) { + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", pMeta->dbFName, pMeta->tbName); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { - ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); + if (CTG_IS_META_BOTH(pMeta->metaType) && TSDB_SUPER_TABLE != pMeta->tbMeta->tableType) { + ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, pMeta->tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pMeta->dbFName, pMeta->dbId, &dbCache)); if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, pMeta->dbFName, pMeta->dbId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - int32_t metaSize = CTG_META_SIZE(output->tbMeta); - - CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); + if (CTG_IS_META_TABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { + int32_t metaSize = CTG_META_SIZE(pMeta->tbMeta); + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->tbName, pMeta->tbMeta, metaSize)); + pMeta->tbMeta = NULL; } - if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); + if (CTG_IS_META_CTABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { + SCTableMeta* ctbMeta = taosMemoryMalloc(sizeof(SCTableMeta)); + if (NULL == ctbMeta) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + memcpy(ctbMeta, &pMeta->ctbMeta, sizeof(SCTableMeta)); + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->ctbName, (STableMeta *)ctbMeta, sizeof(SCTableMeta))); } _return: - if (output) { - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); + if (pMeta) { + taosMemoryFreeClear(pMeta->tbMeta); + taosMemoryFreeClear(pMeta); } taosMemoryFreeClear(msg); @@ -1435,22 +1552,17 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { + if (taosHashRemove(dbCache->stbCache, &msg->suid, sizeof(msg->suid))) { ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(stblNum, 1); } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { + if (taosHashRemove(dbCache->tbCache, msg->stbName, strlen(msg->stbName))) { ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(tblNum, 1); } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); @@ -1477,21 +1589,18 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); + ctgDebug("dbId %" PRIx64 " not match with curId %"PRIx64", dbFName:%s, tbName:%s"msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + if (taosHashRemove(dbCache->tbCache, msg->tbName, strlen(msg->tbName))) { + ctgError("tb %s not exist in cache, dbFName:%s", msg->tbName, msg->dbFName); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } else { CTG_CACHE_STAT_DEC(tblNum, 1); } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + ctgDebug("table %s removed from cache, dbFName:%s", msg->tbName, msg->dbFName); _return: @@ -1553,7 +1662,6 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { _return: - taosHashCleanup(msg->userAuth.createdDbs); taosHashCleanup(msg->userAuth.readDbs); taosHashCleanup(msg->userAuth.writeDbs); @@ -1569,24 +1677,22 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; - CTG_ERR_RET(ctgAcquireDBCache(pCtg, msg->dbFName, &dbCache)); + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not exist, ignore epset update", msg->dbFName); goto _return; } - SDBVgInfo *vgInfo = NULL; - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); - - if (NULL == dbCache->vgInfo) { - ctgWReleaseVgInfo(dbCache); + CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); + + SDBVgInfo *vgInfo = dbCache->vgCache.vgInfo; + if (NULL == vgInfo) { ctgDebug("vgroup in db %s not cached, ignore epset update", msg->dbFName); goto _return; } - SVgroupInfo* pInfo = taosHashGet(dbCache->vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); + SVgroupInfo* pInfo = taosHashGet(vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); if (NULL == pInfo) { - ctgWReleaseVgInfo(dbCache); ctgDebug("no vgroup %d in db %s, ignore epset update", msg->vgId, msg->dbFName); goto _return; } @@ -1599,12 +1705,10 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { pInfo->epSet = msg->epSet; - ctgWReleaseVgInfo(dbCache); - _return: if (dbCache) { - ctgReleaseDBCache(msg->pCtg, dbCache); + ctgWUnlockVgInfo(dbCache); } taosMemoryFreeClear(msg); @@ -1612,6 +1716,32 @@ _return: CTG_RET(code); } +int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgUpdateTbIndexMsg *msg = operation->data; + SCatalog* pCtg = msg->pCtg; + STableIndex* pIndex = msg->pIndex; + SCtgDBCache *dbCache = NULL; + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, msg->dbFName, 0, &dbCache)); + if (NULL == dbCache) { + CTG_ERR_JRET(code); + } + + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->tbName, &pIndex)); + +_return: + + if (pIndex) { + taosArrayDestroyEx(pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pIndex); + } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void ctgUpdateThreadUnexpectedStopped(void) { if (CTG_IS_LOCKED(&gCtgMgmt.lock) > 0) CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index d70a176631..52f29e3acb 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -266,11 +266,11 @@ int32_t ctgdGetStatNum(char *option, void *res) { } int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; + return dbCache->table.tbCache ? (int32_t)taosHashGetSize(dbCache->table.tbCache) : 0; } int32_t ctgdGetStbNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0; + return dbCache->table.stbCache ? (int32_t)taosHashGetSize(dbCache->table.stbCache) : 0; } int32_t ctgdGetRentNum(SCtgRentMgmt *rent) { @@ -363,8 +363,8 @@ void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { dbFName = taosHashGetKey(pIter, &len); - int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; - int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; + int32_t metaNum = dbCache->table.tbCache ? taosHashGetSize(dbCache->table.tbCache) : 0; + int32_t stbNum = dbCache->table.stbCache ? taosHashGetSize(dbCache->table.stbCache) : 0; int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; int32_t hashMethod = -1; int32_t vgNum = 0; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index bfd5057c69..c2ebd9e210 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -448,10 +448,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n } if (pTask) { - void* pOut = taosMemoryCalloc(1, POINTER_BYTES); - if (NULL == pOut) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); - } + void* pOut = NULL; CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index a792c555ee..7ab9a6ab9c 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -110,25 +110,39 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { taosMemoryFreeClear(mgmt->slots); } - -void ctgFreeTbMetaCache(SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - int32_t stblNum = taosHashGetSize(cache->stbCache); - taosHashCleanup(cache->stbCache); - cache->stbCache = NULL; - CTG_CACHE_STAT_DEC(stblNum, stblNum); +void ctgFreeStbMetaCache(SCtgDBCache *dbCache) { + if (NULL == dbCache->stbCache) { + return; } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); - CTG_LOCK(CTG_WRITE, &cache->metaLock); - if (cache->metaCache) { - int32_t tblNum = taosHashGetSize(cache->metaCache); - taosHashCleanup(cache->metaCache); - cache->metaCache = NULL; - CTG_CACHE_STAT_DEC(tblNum, tblNum); + int32_t stblNum = taosHashGetSize(dbCache->stbCache); + taosHashCleanup(dbCache->stbCache); + dbCache->stbCache = NULL; + CTG_CACHE_STAT_DEC(stblNum, stblNum); +} + +void ctgFreeTbCacheImpl(SCtgTbCache *pCache) { + taosMemoryFreeClear(pCache->pMeta); + if (pCache->pIndex) { + taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pCache->pIndex); } - CTG_UNLOCK(CTG_WRITE, &cache->metaLock); +} + +void ctgFreeTbCache(SCtgDBCache *dbCache) { + if (NULL == dbCache->tbCache) { + return; + } + + int32_t tblNum = taosHashGetSize(dbCache->tbCache); + SCtgTbCache *pCache = taosHashIterate(dbCache->tbCache, NULL); + while (NULL != pCache) { + ctgFreeTbCacheImpl(pCache); + pCache = taosHashIterate(dbCache->tbCache, pCache); + } + taosHashCleanup(dbCache->tbCache); + dbCache->tbCache = NULL; + CTG_CACHE_STAT_DEC(tblNum, tblNum); } void ctgFreeVgInfo(SDBVgInfo *vgInfo) { @@ -144,16 +158,18 @@ void ctgFreeVgInfo(SDBVgInfo *vgInfo) { taosMemoryFreeClear(vgInfo); } +void ctgFreeVgInfoCache(SCtgDBCache *dbCache) { + ctgFreeVgInfo(dbCache->vgCache.vgInfo); +} + void ctgFreeDbCache(SCtgDBCache *dbCache) { if (NULL == dbCache) { return; } - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - ctgFreeVgInfo (dbCache->vgInfo); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - - ctgFreeTbMetaCache(&dbCache->tbCache); + ctgFreeVgInfoCache(dbCache); + ctgFreeStbMetaCache(dbCache); + ctgFreeTbCache(dbCache); } @@ -167,16 +183,13 @@ void ctgFreeHandle(SCatalog* pCtg) { void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; - atomic_store_8(&dbCache->deleted, 1); - ctgFreeDbCache(dbCache); pIter = taosHashIterate(pCtg->dbCache, pIter); } taosHashCleanup(pCtg->dbCache); - CTG_CACHE_STAT_DEC(dbNum, dbNum); } @@ -186,14 +199,12 @@ void ctgFreeHandle(SCatalog* pCtg) { void *pIter = taosHashIterate(pCtg->userCache, NULL); while (pIter) { SCtgUserAuth *userCache = pIter; - ctgFreeSCtgUserAuth(userCache); pIter = taosHashIterate(pCtg->userCache, pIter); } taosHashCleanup(pCtg->userCache); - CTG_CACHE_STAT_DEC(userNum, userNum); } @@ -252,9 +263,9 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { break; } case TDMT_MND_GET_TABLE_INDEX: { - SArray** pOut = (SArray**)pCtx->out; + STableIndex* pOut = (STableIndex*)pCtx->out; if (pOut) { - taosArrayDestroyEx(*pOut, tFreeSTableIndexInfo); + taosArrayDestroyEx(pOut->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(pCtx->out); } break; @@ -640,6 +651,27 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) return TSDB_CODE_SUCCESS; } +int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { + if (NULL == pIndex) { + *pRes = NULL; + return TSDB_CODE_SUCCESS; + } + + int32_t num = taosArrayGetSize(pIndex); + *pRes = taosArrayInit(num, sizeof(STableIndexInfo)); + if (NULL == *pRes) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + for (int32_t i = 0; i < num; ++i) { + STableIndexInfo *pInfo = taosArrayGet(pIndex, i); + taosArrayPush(*pRes, pInfo); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask) { if (msgType == TDMT_VND_TABLE_META) { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 04dbc26774..254525f78f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -895,9 +895,9 @@ void *ctgTestSetCtableMetaThread(void *param) { output = (STableMetaOutput *)taosMemoryMalloc(sizeof(STableMetaOutput)); ctgTestBuildCTableMetaOutput(output); - SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTbMetaMsg *msg = (SCtgUpdateTbMetaMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); msg->pCtg = pCtg; - msg->output = output; + msg->pMeta = output; operation.data = msg; code = ctgOpUpdateTbMeta(&operation); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 10d0f4a44a..edc468a6e2 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -484,13 +484,17 @@ int32_t queryProcessGetTbIndexRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_INVALID_INPUT; } - STableIndexRsp out = {0}; - if (tDeserializeSTableIndexRsp(msg, msgSize, &out) != 0) { + STableIndexRsp *out = taosMemoryCalloc(1, sizeof(STableIndexRsp)); + if (NULL == out) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (tDeserializeSTableIndexRsp(msg, msgSize, out) != 0) { qError("tDeserializeSTableIndexRsp failed, msgSize:%d", msgSize); return TSDB_CODE_INVALID_MSG; } - *(void **)output = out.pIndex; + *(void **)output = out; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ae0eda5449..451607e7d0 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -837,7 +837,6 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { SQWorker *mgmt = qwAcquire(refId); if (NULL == mgmt) { QW_DLOG("qwAcquire %" PRIx64 "failed", refId); - taosMemoryFree(param); return; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 6704bfdde3..a446aa7bdb 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1476,6 +1476,11 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; if (field->desc) { + if (QUERY_NODE_VALUE != nodeType(field->desc)) { + qDebug("VAL%d => [type:not value node][val:NIL]", i); //TODO + continue; + } + SValueNode *var = (SValueNode *)field->desc; SDataType *dType = &var->node.resType; if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index a34b332b9d..2357f760d1 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -458,6 +458,9 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void } SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy) { + if (NULL == pSrc) { + return NULL; + } ASSERT(pSrc->elemSize == sizeof(void*)); SArray* pArray = taosArrayInit(pSrc->size, sizeof(void*)); for (int32_t i = 0; i < pSrc->size; i++) { From aad288ab1751e73e52079427139165d941209477 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 12 Jun 2022 15:21:56 +0800 Subject: [PATCH 04/78] refactor(sync): add debug log --- include/libs/sync/sync.h | 13 ++- source/dnode/mnode/impl/src/mndMain.c | 12 +-- source/dnode/mnode/impl/src/mndSync.c | 7 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +- source/dnode/vnode/src/vnd/vnodeSync.c | 3 + source/libs/sync/inc/syncInt.h | 4 + source/libs/sync/src/syncAppendEntries.c | 2 +- source/libs/sync/src/syncCommit.c | 2 +- source/libs/sync/src/syncMain.c | 130 +++++++++++++++++------ 9 files changed, 124 insertions(+), 55 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 3a77cc1e19..2673bc382b 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -47,8 +47,9 @@ typedef enum { typedef enum { TAOS_SYNC_PROPOSE_SUCCESS = 0, TAOS_SYNC_PROPOSE_NOT_LEADER = 1, - TAOS_SYNC_PROPOSE_OTHER_ERROR = 2, - TAOS_SYNC_ONLY_ONE_REPLICA = 3, + TAOS_SYNC_ONLY_ONE_REPLICA = 2, + TAOS_SYNC_NOT_IN_NEW_CONFIG = 3, + TAOS_SYNC_OTHER_ERROR = 100, } ESyncProposeCode; typedef enum { @@ -199,15 +200,13 @@ bool syncIsRestoreFinish(int64_t rid); int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta); int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg); -int32_t syncReconfigRaw(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg); + +// build SRpcMsg, need to call syncPropose with SRpcMsg +int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg); int32_t syncLeaderTransfer(int64_t rid); int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader); -// to be moved to static -void syncStartNormal(int64_t rid); -void syncStartStandBy(int64_t rid); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index d8a61461dd..8b86845627 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -382,22 +382,22 @@ void mndStop(SMnode *pMnode) { int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SSyncMgmt *pMgmt = &pMnode->syncMgmt; - int32_t code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + int32_t code = TAOS_SYNC_OTHER_ERROR; if (!syncEnvIsStart()) { mError("failed to process sync msg:%p type:%s since syncEnv stop", pMsg, TMSG_INFO(pMsg->msgType)); - return TAOS_SYNC_PROPOSE_OTHER_ERROR; + return TAOS_SYNC_OTHER_ERROR; } SSyncNode *pSyncNode = syncNodeAcquire(pMgmt->sync); if (pSyncNode == NULL) { mError("failed to process sync msg:%p type:%s since syncNode is null", pMsg, TMSG_INFO(pMsg->msgType)); - return TAOS_SYNC_PROPOSE_OTHER_ERROR; + return TAOS_SYNC_OTHER_ERROR; } if (mndAcquireSyncRef(pMnode) != 0) { mError("failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); - return TAOS_SYNC_PROPOSE_OTHER_ERROR; + return TAOS_SYNC_OTHER_ERROR; } char logBuf[512] = {0}; @@ -457,7 +457,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { } else { mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + code = TAOS_SYNC_OTHER_ERROR; } } else { @@ -495,7 +495,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { syncAppendEntriesReplyDestroy(pSyncMsg); } else { mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + code = TAOS_SYNC_OTHER_ERROR; } } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index e0b4cc6a57..24b4f1926d 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -236,7 +236,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { tsem_wait(&pMgmt->syncSem); } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) { terrno = TSDB_CODE_APP_NOT_READY; - } else if (code == TAOS_SYNC_PROPOSE_OTHER_ERROR) { + } else if (code == TAOS_SYNC_OTHER_ERROR) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; } else { terrno = TSDB_CODE_APP_ERROR; @@ -254,13 +254,16 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { void mndSyncStart(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; syncSetMsgCb(pMgmt->sync, &pMnode->msgCb); + syncStart(pMgmt->sync); + mDebug("mnode sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); +/* if (pMgmt->standby) { syncStartStandBy(pMgmt->sync); } else { syncStart(pMgmt->sync); } - mDebug("mnode sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); +*/ } void mndSyncStop(SMnode *pMnode) {} diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 51ed739693..4d1a5060dd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -291,7 +291,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { } int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - int32_t ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; + int32_t ret = TAOS_SYNC_OTHER_ERROR; if (syncEnvIsStart()) { SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); @@ -372,13 +372,13 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } else { vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType); - ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; + ret = TAOS_SYNC_OTHER_ERROR; } syncNodeRelease(pSyncNode); } else { vError("==vnodeProcessSyncReq== error syncEnv stop"); - ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; + ret = TAOS_SYNC_OTHER_ERROR; } return ret; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index b4021ec73d..ec95451b31 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -290,11 +290,14 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { void vnodeSyncStart(SVnode *pVnode) { syncSetMsgCb(pVnode->sync, &pVnode->msgCb); + syncStart(pVnode->sync); + /* if (pVnode->config.standby) { syncStartStandBy(pVnode->sync); } else { syncStart(pVnode->sync); } + */ } void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index e7777af749..8a44dcd9bc 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -168,6 +168,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeStart(SSyncNode* pSyncNode); void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); +int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak); // option bool syncNodeSnapshotEnable(SSyncNode* pSyncNode); @@ -232,6 +233,9 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId); SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId); +void syncStartNormal(int64_t rid); +void syncStartStandBy(int64_t rid); + // for debug -------------- void syncNodePrint(SSyncNode* pObj); void syncNodePrint2(char* s, SSyncNode* pObj); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 29ee263756..08a4081ad3 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -995,7 +995,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs ths->commitIndex = snapshot.lastApplyIndex; sDebug("vgId:%d sync event commit by snapshot from index:%ld to index:%ld, %s", ths->vgId, commitBegin, - commitEnd, syncUtilState2String(ths->state)); + commitEnd, syncUtilState2String(ths->state)); } SyncIndex beginIndex = ths->commitIndex + 1; diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index eac9c1fbd8..3424fac5e7 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -57,7 +57,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { pSyncNode->commitIndex = snapshot.lastApplyIndex; sDebug("vgId:%d sync event commit by snapshot from index:%ld to index:%ld, %s", pSyncNode->vgId, - pSyncNode->commitIndex, snapshot.lastApplyIndex, syncUtilState2String(pSyncNode->state)); + pSyncNode->commitIndex, snapshot.lastApplyIndex, syncUtilState2String(pSyncNode->state)); } // update commit index diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 4d410644bb..fbac8c3d9d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -149,12 +149,12 @@ void syncStop(int64_t rid) { int32_t syncSetStandby(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return -1; + return TAOS_SYNC_OTHER_ERROR; } if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); - return -1; + return TAOS_SYNC_OTHER_ERROR; } // state change @@ -173,14 +173,68 @@ int32_t syncSetStandby(int64_t rid) { return 0; } -int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { - int32_t ret = 0; - char* newconfig = syncCfg2Str((SSyncCfg*)pSyncCfg); +int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_OTHER_ERROR; + } + ASSERT(rid == pSyncNode->rid); + int32_t ret = 0; + bool IamInNew = false; + for (int i = 0; i < pNewCfg->replicaNum; ++i) { + SRaftId newId; + newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); + newId.vgId = pSyncNode->vgId; + if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + IamInNew = true; + } + } + if (!IamInNew) { + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return TAOS_SYNC_NOT_IN_NEW_CONFIG; + } + + char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); + pRpcMsg->msgType = TDMT_SYNC_CONFIG_CHANGE; + pRpcMsg->info.noResp = 1; + pRpcMsg->contLen = strlen(newconfig) + 1; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + snprintf(pRpcMsg->pCont, pRpcMsg->contLen, "%s", newconfig); + taosMemoryFree(newconfig); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return ret; +} + +int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_OTHER_ERROR; + } + ASSERT(rid == pSyncNode->rid); + + bool IamInNew = false; + for (int i = 0; i < pNewCfg->replicaNum; ++i) { + SRaftId newId; + newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); + newId.vgId = pSyncNode->vgId; + if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + IamInNew = true; + } + } + if (!IamInNew) { + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return TAOS_SYNC_NOT_IN_NEW_CONFIG; + } + + char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); if (gRaftDetailLog) { sInfo("==syncReconfig== newconfig:%s", newconfig); } + int32_t ret = 0; + SRpcMsg rpcMsg = {0}; rpcMsg.msgType = TDMT_SYNC_CONFIG_CHANGE; rpcMsg.info.noResp = 1; @@ -188,27 +242,42 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", newconfig); taosMemoryFree(newconfig); - ret = syncPropose(rid, &rpcMsg, false); + ret = syncNodePropose(pSyncNode, &rpcMsg, false); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } int32_t syncLeaderTransfer(int64_t rid) { - int32_t ret = 0; + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_OTHER_ERROR; + } + ASSERT(rid == pSyncNode->rid); + if (pSyncNode->peersNum > 0) { + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return TAOS_SYNC_OTHER_ERROR; + } + + SNodeInfo newLeader = (pSyncNode->peersNodeInfo)[0]; + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + + int32_t ret = syncLeaderTransferTo(rid, newLeader); return ret; } int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return false; + return TAOS_SYNC_OTHER_ERROR; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); int32_t ret = 0; if (pSyncNode->replicaNum == 1) { - taosReleaseRef(tsNodeRefId, pSyncNode->rid); sError("only one replica, cannot drop leader"); + taosReleaseRef(tsNodeRefId, pSyncNode->rid); return TAOS_SYNC_ONLY_ONE_REPLICA; } @@ -220,26 +289,11 @@ int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader) { syncLeaderTransfer2RpcMsg(pMsg, &rpcMsg); syncLeaderTransferDestroy(pMsg); - ret = syncPropose(rid, &rpcMsg, false); - + ret = syncNodePropose(pSyncNode, &rpcMsg, false); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } -int32_t syncReconfigRaw(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg) { - int32_t ret = 0; - char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); - - pRpcMsg->msgType = TDMT_SYNC_CONFIG_CHANGE; - pRpcMsg->info.noResp = 1; - pRpcMsg->contLen = strlen(newconfig) + 1; - pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); - snprintf(pRpcMsg->pCont, pRpcMsg->contLen, "%s", newconfig); - taosMemoryFree(newconfig); - - return ret; -} - bool syncCanLeaderTransfer(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -272,8 +326,6 @@ bool syncCanLeaderTransfer(int64_t rid) { return matchOK; } -int32_t syncGiveUpLeader(int64_t rid) { return 0; } - int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { int32_t ret = syncPropose(rid, pMsg, isWeak); return ret; @@ -467,10 +519,19 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return TAOS_SYNC_PROPOSE_OTHER_ERROR; + return TAOS_SYNC_OTHER_ERROR; } assert(rid == pSyncNode->rid); sDebug("vgId:%d sync event propose msgType:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); + ret = syncNodePropose(pSyncNode, pMsg, isWeak); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return ret; +} + +int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) { + int32_t ret = TAOS_SYNC_PROPOSE_SUCCESS; + sDebug("vgId:%d sync event propose msgType:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { SRespStub stub; @@ -485,15 +546,14 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { if (pSyncNode->FpEqMsg != NULL && (*pSyncNode->FpEqMsg)(pSyncNode->msgcb, &rpcMsg) == 0) { ret = TAOS_SYNC_PROPOSE_SUCCESS; } else { - sTrace("syncPropose pSyncNode->FpEqMsg is NULL"); + sError("syncPropose pSyncNode->FpEqMsg is NULL"); } syncClientRequestDestroy(pSyncMsg); } else { - sTrace("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state)); + sError("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state)); ret = TAOS_SYNC_PROPOSE_NOT_LEADER; } - taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } @@ -1241,7 +1301,7 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { sDebug("vgId:%d sync event become follower, isStandBy:%d, %s", pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, - debugStr); + debugStr); // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -1276,7 +1336,7 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { sDebug("vgId:%d sync event become leader, isStandBy:%d, %s", pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, - debugStr); + debugStr); // state change pSyncNode->state = TAOS_SYNC_STATE_LEADER; @@ -1885,7 +1945,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, int32_t code = 0; ESyncState state = flag; sDebug("vgId:%d sync event commit by wal from index:%" PRId64 " to index:%" PRId64 ", %s", ths->vgId, beginIndex, - endIndex, syncUtilState2String(state)); + endIndex, syncUtilState2String(state)); // execute fsm if (ths->pFsm != NULL) { From 17f6275015745c9b348a74c7b390964be34df239 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 10:38:45 +0800 Subject: [PATCH 05/78] add sma cache --- include/common/tmsg.h | 16 +++---- include/libs/catalog/catalog.h | 7 ++-- source/client/src/clientHb.c | 19 +++++---- source/common/src/tmsg.c | 8 ++-- source/dnode/mnode/impl/inc/mndStb.h | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 2 +- source/dnode/mnode/impl/src/mndSma.c | 17 ++++++-- source/dnode/mnode/impl/src/mndStb.c | 51 +++++++++++++++++------ source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/catalog.c | 4 +- source/libs/catalog/src/ctgCache.c | 40 ++++++++++++++---- source/libs/catalog/src/ctgUtil.c | 8 ++-- source/libs/catalog/test/catalogTests.cpp | 8 ++-- 13 files changed, 124 insertions(+), 60 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7c37484ef6..ed35e46c98 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1136,12 +1136,13 @@ int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp void tFreeSTableMetaRsp(STableMetaRsp* pRsp); typedef struct { - SArray* pArray; // Array of STableMetaRsp -} STableMetaBatchRsp; + SArray* pArray; // Array of STableMetaRsp + STableIndexRsp* pIndexRsp; +} SSTbHbRsp; -int32_t tSerializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp); -int32_t tDeserializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp); -void tFreeSTableMetaBatchRsp(STableMetaBatchRsp* pRsp); +int32_t tSerializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); +int32_t tDeserializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); +void tFreeSSTbHbRsp(SSTbHbRsp* pRsp); typedef struct { int32_t numOfTables; @@ -2513,8 +2514,9 @@ typedef struct { } STableIndexInfo; typedef struct { - int32_t version; - SArray* pIndex; + uint64_t suid; + int32_t version; + SArray* pIndex; } STableIndexRsp; int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 89fdeac7b9..80b677dbd2 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -98,14 +98,15 @@ typedef struct SCatalogCfg { uint32_t stbRentSec; } SCatalogCfg; -typedef struct SSTableMetaVersion { +typedef struct SSTableVersion { char dbFName[TSDB_DB_FNAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; uint64_t dbId; uint64_t suid; int16_t sversion; int16_t tversion; -} SSTableMetaVersion; + int32_t smaVer; +} SSTableVersion; typedef struct SDbVgVersion { char dbFName[TSDB_DB_FNAME_LEN]; @@ -269,7 +270,7 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t int32_t catalogGetQnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray* pQnodeList); -int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); +int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion **stables, uint32_t *num); int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 9df0e0c1c8..1f6b00466c 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -99,15 +99,15 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; - STableMetaBatchRsp batchMetaRsp = {0}; - if (tDeserializeSTableMetaBatchRsp(value, valueLen, &batchMetaRsp) != 0) { + SSTbHbRsp hbRsp = {0}; + if (tDeserializeSSTbHbRsp(value, valueLen, &hbRsp) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } - int32_t numOfBatchs = taosArrayGetSize(batchMetaRsp.pArray); + int32_t numOfBatchs = taosArrayGetSize(hbRsp.pArray); for (int32_t i = 0; i < numOfBatchs; ++i) { - STableMetaRsp *rsp = taosArrayGet(batchMetaRsp.pArray, i); + STableMetaRsp *rsp = taosArrayGet(hbRsp.pArray, i); if (rsp->numOfColumns < 0) { tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); @@ -116,7 +116,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { tscError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -124,7 +124,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } } - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); return TSDB_CODE_SUCCESS; } @@ -455,7 +455,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl } int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { - SSTableMetaVersion *stbs = NULL; + SSTableVersion *stbs = NULL; uint32_t stbNum = 0; int32_t code = 0; @@ -469,15 +469,16 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC } for (int32_t i = 0; i < stbNum; ++i) { - SSTableMetaVersion *stb = &stbs[i]; + SSTableVersion *stb = &stbs[i]; stb->suid = htobe64(stb->suid); stb->sversion = htons(stb->sversion); stb->tversion = htons(stb->tversion); + stb->smaVer = htonl(stb->smaVer); } SKv kv = { .key = HEARTBEAT_KEY_STBINFO, - .valueLen = sizeof(SSTableMetaVersion) * stbNum, + .valueLen = sizeof(SSTableVersion) * stbNum, .value = stbs, }; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3805a8b5b4..138aaef3da 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2437,6 +2437,7 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeU64(&encoder, pRsp->suid) < 0) return -1; if (tEncodeI32(&encoder, pRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; @@ -2472,6 +2473,7 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeU64(&decoder, &pRsp->suid) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->version) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; @@ -2632,7 +2634,7 @@ int32_t tSerializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp) return tlen; } -int32_t tSerializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) { +int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2663,7 +2665,7 @@ int32_t tDeserializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp return 0; } -int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) { +int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -2691,7 +2693,7 @@ int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatc void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { taosMemoryFreeClear(pRsp->pSchemas); } -void tFreeSTableMetaBatchRsp(STableMetaBatchRsp *pRsp) { +void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); for (int32_t i = 0; i < numOfBatch; ++i) { STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 28d3215b98..d3de4e3b3f 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -27,7 +27,7 @@ void mndCleanupStb(SMnode *pMnode); SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName); void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); SSdbRaw *mndStbActionEncode(SStbObj *pStb); -int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t numOfStbs, void **ppRsp, +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbs, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen); int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index bacdf2f366..4cfcf389aa 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -433,7 +433,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb case HEARTBEAT_KEY_STBINFO: { void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); + mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv1 = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv1); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index cde36eac58..edb8a29ac0 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -872,18 +872,29 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp return code; } -static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIndexRsp *rsp, bool *exist) { +static int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist) { int32_t code = 0; SSmaObj *pSma = NULL; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; STableIndexInfo info; + SStbObj* pStb = mndAcquireStb(pMnode, tbFName); + if (NULL == pStb) { + *exist = false; + return TSDB_CODE_SUCCESS; + } + + rsp->suid = pStb->uid; + rsp->version = pStb->smaVer; + mndReleaseStb(pMnode, pStb); + + while (1) { pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); if (pIter == NULL) break; - if (pSma->stb[0] != indexReq->tbFName[0] || strcmp(pSma->stb, indexReq->tbFName)) { + if (pSma->stb[0] != tbFName[0] || strcmp(pSma->stb, tbFName)) { continue; } @@ -995,7 +1006,7 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { goto _OVER; } - code = mndGetTableSma(pMnode, &indexReq, &rsp, &exist); + code = mndGetTableSma(pMnode, indexReq.tbFName, &rsp, &exist); if (code) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9981dc8530..4b5b9e8a7b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1271,7 +1271,7 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa return 0; } -static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { +static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp, int32_t *smaVer) { char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); @@ -1288,6 +1288,10 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return -1; } + if (smaVer) { + *smaVer = pStb->smaVer; + } + int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp); mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); @@ -1667,51 +1671,72 @@ _OVER: return code; } -int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen) { - STableMetaBatchRsp batchMetaRsp = {0}; - batchMetaRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); - if (batchMetaRsp.pArray == NULL) { + SSTbHbRsp hbRsp = {0}; + hbRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); + if (hbRsp.pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + hbRsp.pIndexRsp = taosMemoryCalloc(1, sizeof(STableIndexRsp)); + if (NULL == hbRsp.pIndexRsp) { + taosArrayDestroy(hbRsp.pArray); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + for (int32_t i = 0; i < numOfStbs; ++i) { - SSTableMetaVersion *pStbVersion = &pStbVersions[i]; + SSTableVersion *pStbVersion = &pStbVersions[i]; pStbVersion->suid = be64toh(pStbVersion->suid); pStbVersion->sversion = ntohs(pStbVersion->sversion); pStbVersion->tversion = ntohs(pStbVersion->tversion); + pStbVersion->smaVer = ntohl(pStbVersion->smaVer); STableMetaRsp metaRsp = {0}; + int32_t smaVer = 0; mDebug("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName); - if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp) != 0) { + if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) { metaRsp.numOfColumns = -1; metaRsp.suid = pStbVersion->suid; + taosArrayPush(hbRsp.pArray, &metaRsp); + continue; } if (pStbVersion->sversion != metaRsp.sversion || pStbVersion->tversion != metaRsp.tversion) { - taosArrayPush(batchMetaRsp.pArray, &metaRsp); + taosArrayPush(hbRsp.pArray, &metaRsp); } else { tFreeSTableMetaRsp(&metaRsp); } + + if (pStbVersion->smaVer != smaVer) { + bool exist = false; + char tbFName[TSDB_TABLE_FNAME_LEN]; + sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); + int32_t code = mndGetTableSma(pMnode, tbFName, hbRsp.pIndexRsp, &exist); + if (code || !exist) { + taosMemoryFreeClear(hbRsp.pIndexRsp); + } + } } - int32_t rspLen = tSerializeSTableMetaBatchRsp(NULL, 0, &batchMetaRsp); + int32_t rspLen = tSerializeSSTbHbRsp(NULL, 0, &hbRsp); if (rspLen < 0) { - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); terrno = TSDB_CODE_INVALID_MSG; return -1; } void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSTableMetaBatchRsp(pRsp, rspLen, &batchMetaRsp); - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tSerializeSSTbHbRsp(pRsp, rspLen, &hbRsp); + tFreeSSTbHbRsp(&hbRsp); *ppRsp = pRsp; *pRspLen = rspLen; return 0; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index ebd4d56aaf..82bb151d86 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -156,7 +156,7 @@ typedef struct SCtgDBCache { typedef struct SCtgRentSlot { SRWLatch lock; bool needSort; - SArray *meta; // element is SDbVgVersion or SSTableMetaVersion + SArray *meta; // element is SDbVgVersion or SSTableVersion } SCtgRentSlot; typedef struct SCtgRentMgmt { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index bac13bf476..d01fa00878 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1074,14 +1074,14 @@ _return: CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, uint32_t *num) { +int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableVersion **stables, uint32_t *num) { CTG_API_ENTER(); if (NULL == pCtg || NULL == stables || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableVersion))); } int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 2b56eb6b49..49714ac2de 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -970,7 +970,7 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); if (NULL == orig) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + qDebug("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -983,7 +983,7 @@ _return: CTG_UNLOCK(CTG_WRITE, &slot->lock); if (code) { - qWarn("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); + qDebug("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); } @@ -1233,6 +1233,27 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } +int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char* dbFName, char* tbName, uint64_t dbId, uint64_t suid, SCtgTbCache* pCache) { + SSTableVersion metaRent = {.dbId = dbId, .suid = suid}; + if (pCache->pMeta) { + metaRent.sversion = pCache->pMeta->sversion; + metaRent.tversion = pCache->pMeta->tversion; + } + + if (pCache->pIndex) { + metaRent.smaVer = pCache->pIndex->version; + } + + strcpy(metaRent.dbFName, dbFName); + strcpy(metaRent.stbName, tbName); + + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableVersion), ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); + + ctgDebug("db %s,%" PRIx64 " stb %s,%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", + dbFName, dbId, tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer); +} + + int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { taosMemoryFree(meta); @@ -1263,7 +1284,6 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } - ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); origSuid = orig->suid; } } @@ -1303,15 +1323,12 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; - strcpy(metaRent.dbFName, dbFName); - strcpy(metaRent.stbName, tbName); - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); return TSDB_CODE_SUCCESS; } -int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbName, STableIndex **index) { +int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFName, char *tbName, STableIndex **index) { if (NULL == dbCache->tbCache) { taosMemoryFreeClear(*index); ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); @@ -1332,6 +1349,9 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbNam *index = NULL; ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + return TSDB_CODE_SUCCESS; } @@ -1344,6 +1364,8 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbNam *index = NULL; ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); return TSDB_CODE_SUCCESS; } @@ -1728,7 +1750,7 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { CTG_ERR_JRET(code); } - CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->tbName, &pIndex)); + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->dbFName, msg->tbName, &pIndex)); _return: diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 7ab9a6ab9c..f0ed60a689 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -546,9 +546,9 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName } int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { - if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { + if (*(uint64_t *)key1 < ((SSTableVersion*)key2)->suid) { return -1; - } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { + } else if (*(uint64_t *)key1 > ((SSTableVersion*)key2)->suid) { return 1; } else { return 0; @@ -566,9 +566,9 @@ int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) { } int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { + if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) { return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + } else if (((SSTableVersion*)key1)->suid > ((SSTableVersion*)key2)->suid) { return 1; } else { return 0; diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 254525f78f..aab341544f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -989,7 +989,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -1098,7 +1098,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -1220,7 +1220,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -2299,7 +2299,7 @@ TEST(rentTest, allRent) { SArray *vgList = NULL; ctgTestStop = false; SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stable = NULL; + SSTableVersion *stable = NULL; uint32_t num = 0; ctgTestInitLogFile(); From 34c906a1e11ef96187dcaf609732ccbbf10908e8 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 13 Jun 2022 10:39:32 +0800 Subject: [PATCH 06/78] refactor(sync): add debug log --- source/libs/sync/src/syncMain.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fbac8c3d9d..fa61bf1b08 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -219,15 +219,22 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { SRaftId newId; newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); newId.vgId = pSyncNode->vgId; + + sTrace("new[%d]: %lu, %d, my: %lu, %d", i, newId.addr, newId.vgId, pSyncNode->myRaftId.addr, pSyncNode->myRaftId.vgId); + if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { IamInNew = true; } } + + if (!IamInNew) { + sError("sync reconfig error, not in new config"); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return TAOS_SYNC_NOT_IN_NEW_CONFIG; } + char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); if (gRaftDetailLog) { sInfo("==syncReconfig== newconfig:%s", newconfig); From 803363571704761d603b96ce8098192e887eb502 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 11:29:53 +0800 Subject: [PATCH 07/78] other: add debug logs --- source/common/src/trow.c | 17 +++ source/dnode/vnode/src/sma/smaTimeRange.c | 178 ++++++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +- 3 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 source/dnode/vnode/src/sma/smaTimeRange.c diff --git a/source/common/src/trow.c b/source/common/src/trow.c index c8a28d7f28..fb4413478d 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -523,8 +523,24 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SCellVal sVal = {0}; if (pRowCol->colId == pDataCol->colId) { if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) { + ASSERT(0); return terrno; } + + if (pRowCol->colId == 2) { + ASSERT(sVal.valType == 0); + int32_t val = *(int32_t *)sVal.val; + ASSERT(val == 7); + } else if (pRowCol->colId == 3) { + ASSERT(sVal.valType == 0); + int64_t val = *(int64_t *)sVal.val; + ASSERT(val == 77777); + } else if (pRowCol->colId == 4) { + ASSERT(sVal.valType == 0); + int16_t val = *(int16_t *)sVal.val; + ASSERT(val == 777); + } + tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; @@ -535,6 +551,7 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; + ASSERT(0); } } #if 0 diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c new file mode 100644 index 0000000000..bca5b1543e --- /dev/null +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" +#include "tsdb.h" + +#define SMA_STORAGE_MINUTES_MAX 86400 +#define SMA_STORAGE_MINUTES_DAY 1440 +#define SMA_STORAGE_SPLIT_FACTOR 14400 // least records in tsma file + +/** + * @brief Judge the tsma file split days + * + * @param pCfg + * @param pCont + * @param contLen + * @param days unit is minute + * @return int32_t + */ +int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { + SDecoder coder = {0}; + tDecoderInit(&coder, pCont, contLen); + + STSma tsma = {0}; + if (tDecodeSVCreateTSmaReq(&coder, &tsma) < 0) { + terrno = TSDB_CODE_MSG_DECODE_ERROR; + goto _err; + } + STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; + int64_t sInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND); + if (sInterval <= 0) { + *days = pTsdbCfg->days; + return 0; + } + int64_t records = pTsdbCfg->days * 60 / sInterval; + if (records >= SMA_STORAGE_SPLIT_FACTOR) { + *days = pTsdbCfg->days; + } else { + int64_t mInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE); + int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2; + + if (daysPerFile > SMA_STORAGE_MINUTES_MAX) { + *days = SMA_STORAGE_MINUTES_MAX; + } else { + *days = (int32_t)daysPerFile; + } + + if (*days < pTsdbCfg->days) { + *days = pTsdbCfg->days; + } + } + tDecoderClear(&coder); + return 0; +_err: + tDecoderClear(&coder); + return -1; +} + +/** + * @brief create tsma meta and result stable + * + * @param pSma + * @param version + * @param pMsg + * @return int32_t + */ +int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { + SSmaCfg *pCfg = (SSmaCfg *)pMsg; + + if (TD_VID(pSma->pVnode) == pCfg->dstVgId) { + // create tsma meta in dstVgId + if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { + return -1; + } + + // create stable to save tsma result in dstVgId + SVCreateStbReq pReq = {0}; + pReq.name = pCfg->dstTbName; + pReq.suid = pCfg->dstTbUid; + pReq.schemaRow = pCfg->schemaRow; + pReq.schemaTag = pCfg->schemaTag; + + if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) { + return -1; + } + } + + return 0; +} + +/** + * @brief Insert/Update Time-range-wise SMA data. + * + * @param pSma + * @param msg + * @return int32_t + */ +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { + const SArray *pDataBlocks = (const SArray *)msg; + // TODO: destroy SSDataBlocks(msg) + if (!pDataBlocks) { + terrno = TSDB_CODE_TSMA_INVALID_PTR; + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma)); + return terrno; + } + + if (taosArrayGetSize(pDataBlocks) <= 0) { + terrno = TSDB_CODE_TSMA_INVALID_PARA; + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is empty", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != 0) { + terrno = TSDB_CODE_TSMA_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = NULL; + SSmaStatItem *pItem = NULL; + + if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) { + terrno = TSDB_CODE_TSMA_INVALID_STAT; + return TSDB_CODE_FAILED; + } + + tdRefSmaStat(pSma, pStat); + pItem = &pStat->tsmaStatItem; + + ASSERT(pItem); + + if (!pItem->pTSma) { + // cache smaMeta + STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); + if (!pTSma) { + terrno = TSDB_CODE_TSMA_NO_INDEX_IN_META; + smaWarn("vgId:%d, tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + pItem->pTSma = pTSma; + } + + STSma *pTSma = pItem->pTSma; + + ASSERT(pTSma->indexUid == indexUid); + + SMetaReader mr = {0}; + + const char *dbName = "testDb"; + if (metaGetTableEntryByName(&mr, dbName) != 0) { + smaDebug("vgId:%d, tsma no table testTb exists for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); + SVCreateStbReq pReq = {0}; + pReq.name = dbName; + pReq.suid = pTSma->dstTbUid; + pReq.schemaRow = pCfg->schemaRow; + pReq.schemaTag = pCfg->schemaTag; + } + + SSubmitReq *pSubmitReq = NULL; + buildSubmitReqFromDataBlock(&pSubmitReq, (const SArray *)msg, NULL, pItem->pTSma->dstVgId, + pItem->pTSma->dstTbUid); + + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ab2efa4791..9bb0834547 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -15,6 +15,8 @@ #include "vnd.h" +int32_t gForceCommit = 1; + static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -182,7 +184,8 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } // commit if need - if (vnodeShouldCommit(pVnode)) { + if ((gForceCommit == 1) || vnodeShouldCommit(pVnode)) { + gForceCommit = 0; vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change vnodeCommit(pVnode); @@ -719,6 +722,7 @@ static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const } static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + gForceCommit = 1; SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; SSubmitRsp submitRsp = {0}; SSubmitMsgIter msgIter = {0}; From b1093b4a791709b28a1c64423f59f4c19f3ed023 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 13 Jun 2022 12:13:35 +0800 Subject: [PATCH 08/78] fix(sync): use fqdn instead of raftId --- source/libs/sync/src/syncMain.c | 25 +++++++++++------- source/libs/sync/test/CMakeLists.txt | 14 ++++++++++ source/libs/sync/test/syncRaftIdCheck.cpp | 32 +++++++++++++++++++++++ 3 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 source/libs/sync/test/syncRaftIdCheck.cpp diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fa61bf1b08..82fde80be2 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -35,7 +35,7 @@ #include "syncVoteMgr.h" #include "tref.h" -bool gRaftDetailLog = false; +bool gRaftDetailLog = true; static int32_t tsNodeRefId = -1; @@ -216,25 +216,30 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { bool IamInNew = false; for (int i = 0; i < pNewCfg->replicaNum; ++i) { - SRaftId newId; - newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); - newId.vgId = pSyncNode->vgId; - - sTrace("new[%d]: %lu, %d, my: %lu, %d", i, newId.addr, newId.vgId, pSyncNode->myRaftId.addr, pSyncNode->myRaftId.vgId); - - if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + if (strcmp((pNewCfg->nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 && + (pNewCfg->nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) { IamInNew = true; } + + /* + // some problem in inet_addr + + SRaftId newId = EMPTY_RAFT_ID; + newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); + newId.vgId = pSyncNode->vgId; + + if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + IamInNew = true; + } + */ } - if (!IamInNew) { sError("sync reconfig error, not in new config"); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return TAOS_SYNC_NOT_IN_NEW_CONFIG; } - char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); if (gRaftDetailLog) { sInfo("==syncReconfig== newconfig:%s", newconfig); diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index d39035ba53..725343e373 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -1,4 +1,5 @@ add_executable(syncTest "") +add_executable(syncRaftIdCheck "") add_executable(syncEnvTest "") add_executable(syncPingTimerTest "") add_executable(syncIOTickQTest "") @@ -54,6 +55,10 @@ target_sources(syncTest PRIVATE "syncTest.cpp" ) +target_sources(syncRaftIdCheck + PRIVATE + "syncRaftIdCheck.cpp" +) target_sources(syncEnvTest PRIVATE "syncEnvTest.cpp" @@ -257,6 +262,11 @@ target_include_directories(syncTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncRaftIdCheck + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncEnvTest PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -508,6 +518,10 @@ target_link_libraries(syncTest sync gtest_main ) +target_link_libraries(syncRaftIdCheck + sync + gtest_main +) target_link_libraries(syncEnvTest sync gtest_main diff --git a/source/libs/sync/test/syncRaftIdCheck.cpp b/source/libs/sync/test/syncRaftIdCheck.cpp new file mode 100644 index 0000000000..f23c0db82b --- /dev/null +++ b/source/libs/sync/test/syncRaftIdCheck.cpp @@ -0,0 +1,32 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncUtil.h" + +void usage(char* exe) { + printf("Usage: %s host port \n", exe); + printf("Usage: %s u64 \n", exe); +} + +int main(int argc, char** argv) { + if (argc == 2) { + uint64_t u64 = atoll(argv[1]); + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + printf("%s:%d \n", host, port); + + } else if (argc == 3) { + uint64_t u64; + char* host = argv[1]; + uint16_t port = atoi(argv[2]); + syncUtilAddr2U64(host, port); + printf("%lu \n", u64); + } else { + usage(argv[0]); + exit(-1); + } + + return 0; +} From c0964e03bee2cf02445dd797a630544cbf9f2ed1 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 12:51:21 +0800 Subject: [PATCH 09/78] other: add debug logs --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 90093f2510..40a23447b7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1348,6 +1348,24 @@ static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLi isLast = true; } + SDataCols *pCols = pCommith->pDataCols; + SDataCol pCol_1 = pCols->cols[1]; + SDataCol pCol_2 = pCols->cols[2]; + SDataCol pCol_3 = pCols->cols[3]; + SCellVal sVal = {0}; + tdGetColDataOfRow(&sVal, &pCol_1, 0, pCols->bitmapMode); + ASSERT(sVal.valType == 0); + int32_t val1 = *(int32_t *)sVal.val; + ASSERT(val1 == 7); + tdGetColDataOfRow(&sVal, &pCol_2, 0, pCols->bitmapMode); + ASSERT(sVal.valType == 0); + int64_t val2 = *(int64_t *)sVal.val; + ASSERT(val2 == 77777); + tdGetColDataOfRow(&sVal, &pCol_3, 0, pCols->bitmapMode); + ASSERT(sVal.valType == 0); + int16_t val3 = *(int16_t *)sVal.val; + ASSERT(val3 == 777); + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { From eff9f9be4db8c0b5081c0d6b14a87e0c30128a35 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 14:06:11 +0800 Subject: [PATCH 10/78] add sma update --- include/common/tmsg.h | 6 +- source/client/src/clientHb.c | 16 ++++- source/common/src/tmsg.c | 102 +++++++++++++++++++++++---- source/dnode/mnode/impl/src/mndSma.c | 2 + source/dnode/mnode/impl/src/mndStb.c | 24 ++++--- source/libs/catalog/inc/catalogInt.h | 4 +- source/libs/catalog/src/catalog.c | 54 ++++++++++++-- source/libs/catalog/src/ctgAsync.c | 3 +- source/libs/catalog/src/ctgCache.c | 16 ++--- source/libs/catalog/src/ctgRemote.c | 8 ++- source/libs/qcom/src/querymsg.c | 8 +-- 11 files changed, 189 insertions(+), 54 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ed35e46c98..8a2b2a31f2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1136,8 +1136,8 @@ int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp void tFreeSTableMetaRsp(STableMetaRsp* pRsp); typedef struct { - SArray* pArray; // Array of STableMetaRsp - STableIndexRsp* pIndexRsp; + SArray* pMetaRsp; // Array of STableMetaRsp + SArray* pIndexRsp; // Array of STableIndexRsp; } SSTbHbRsp; int32_t tSerializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); @@ -2514,6 +2514,8 @@ typedef struct { } STableIndexInfo; typedef struct { + char tbName[TSDB_TABLE_NAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; uint64_t suid; int32_t version; SArray* pIndex; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 1f6b00466c..e111883381 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -105,9 +105,9 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo return -1; } - int32_t numOfBatchs = taosArrayGetSize(hbRsp.pArray); - for (int32_t i = 0; i < numOfBatchs; ++i) { - STableMetaRsp *rsp = taosArrayGet(hbRsp.pArray, i); + int32_t numOfMeta = taosArrayGetSize(hbRsp.pMetaRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *rsp = taosArrayGet(hbRsp.pMetaRsp, i); if (rsp->numOfColumns < 0) { tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); @@ -124,6 +124,16 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } } + int32_t numOfIndex = taosArrayGetSize(hbRsp.pIndexRsp); + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp *rsp = taosArrayGet(hbRsp.pIndexRsp, i); + + catalogUpdateTableIndex(pCatalog, rsp); + } + + taosArrayDestroy(hbRsp.pIndexRsp); + hbRsp.pIndexRsp = NULL; + tFreeSSTbHbRsp(&hbRsp); return TSDB_CODE_SUCCESS; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 138aaef3da..5d0cc2b28d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2437,6 +2437,8 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->tbName) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->dbFName) < 0) return -1; if (tEncodeU64(&encoder, pRsp->suid) < 0) return -1; if (tEncodeI32(&encoder, pRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->pIndex); @@ -2473,6 +2475,8 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->tbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->dbFName) < 0) return -1; if (tDecodeU64(&decoder, &pRsp->suid) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->version) < 0) return -1; int32_t num = 0; @@ -2640,12 +2644,30 @@ int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; - for (int32_t i = 0; i < numOfBatch; ++i) { - STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); + int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + if (tEncodeI32(&encoder, numOfMeta) < 0) return -1; + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pMetaRsp, i); if (tEncodeSTableMetaRsp(&encoder, pMetaRsp) < 0) return -1; } + + int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + STableIndexRsp *pIndexRsp = taosArrayGet(pRsp->pIndexRsp, i); + if (tEncodeCStr(&encoder, pIndexRsp->tbName) < 0) return -1; + if (tEncodeCStr(&encoder, pIndexRsp->dbFName) < 0) return -1; + if (tEncodeU64(&encoder, pIndexRsp->suid) < 0) return -1; + if (tEncodeI32(&encoder, pIndexRsp->version) < 0) return -1; + int32_t num = taosArrayGetSize(pIndexRsp->pIndex); + if (tEncodeI32(&encoder, num) < 0) return -1; + if (num > 0) { + for (int32_t i = 0; i < num; ++i) { + STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, i); + if (tSerializeSTableIndexInfo(&encoder, pInfo) < 0) return -1; + } + } + } + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2671,20 +2693,53 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + if (tDecodeI32(&decoder, &numOfMeta) < 0) return -1; - pRsp->pArray = taosArrayInit(numOfBatch, sizeof(STableMetaRsp)); - if (pRsp->pArray == NULL) { + pRsp->pMetaRsp = taosArrayInit(numOfMeta, sizeof(STableMetaRsp)); + if (pRsp->pMetaRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - for (int32_t i = 0; i < numOfBatch; ++i) { + for (int32_t i = 0; i < numOfMeta; ++i) { STableMetaRsp tableMetaRsp = {0}; if (tDecodeSTableMetaRsp(&decoder, &tableMetaRsp) < 0) return -1; - taosArrayPush(pRsp->pArray, &tableMetaRsp); + taosArrayPush(pRsp->pMetaRsp, &tableMetaRsp); } + + int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + if (tDecodeI32(&decoder, &numOfIndex) < 0) return -1; + + pRsp->pIndexRsp = taosArrayInit(numOfIndex, sizeof(STableIndexRsp)); + if (pRsp->pIndexRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp tableIndexRsp = {0}; + if (tDecodeCStrTo(&decoder, tableIndexRsp.tbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, tableIndexRsp.dbFName) < 0) return -1; + if (tDecodeU64(&decoder, &tableIndexRsp.suid) < 0) return -1; + if (tDecodeI32(&decoder, &tableIndexRsp.version) < 0) return -1; + int32_t num = 0; + if (tDecodeI32(&decoder, &num) < 0) return -1; + if (num > 0) { + tableIndexRsp.pIndex = taosArrayInit(num, sizeof(STableIndexInfo)); + if (NULL == tableIndexRsp.pIndex) return -1; + STableIndexInfo info; + for (int32_t i = 0; i < num; ++i) { + if (tDeserializeSTableIndexInfo(&decoder, &info) < 0) return -1; + if (NULL == taosArrayPush(tableIndexRsp.pIndex, &info)) { + taosMemoryFree(info.expr); + return -1; + } + } + } + taosArrayPush(pRsp->pIndexRsp, &tableIndexRsp); + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -2693,14 +2748,33 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { taosMemoryFreeClear(pRsp->pSchemas); } +void tFreeSTableIndexRsp(void *info) { + if (NULL == info) { + return; + } + + STableIndexRsp *pInfo = (STableIndexRsp *)info; + + taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo); +} + + void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - for (int32_t i = 0; i < numOfBatch; ++i) { - STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); + int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pMetaRsp, i); tFreeSTableMetaRsp(pMetaRsp); } - taosArrayDestroy(pRsp->pArray); + taosArrayDestroy(pRsp->pMetaRsp); + + int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp *pIndexRsp = taosArrayGet(pRsp->pIndexRsp, i); + tFreeSTableIndexRsp(pIndexRsp); + } + + taosArrayDestroy(pRsp->pIndexRsp); } int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index edb8a29ac0..395e2b5255 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -885,6 +885,8 @@ static int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp return TSDB_CODE_SUCCESS; } + strcpy(rsp->dbFName, pStb->db); + strcpy(rsp->tbName, pStb->name); rsp->suid = pStb->uid; rsp->version = pStb->smaVer; mndReleaseStb(pMnode, pStb); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 4b5b9e8a7b..9db0ab053f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1674,15 +1674,15 @@ _OVER: int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen) { SSTbHbRsp hbRsp = {0}; - hbRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); - if (hbRsp.pArray == NULL) { + hbRsp.pMetaRsp = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); + if (hbRsp.pMetaRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - hbRsp.pIndexRsp = taosMemoryCalloc(1, sizeof(STableIndexRsp)); + hbRsp.pIndexRsp = taosArrayInit(numOfStbs, sizeof(STableIndexRsp)); if (NULL == hbRsp.pIndexRsp) { - taosArrayDestroy(hbRsp.pArray); + taosArrayDestroy(hbRsp.pMetaRsp); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -1700,12 +1700,12 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) { metaRsp.numOfColumns = -1; metaRsp.suid = pStbVersion->suid; - taosArrayPush(hbRsp.pArray, &metaRsp); + taosArrayPush(hbRsp.pMetaRsp, &metaRsp); continue; } if (pStbVersion->sversion != metaRsp.sversion || pStbVersion->tversion != metaRsp.tversion) { - taosArrayPush(hbRsp.pArray, &metaRsp); + taosArrayPush(hbRsp.pMetaRsp, &metaRsp); } else { tFreeSTableMetaRsp(&metaRsp); } @@ -1713,11 +1713,19 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t if (pStbVersion->smaVer != smaVer) { bool exist = false; char tbFName[TSDB_TABLE_FNAME_LEN]; + STableIndexRsp indexRsp = {0}; sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); - int32_t code = mndGetTableSma(pMnode, tbFName, hbRsp.pIndexRsp, &exist); + int32_t code = mndGetTableSma(pMnode, tbFName, &indexRsp, &exist); if (code || !exist) { - taosMemoryFreeClear(hbRsp.pIndexRsp); + indexRsp.suid = pStbVersion->suid; + indexRsp.version = -1; + indexRsp.pIndex = NULL; } + + strcpy(indexRsp->dbFName, pStbVersion->dbFName); + strcpy(indexRsp->tbName, pStbVersion->stbName); + + taosArrayPush(hbRsp.pIndexRsp, &indexRsp); } } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 82bb151d86..11ae5b3770 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -319,8 +319,6 @@ typedef struct SCtgUpdateUserMsg { typedef struct SCtgUpdateTbIndexMsg { SCatalog* pCtg; - char dbFName[TSDB_DB_FNAME_LEN]; - char tbName[TSDB_TABLE_NAME_LEN]; STableIndex* pIndex; } SCtgUpdateTbIndexMsg; @@ -503,7 +501,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet); -int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp); +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex *pIndex, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d01fa00878..de74dc1c68 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -751,6 +751,30 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp *pRsp) { + CTG_API_ENTER(); + + int32_t code = 0; + + if (NULL == pCtg || NULL == pRsp) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + STableIndex* pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pIndex, pRsp, sizeof(STableIndex)); + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); + +_return: + + CTG_API_LEAVE(code); +} + + int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { CTG_API_ENTER(); @@ -1153,16 +1177,36 @@ int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SNam CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - int32_t code = 0; - CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); + if (*pRes) { + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } - SArray* pInfo = NULL; - CTG_ERR_JRET(ctgCloneTableIndex(*pRes, &pInfo)); + STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } - CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, pInfo, false)); + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); + + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); + *pRes = pInfo; + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, &pIndex, false)); + + CTG_API_LEAVE(code); + _return: + tFreeSTableIndexRsp(pIndex); + taosMemoryFree(pIndex); + + taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); + *pRes = NULL; + CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 69eddd7ba6..36e5b33521 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -876,10 +876,9 @@ int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf SArray* pInfo = NULL; CTG_ERR_JRET(ctgCloneTableIndex(pOut->pIndex, &pInfo)); pTask->res = pInfo; - pTask->msgCtx.out = NULL; SCtgTbIndexCtx* ctx = pTask->taskCtx; - CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, ctx->pName, pOut, false)); + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, (STableIndex**)&pTask->msgCtx.out, false)); _return: diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 49714ac2de..f2502f5943 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -838,7 +838,7 @@ _return: CTG_RET(code); } -int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp) { +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_UPDATE_TB_INDEX; @@ -851,19 +851,19 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pInde } msg->pCtg = pCtg; - msg->pIndex = pIndex; - tNameGetFullDbName(pName, msg->dbFName); - strcpy(msg->tbName, pName->tname); + msg->pIndex = *pIndex; op->data = msg; CTG_ERR_JRET(ctgEnqueue(pCtg, op)); - + + *pIndex = NULL; return TSDB_CODE_SUCCESS; _return: - taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); + taosArrayDestroyEx(*pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(*pIndex); taosMemoryFreeClear(msg); CTG_RET(code); @@ -1745,12 +1745,12 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { STableIndex* pIndex = msg->pIndex; SCtgDBCache *dbCache = NULL; - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, msg->dbFName, 0, &dbCache)); + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache)); if (NULL == dbCache) { CTG_ERR_JRET(code); } - CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->dbFName, msg->tbName, &pIndex)); + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); _return: diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index c2ebd9e210..178025704f 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -431,7 +431,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const return TSDB_CODE_SUCCESS; } -int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, SArray** out, SCtgTask* pTask) { +int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask) { char *msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_TABLE_INDEX; @@ -448,7 +448,11 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n } if (pTask) { - void* pOut = NULL; + void* pOut = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index edc468a6e2..b89c220519 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -484,17 +484,11 @@ int32_t queryProcessGetTbIndexRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_INVALID_INPUT; } - STableIndexRsp *out = taosMemoryCalloc(1, sizeof(STableIndexRsp)); - if (NULL == out) { - return TSDB_CODE_OUT_OF_MEMORY; - } - + STableIndexRsp *out = (STableIndexRsp*)output; if (tDeserializeSTableIndexRsp(msg, msgSize, out) != 0) { qError("tDeserializeSTableIndexRsp failed, msgSize:%d", msgSize); return TSDB_CODE_INVALID_MSG; } - - *(void **)output = out; return TSDB_CODE_SUCCESS; } From eb6b87b3e3609e9d3820a2707126a96d444e9613 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 13 Jun 2022 14:15:05 +0800 Subject: [PATCH 11/78] fix: invalid read/write --- source/libs/index/src/indexFilter.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 6197edd474..1cc2679b06 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -97,7 +97,14 @@ static int32_t sifGetOperParamNum(EOperatorType ty) { } return 2; } -static int32_t sifValidateColumn(SColumnNode *cn) { +static int32_t sifValidOp(EOperatorType ty) { + if ((ty >= OP_TYPE_ADD && ty <= OP_TYPE_BIT_OR) || (ty == OP_TYPE_IN || ty == OP_TYPE_NOT_IN) || + (ty == OP_TYPE_LIKE || ty == OP_TYPE_NOT_LIKE || ty == OP_TYPE_MATCH || ty == OP_TYPE_NMATCH)) { + return -1; + } + return 0; +} +static int32_t sifValidColumn(SColumnNode *cn) { // add more check if (cn == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; @@ -197,7 +204,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { case QUERY_NODE_COLUMN: { SColumnNode *cn = (SColumnNode *)node; /*only support tag column*/ - SIF_ERR_RET(sifValidateColumn(cn)); + SIF_ERR_RET(sifValidColumn(cn)); param->colId = cn->colId; param->colValType = cn->node.resType.type; @@ -505,6 +512,11 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { int32_t code = 0; + if (sifValidOp(node->opType) < 0) { + output->status = SFLT_NOT_INDEX; + return code; + } + int32_t nParam = sifGetOperParamNum(node->opType); if (nParam <= 1) { output->status = SFLT_NOT_INDEX; @@ -516,6 +528,12 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { SIFParam *params = NULL; SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); + + if (params[0].status == SFLT_NOT_INDEX || (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { + output->status = SFLT_NOT_INDEX; + return code; + } + // ugly code, refactor later output->arg = ctx->arg; sif_func_t operFn = sifNullFunc; From 7a78d81232317e63e6ee80408bfd3ba984ea967c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 15:36:28 +0800 Subject: [PATCH 12/78] fix compile issue --- include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 2 + source/common/src/tmsg.c | 1 - source/dnode/mnode/impl/inc/mndSma.h | 1 + source/dnode/mnode/impl/src/mndSma.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 9 +-- source/libs/catalog/inc/catalogInt.h | 14 +++-- source/libs/catalog/src/catalog.c | 64 ++++++++++----------- source/libs/catalog/src/ctgAsync.c | 12 ++-- source/libs/catalog/src/ctgCache.c | 85 +++++++++++++++++++++------- source/libs/catalog/src/ctgDbg.c | 18 +++--- source/libs/catalog/src/ctgUtil.c | 10 ++++ 12 files changed, 142 insertions(+), 77 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8a2b2a31f2..b4f60c2761 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1134,6 +1134,7 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); void tFreeSTableMetaRsp(STableMetaRsp* pRsp); +void tFreeSTableIndexRsp(void *info); typedef struct { SArray* pMetaRsp; // Array of STableMetaRsp diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 80b677dbd2..dba4ac39d2 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -282,6 +282,8 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes); +int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp *pRsp); + int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* pInfo); int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 5d0cc2b28d..63153d68b4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2758,7 +2758,6 @@ void tFreeSTableIndexRsp(void *info) { taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo); } - void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); for (int32_t i = 0; i < numOfMeta; ++i) { diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 4a80f619d3..63530b403d 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,6 +26,7 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); +int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 395e2b5255..9c28763082 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -872,7 +872,7 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp return code; } -static int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist) { +int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist) { int32_t code = 0; SSmaObj *pSma = NULL; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9db0ab053f..8d936e68ce 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -27,6 +27,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndSma.h" #include "tname.h" #define STB_VER_NUMBER 1 @@ -1638,7 +1639,7 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { } } else { mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); - if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { + if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp, NULL) != 0) { goto _OVER; } } @@ -1710,7 +1711,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t tFreeSTableMetaRsp(&metaRsp); } - if (pStbVersion->smaVer != smaVer) { + if (pStbVersion->smaVer && pStbVersion->smaVer != smaVer) { bool exist = false; char tbFName[TSDB_TABLE_FNAME_LEN]; STableIndexRsp indexRsp = {0}; @@ -1722,8 +1723,8 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t indexRsp.pIndex = NULL; } - strcpy(indexRsp->dbFName, pStbVersion->dbFName); - strcpy(indexRsp->tbName, pStbVersion->stbName); + strcpy(indexRsp.dbFName, pStbVersion->dbFName); + strcpy(indexRsp.tbName, pStbVersion->stbName); taosArrayPush(hbRsp.pIndexRsp, &indexRsp); } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 11ae5b3770..df4b9b6ea6 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -491,8 +491,8 @@ void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache); void ctgRUnlockVgInfo(SCtgDBCache *dbCache); int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist); int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); -int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); -int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); +int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); +int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId); int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncReq); int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq); @@ -501,13 +501,18 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet); -int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex *pIndex, bool syncOp); +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask); +void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache); +int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, SName* pTableName, SArray** pRes); +int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp); +int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation); +int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation); @@ -516,7 +521,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildU int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray *out, SCtgTask* pTask); int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask); int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *indexName, SIndexInfo *out, SCtgTask* pTask); -int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName* name, SArray** out, SCtgTask* pTask); +int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask); int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *funcName, SFuncInfo *out, SCtgTask* pTask); int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask); int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask); @@ -545,6 +550,7 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ char *ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask); int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); +void ctgFreeSTableIndex(void *info); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index de74dc1c68..2ef7771c11 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -350,7 +350,7 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo *pConn, const char* user, co *pass = false; - CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, (char*)user, (char*)dbFName, type, &inCache, pass)); if (inCache) { return TSDB_CODE_SUCCESS; @@ -382,15 +382,38 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SArray** pRes) { +int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pRes) { CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); if (*pRes) { return TSDB_CODE_SUCCESS; } - CTG_ERR_RET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); + + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); + + *pRes = pInfo; + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); return TSDB_CODE_SUCCESS; + +_return: + + tFreeSTableIndexRsp(pIndex); + taosMemoryFree(pIndex); + + taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); + *pRes = NULL; + + CTG_RET(code); } @@ -416,7 +439,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTabl CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo)); if (dbCache) { - vgHash = dbCache->vgInfo->vgHash; + vgHash = dbCache->vgCache.vgInfo->vgHash; } else { vgHash = vgInfo->vgHash; } @@ -640,9 +663,9 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - *version = dbCache->vgInfo->vgVersion; + *version = dbCache->vgCache.vgInfo->vgVersion; *dbId = dbCache->dbId; - *tableNum = dbCache->vgInfo->numOfTable; + *tableNum = dbCache->vgCache.vgInfo->numOfTable; ctgReleaseVgInfoToCache(pCtg, dbCache); @@ -969,7 +992,7 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SDBVgInfo *vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); _return: @@ -1177,36 +1200,11 @@ int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SNam CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); - if (*pRes) { - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } - - STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); - if (NULL == pIndex) { - CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); - } - int32_t code = 0; - CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); - - SArray* pInfo = NULL; - CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); - - *pRes = pInfo; - - CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, &pIndex, false)); - - CTG_API_LEAVE(code); + CTG_ERR_JRET(ctgGetTbIndex(pCtg, pConn, (SName*)pTableName, pRes)); _return: - tFreeSTableIndexRsp(pIndex); - taosMemoryFree(pIndex); - - taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); - *pRes = NULL; - CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 36e5b33521..b893201c53 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -685,7 +685,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); @@ -1016,7 +1016,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); @@ -1064,7 +1064,7 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { - CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgInfo->vgHash, (SArray**)&pTask->res)); + CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgCache.vgInfo->vgHash, (SArray**)&pTask->res)); CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1098,7 +1098,7 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { if (NULL == pTask->res) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1171,9 +1171,9 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask *pTask) { SDbInfo* pInfo = (SDbInfo*)pTask->res; CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { - pInfo->vgVer = dbCache->vgInfo->vgVersion; + pInfo->vgVer = dbCache->vgCache.vgInfo->vgVersion; pInfo->dbId = dbCache->dbId; - pInfo->tbNum = dbCache->vgInfo->numOfTable; + pInfo->tbNum = dbCache->vgCache.vgInfo->numOfTable; } else { pInfo->vgVer = CTG_DEFAULT_INVALID_VERSION; } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index f2502f5943..1fa61f1c59 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -64,8 +64,12 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { CTG_OP_UPDATE_TB_INDEX, "update tbIndex", ctgOpUpdateTbIndex - } - + }, + { + CTG_OP_DROP_TB_INDEX, + "drop tbIndex", + ctgOpDropTbIndex + } }; @@ -305,6 +309,7 @@ _return: int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -383,7 +388,7 @@ _return: CTG_RET(code); } -int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, +int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName) { *sver = -1; *tver = -1; @@ -444,10 +449,10 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t * } -int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { +int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char *tableName, int32_t *tbType) { SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; - ctgAcquireTbMetaFromCache(pCtg, dbFName, tableName, &dbCache, &tbCache); + CTG_ERR_RET(ctgAcquireTbMetaFromCache(pCtg, dbFName, tableName, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; @@ -461,7 +466,7 @@ int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char * return TSDB_CODE_SUCCESS; } -int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, const SName* pTableName, SArray** pRes) { +int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, SName* pTableName, SArray** pRes) { int32_t code = 0; SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; @@ -485,7 +490,7 @@ _return: CTG_RET(code); } -int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { +int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { if (NULL == pCtg->userCache) { ctgDebug("empty user auth cache, user:%s", user); goto _return; @@ -862,7 +867,7 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncO _return: - taosArrayDestroyEx(*pIndex, tFreeSTableIndexInfo); + taosArrayDestroyEx((*pIndex)->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(*pIndex); taosMemoryFreeClear(msg); @@ -893,7 +898,6 @@ int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp) { _return: - taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(msg); CTG_RET(code); @@ -1172,7 +1176,7 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d CTG_LOCK(CTG_WRITE, &dbCache->dbLock); atomic_store_8(&dbCache->deleted, 1); - ctgRemoveStbRent(pCtg, &dbCache); + ctgRemoveStbRent(pCtg, dbCache); ctgFreeDbCache(dbCache); CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); @@ -1251,6 +1255,8 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char* dbFName, char* tbName, uin ctgDebug("db %s,%" PRIx64 " stb %s,%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", dbFName, dbId, tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer); + + return TSDB_CODE_SUCCESS; } @@ -1330,32 +1336,40 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFName, char *tbName, STableIndex **index) { if (NULL == dbCache->tbCache) { + ctgFreeSTableIndex(*index); taosMemoryFreeClear(*index); ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } STableIndex* pIndex = *index; + uint64_t suid = pIndex->suid; SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { SCtgTbCache cache = {0}; cache.pIndex = pIndex; - if (taosHashPut(table->tbCache, tbName, strlen(tbName), &cache, sizeof(cache)) != 0) { + if (taosHashPut(dbCache->tbCache, tbName, strlen(tbName), &cache, sizeof(cache)) != 0) { + ctgFreeSTableIndex(*index); taosMemoryFreeClear(*index); ctgError("taosHashPut new tbCache failed, tbName:%s", tbName); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } *index = NULL; - ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); - - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); + if (suid) { + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + } + return TSDB_CODE_SUCCESS; } if (pCache->pIndex) { + if (0 == suid) { + suid = pCache->pIndex->suid; + } taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(pCache->pIndex); } @@ -1363,9 +1377,11 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFNa pCache->pIndex = pIndex; *index = NULL; - ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + if (suid) { + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, suid, pCache)); + } return TSDB_CODE_SUCCESS; } @@ -1390,6 +1406,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { SCtgUpdateVgMsg *msg = operation->data; SDBVgInfo* dbInfo = msg->dbInfo; char* dbFName = msg->dbFName; + SCatalog* pCtg = msg->pCtg; if (NULL == dbInfo->vgHash) { return TSDB_CODE_SUCCESS; @@ -1611,7 +1628,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId %" PRIx64 " not match with curId %"PRIx64", dbFName:%s, tbName:%s"msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); + ctgDebug("dbId %" PRIx64 " not match with curId %"PRIx64", dbFName:%s, tbName:%s", msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); return TSDB_CODE_SUCCESS; } @@ -1746,9 +1763,39 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { SCtgDBCache *dbCache = NULL; CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache)); - if (NULL == dbCache) { - CTG_ERR_JRET(code); + + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); + +_return: + + if (pIndex) { + taosArrayDestroyEx(pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pIndex); } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgDropTbIndexMsg *msg = operation->data; + SCatalog* pCtg = msg->pCtg; + SCtgDBCache *dbCache = NULL; + + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + STableIndex* pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + strcpy(pIndex->tbName, msg->tbName); + strcpy(pIndex->dbFName, msg->dbFName); + pIndex->version = -1; CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 52f29e3acb..2403dc3c5a 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -266,11 +266,11 @@ int32_t ctgdGetStatNum(char *option, void *res) { } int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) { - return dbCache->table.tbCache ? (int32_t)taosHashGetSize(dbCache->table.tbCache) : 0; + return dbCache->tbCache ? (int32_t)taosHashGetSize(dbCache->tbCache) : 0; } int32_t ctgdGetStbNum(SCtgDBCache *dbCache) { - return dbCache->table.stbCache ? (int32_t)taosHashGetSize(dbCache->table.stbCache) : 0; + return dbCache->stbCache ? (int32_t)taosHashGetSize(dbCache->stbCache) : 0; } int32_t ctgdGetRentNum(SCtgRentMgmt *rent) { @@ -363,17 +363,17 @@ void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { dbFName = taosHashGetKey(pIter, &len); - int32_t metaNum = dbCache->table.tbCache ? taosHashGetSize(dbCache->table.tbCache) : 0; - int32_t stbNum = dbCache->table.stbCache ? taosHashGetSize(dbCache->table.stbCache) : 0; + int32_t metaNum = dbCache->tbCache ? taosHashGetSize(dbCache->tbCache) : 0; + int32_t stbNum = dbCache->stbCache ? taosHashGetSize(dbCache->stbCache) : 0; int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; int32_t hashMethod = -1; int32_t vgNum = 0; - if (dbCache->vgInfo) { - vgVersion = dbCache->vgInfo->vgVersion; - hashMethod = dbCache->vgInfo->hashMethod; - if (dbCache->vgInfo->vgHash) { - vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); + if (dbCache->vgCache.vgInfo) { + vgVersion = dbCache->vgCache.vgInfo->vgVersion; + hashMethod = dbCache->vgCache.vgInfo->hashMethod; + if (dbCache->vgCache.vgInfo->vgHash) { + vgNum = taosHashGetSize(dbCache->vgCache.vgInfo->vgHash); } } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index f0ed60a689..1cb2b99a67 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -44,6 +44,16 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { } } +void ctgFreeSTableIndex(void *info) { + if (NULL == info) { + return; + } + + STableIndex *pInfo = (STableIndex *)info; + + taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo); +} + void ctgFreeSMetaData(SMetaData* pData) { taosArrayDestroy(pData->pTableMeta); pData->pTableMeta = NULL; From f47a28fe8a487329a17686f762a7e54c38f0c373 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 13 Jun 2022 15:47:43 +0800 Subject: [PATCH 13/78] enh(sync): add leader transfer --- include/libs/sync/sync.h | 1 + include/libs/sync/syncTools.h | 1 + source/libs/sync/src/syncMain.c | 68 ++++++++++++++++++++++---- source/libs/sync/test/syncTestTool.cpp | 40 +++++++++++---- 4 files changed, 92 insertions(+), 18 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 2673bc382b..a369b81d26 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -111,6 +111,7 @@ typedef struct SSyncFSM { void (*FpRestoreFinishCb)(struct SSyncFSM* pFsm); void (*FpReConfigCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta); + void (*FpLeaderTransferCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot); diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 68a33d48cb..fbf124bf47 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -467,6 +467,7 @@ typedef struct SyncLeaderTransfer { SRaftId srcId; SRaftId destId; */ + SNodeInfo newNodeInfo; SRaftId newLeaderId; } SyncLeaderTransfer; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 82fde80be2..904cdae94e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -35,7 +35,7 @@ #include "syncVoteMgr.h" #include "tref.h" -bool gRaftDetailLog = true; +bool gRaftDetailLog = false; static int32_t tsNodeRefId = -1; @@ -183,13 +183,21 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg int32_t ret = 0; bool IamInNew = false; for (int i = 0; i < pNewCfg->replicaNum; ++i) { - SRaftId newId; - newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); - newId.vgId = pSyncNode->vgId; - if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + if (strcmp((pNewCfg->nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 && + (pNewCfg->nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) { IamInNew = true; } + + /* + SRaftId newId; + newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); + newId.vgId = pSyncNode->vgId; + if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + IamInNew = true; + } + */ } + if (!IamInNew) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); return TAOS_SYNC_NOT_IN_NEW_CONFIG; @@ -267,7 +275,7 @@ int32_t syncLeaderTransfer(int64_t rid) { } ASSERT(rid == pSyncNode->rid); - if (pSyncNode->peersNum > 0) { + if (pSyncNode->peersNum == 0) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); return TAOS_SYNC_OTHER_ERROR; } @@ -296,6 +304,7 @@ int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader) { SyncLeaderTransfer* pMsg = syncLeaderTransferBuild(pSyncNode->vgId); pMsg->newLeaderId.addr = syncUtilAddr2U64(newLeader.nodeFqdn, newLeader.nodePort); pMsg->newLeaderId.vgId = pSyncNode->vgId; + pMsg->newNodeInfo = newLeader; ASSERT(pMsg != NULL); SRpcMsg rpcMsg = {0}; syncLeaderTransfer2RpcMsg(pMsg, &rpcMsg); @@ -1887,10 +1896,51 @@ const char* syncStr(ESyncState state) { } static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { - SyncLeaderTransfer* pSyncLeaderTransfer; - if (syncUtilSameId(&(pSyncLeaderTransfer->newLeaderId), &(ths->myRaftId))) { + SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); + +/* + char host[128]; + uint16_t port; + syncUtilU642Addr(pSyncLeaderTransfer->newLeaderId.addr, host, sizeof(host), &port); + sDebug("vgId:%d sync event, maybe leader transfer to %s:%d %lu", ths->vgId, host, port, + pSyncLeaderTransfer->newLeaderId.addr); +*/ + + sDebug("vgId:%d sync event, begin leader transfer", ths->vgId); + + if (strcmp(pSyncLeaderTransfer->newNodeInfo.nodeFqdn, ths->myNodeInfo.nodeFqdn) == 0 && pSyncLeaderTransfer->newNodeInfo.nodePort == ths->myNodeInfo.nodePort) { + sDebug("vgId:%d sync event, maybe leader transfer to %s:%d %lu", ths->vgId, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, pSyncLeaderTransfer->newNodeInfo.nodePort, + pSyncLeaderTransfer->newLeaderId.addr); + + // reset elect timer now! + int32_t electMS = 1; + int32_t ret = syncNodeRestartElectTimer(ths, electMS); + ASSERT(ret == 0); } + +/* + if (syncUtilSameId(&(pSyncLeaderTransfer->newLeaderId), &(ths->myRaftId))) { + // reset elect timer now! + int32_t electMS = 1; + int32_t ret = syncNodeRestartElectTimer(ths, electMS); + ASSERT(ret == 0); + } +*/ + if (ths->pFsm->FpLeaderTransferCb != NULL) { + SFsmCbMeta cbMeta; + cbMeta.code = 0; + cbMeta.currentTerm = ths->pRaftStore->currentTerm; + cbMeta.flag = 0; + cbMeta.index = pEntry->index; + cbMeta.isWeak = pEntry->isWeak; + cbMeta.seqNum = pEntry->seqNum; + cbMeta.state = ths->state; + cbMeta.term = pEntry->term; + ths->pFsm->FpLeaderTransferCb(ths->pFsm, pRpcMsg, cbMeta); + } + + syncLeaderTransferDestroy(pSyncLeaderTransfer); return 0; } @@ -1992,7 +2042,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ASSERT(code == 0); } - // config change + // leader transfer if (pEntry->originalRpcType == TDMT_SYNC_LEADER_TRANSFER) { code = syncDoLeaderTransfer(ths, &rpcMsg, pEntry); ASSERT(code == 0); diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index 0c8b26e9d9..ebcd7368cc 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -153,6 +153,16 @@ void ReConfigCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMe taosMemoryFree(s); } +void LeaderTransferCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==LeaderTransferCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " + "currentTerm:%lu \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + SSyncFSM* createFsm() { SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); memset(pFsm, 0, sizeof(*pFsm)); @@ -172,6 +182,8 @@ SSyncFSM* createFsm() { pFsm->FpSnapshotStopWrite = SnapshotStopWrite; pFsm->FpSnapshotDoWrite = SnapshotDoWrite; + pFsm->FpLeaderTransferCb = LeaderTransferCb; + return pFsm; } @@ -277,7 +289,8 @@ void usage(char* exe) { printf( "usage: %s replicaNum(1-5) myIndex(0-..) enableSnapshot(0/1) lastApplyIndex(>=-1) lastApplyTerm(>=0) " "writeRecordNum(>=0) " - "isStandBy(0/1) isConfigChange(0-5) iterTimes(>=0) finishLastApplyIndex(>=-1) finishLastApplyTerm(>=0) \n", + "isStandBy(0/1) isConfigChange(0-5) iterTimes(>=0) finishLastApplyIndex(>=-1) finishLastApplyTerm(>=0) " + "leaderTransfer(0/1) \n", exe); } @@ -294,9 +307,9 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) { int main(int argc, char** argv) { sprintf(tsTempDir, "%s", "."); tsAsyncLog = 0; - sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR; + sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR + DEBUG_DEBUG; - if (argc != 12) { + if (argc != 13) { usage(argv[0]); exit(-1); } @@ -312,12 +325,14 @@ int main(int argc, char** argv) { int32_t iterTimes = atoi(argv[9]); int32_t finishLastApplyIndex = atoi(argv[10]); int32_t finishLastApplyTerm = atoi(argv[11]); + int32_t leaderTransfer = atoi(argv[12]); - sTrace( + sInfo( "args: replicaNum:%d, myIndex:%d, enableSnapshot:%d, lastApplyIndex:%d, lastApplyTerm:%d, writeRecordNum:%d, " - "isStandBy:%d, isConfigChange:%d, iterTimes:%d, finishLastApplyIndex:%d, finishLastApplyTerm:%d", + "isStandBy:%d, isConfigChange:%d, iterTimes:%d, finishLastApplyIndex:%d, finishLastApplyTerm:%d, " + "leaderTransfer:%d", replicaNum, myIndex, enableSnapshot, lastApplyIndex, lastApplyTerm, writeRecordNum, isStandBy, isConfigChange, - iterTimes, finishLastApplyIndex, finishLastApplyTerm); + iterTimes, finishLastApplyIndex, finishLastApplyTerm, leaderTransfer); // check parameter assert(replicaNum >= 1 && replicaNum <= 5); @@ -363,24 +378,31 @@ int main(int argc, char** argv) { //--------------------------- int32_t alreadySend = 0; + int32_t leaderTransferWait = 0; while (1) { char* simpleStr = syncNode2SimpleStr(pSyncNode); + leaderTransferWait++; + if (leaderTransferWait == 7) { + sTrace("begin leader transfer ..."); + int32_t ret = syncLeaderTransfer(rid); + } + if (alreadySend < writeRecordNum) { SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); int32_t ret = syncPropose(rid, pRpcMsg, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { - sTrace("%s value%d write not leader", simpleStr, alreadySend); + sTrace("%s value%d write not leader, leaderTransferWait:%d", simpleStr, alreadySend, leaderTransferWait); } else { assert(ret == 0); - sTrace("%s value%d write ok", simpleStr, alreadySend); + sTrace("%s value%d write ok, leaderTransferWait:%d", simpleStr, alreadySend, leaderTransferWait); } alreadySend++; rpcFreeCont(pRpcMsg->pCont); taosMemoryFree(pRpcMsg); } else { - sTrace("%s", simpleStr); + sTrace("%s, leaderTransferWait:%d", simpleStr, leaderTransferWait); } taosMsleep(1000); From bb89ceb6bd0bc4ee03d2c8ae1f047194ad581e06 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 18:46:02 +0800 Subject: [PATCH 14/78] change stb hash value to stbName --- source/client/inc/clientInt.h | 2 +- source/libs/catalog/src/ctgCache.c | 127 ++++++++++++++++++++++++----- source/libs/catalog/src/ctgDbg.c | 2 +- 3 files changed, 108 insertions(+), 23 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 6aa83e9575..d72aa39e4c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -45,7 +45,7 @@ extern "C" { #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms -#define SYNC_ON_TOP_OF_ASYNC 1 +#define SYNC_ON_TOP_OF_ASYNC 0 enum { RES_TYPE__QUERY = 1, diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 1fa61f1c59..b938dff932 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -230,6 +230,7 @@ _return: int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { SCtgDBCache *dbCache = NULL; + SCtgTbCache* pCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); @@ -237,7 +238,7 @@ int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, S } int32_t sz = 0; - SCtgTbCache* pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); goto _return; @@ -267,6 +268,54 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t ctgAcquireStbMetaFromCache(SCatalog* pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache** pTb) { + SCtgDBCache* dbCache = NULL; + SCtgTbCache* pCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + char* stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + if (NULL == stName) { + ctgDebug("stb %" PRIx64 " not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + pCache = taosHashAcquire(dbCache->tbCache, stName, strlen(stName)); + if (NULL == pCache) { + ctgDebug("stb %" PRIx64 " name %s not in cache, dbFName:%s", suid, stName, dbFName); + taosHashRelease(dbCache->stbCache, stName); + goto _return; + } + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("stb %" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + *pDb = dbCache; + *pTb = pCache; + + ctgDebug("stb %" PRIx64 " meta got in cache, dbFName:%s", suid, dbFName); + + CTG_CACHE_STAT_INC(tbMetaHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbMetaMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); @@ -348,31 +397,61 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** ctx->tbInfo.dbId = dbCache->dbId; ctx->tbInfo.suid = tbMeta->suid; ctx->tbInfo.tbType = tbMeta->tableType; - + if (tbMeta->tableType != TSDB_CHILD_TABLE) { + int32_t metaSize = CTG_META_SIZE(tbMeta); + *pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == *pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(*pTableMeta, tbMeta, metaSize); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", ctx->pName->tname, tbMeta->tableType, dbFName); return TSDB_CODE_SUCCESS; } + + // PROCESS FOR CHILD TABLE - STableMeta **stbMeta = taosHashGet(dbCache->stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); - if (NULL == stbMeta || NULL == *stbMeta) { - ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); - goto _return; + int32_t metaSize = sizeof(SCTableMeta); + *pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == *pTableMeta) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - if ((*stbMeta)->suid != tbMeta->suid) { - ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%"PRIx64 , (*stbMeta)->suid, tbMeta->suid); + memcpy(*pTableMeta, tbMeta, metaSize); + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", + ctx->pName->tname, ctx->tbInfo.tbType, dbFName); + + dbCache = NULL; + tbCache = NULL; + ctgAcquireStbMetaFromCache(pCtg, dbFName, ctx->tbInfo.suid, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + taosMemoryFreeClear(*pTableMeta); + ctgDebug("stb %" PRIx64 " meta not in cache", ctx->tbInfo.suid); + return TSDB_CODE_SUCCESS; + } + + STableMeta* stbMeta = tbCache->pMeta; + if (stbMeta->suid != ctx->tbInfo.suid) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%"PRIx64 , stbMeta->suid, ctx->tbInfo.suid); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - int32_t metaSize = CTG_META_SIZE(*stbMeta); + metaSize = CTG_META_SIZE(stbMeta); *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); + memcpy(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -419,27 +498,33 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, return TSDB_CODE_SUCCESS; } - STableMeta **stbMeta = taosHashGet(dbCache->stbCache, suid, sizeof(*suid)); - if (NULL == stbMeta || NULL == *stbMeta) { + // PROCESS FOR CHILD TABLE + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); + + ctgAcquireStbMetaFromCache(pCtg, dbFName, *suid, &dbCache, &tbCache); + if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid); + ctgDebug("stb %" PRIx64 " meta not in cache", *suid); return TSDB_CODE_SUCCESS; } - - if ((*stbMeta)->suid != *suid) { + + STableMeta* stbMeta = tbCache->pMeta; + if (stbMeta->suid != *suid) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%" PRIx64 , (*stbMeta)->suid, *suid); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%" PRIx64 , stbMeta->suid, *suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } size_t nameLen = 0; - char *name = taosHashGetKey(*stbMeta, &nameLen); + char *name = taosHashGetKey(tbCache, &nameLen); strncpy(stbName, name, nameLen); stbName[nameLen] = 0; - *sver = (*stbMeta)->sversion; - *tver = (*stbMeta)->tversion; + *sver = stbMeta->sversion; + *tver = stbMeta->tversion; ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -1320,14 +1405,14 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam return TSDB_CODE_SUCCESS; } - if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), &pCache, POINTER_BYTES) != 0) { + if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), tbName, strlen(tbName) + 1) != 0) { ctgError("taosHashPut to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_INC(stblNum, 1); - ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + ctgDebug("stb %" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 2403dc3c5a..cd61abf6cc 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.apiEnable = true}; +SCtgDebug gCTGDebug = {.apiEnable = true, .lockEnable = true}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); From 6d952626882dbdc66a8c4cb1572b1ca69bf9330d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 19:11:28 +0800 Subject: [PATCH 15/78] enh(query): add first/last function distributed implementation --- include/libs/function/functionMgt.h | 2 + source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 57 +++++++++++++++++++++++++ source/libs/function/src/builtinsimpl.c | 4 ++ 4 files changed, 64 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index e86d643c0d..f01e734fde 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -138,6 +138,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_TOP_MERGE, FUNCTION_TYPE_BOTTOM_PARTIAL, FUNCTION_TYPE_BOTTOM_MERGE, + FUNCTION_TYPE_FIRST_PARTIAL, + FUNCTION_TYPE_FIRST_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index a528041964..f3bcad8ede 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -97,6 +97,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx); int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); +int32_t getFirstLastInfoSize(int32_t resBytes); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); bool getTopBotMergeFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 23940a415e..a2849d601e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -954,6 +954,41 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l return TSDB_CODE_SUCCESS; } +static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { + // first(col_list) will be rewritten as first(col) + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return TSDB_CODE_SUCCESS; + } + + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + uint8_t paraType = ((SExprNode*)pPara)->resType.type; + int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes; + if (isPartial) { + if (QUERY_NODE_COLUMN != nodeType(pPara)) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "The parameters of first/last can only be columns"); + } + + pFunc->node.resType = (SDataType){.bytes = getFirstLastInfoSize(paraBytes) + VARSTR_HEADER_SIZE, + .type = TSDB_DATA_TYPE_BINARY}; + } else { + if (TSDB_DATA_TYPE_BINARY != paraType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = ((SExprNode*)pPara)->resType; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateFirstLastPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateFirstLastImpl(pFunc, pErrBuf, len, true); +} + +static int32_t translateFirstLastMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateFirstLastImpl(pFunc, pErrBuf, len, false); +} + static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (1 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -1706,6 +1741,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = firstLastFinalize, .combineFunc = firstCombine, }, + { + .name = "_first_partial", + .type = FUNCTION_TYPE_FIRST_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastPartial, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = firstFunction, + .finalizeFunc = firstLastFinalize, + .combineFunc = firstCombine, + }, + { + .name = "_first_merge", + .type = FUNCTION_TYPE_FIRST_MERGE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastMerge, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = firstFunction, + .finalizeFunc = firstLastFinalize, + .combineFunc = firstCombine, + }, { .name = "last", .type = FUNCTION_TYPE_LAST, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3d64f82eba..44418685b0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2244,6 +2244,10 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) return TSDB_CODE_SUCCESS; } +int32_t getFirstLastInfoSize(int32_t resBytes) { + return resBytes + sizeof(int64_t); +} + bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); pEnv->calcMemSize = pNode->node.resType.bytes + sizeof(int64_t); From d1b149a9f87c134ee47cc5b81d9c43e688ae43ea Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 19:35:56 +0800 Subject: [PATCH 16/78] fix: adjust the initial array size of fs --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbFS.c | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 40a23447b7..4dae9bdeab 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -110,6 +110,8 @@ int32_t tsdbBegin(STsdb *pTsdb) { } int32_t tsdbCommit(STsdb *pTsdb) { + if (!pTsdb) return 0; + int32_t code = 0; SCommitH commith = {0}; SDFileSet *pSet = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index f1941a3bad..055b6c62de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -20,6 +20,7 @@ extern const char *TSDB_LEVEL_DNAME[]; typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; #define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3) +#define TSDB_MAX_INIT_FSETS (365000) static int tsdbComparFidFSet(const void *arg1, const void *arg2); static void tsdbResetFSStatus(SFSStatus *pStatus); @@ -210,6 +211,10 @@ STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg) { return NULL; } + if (maxFSet > TSDB_MAX_INIT_FSETS) { + maxFSet = TSDB_MAX_INIT_FSETS; + } + pfs->cstatus = tsdbNewFSStatus(maxFSet); if (pfs->cstatus == NULL) { tsdbFreeFS(pfs); From 1227d6aed222be463c9b83b2c7a27f4e7682ed28 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 19:39:40 +0800 Subject: [PATCH 17/78] other: adjust debug logs --- source/common/src/trow.c | 15 --------------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 18 ------------------ source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +----- 3 files changed, 1 insertion(+), 38 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index fb4413478d..b1ca406a4a 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -527,20 +527,6 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols return terrno; } - if (pRowCol->colId == 2) { - ASSERT(sVal.valType == 0); - int32_t val = *(int32_t *)sVal.val; - ASSERT(val == 7); - } else if (pRowCol->colId == 3) { - ASSERT(sVal.valType == 0); - int64_t val = *(int64_t *)sVal.val; - ASSERT(val == 77777); - } else if (pRowCol->colId == 4) { - ASSERT(sVal.valType == 0); - int16_t val = *(int16_t *)sVal.val; - ASSERT(val == 777); - } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; @@ -551,7 +537,6 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; - ASSERT(0); } } #if 0 diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 4dae9bdeab..fe89321ae9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1350,24 +1350,6 @@ static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLi isLast = true; } - SDataCols *pCols = pCommith->pDataCols; - SDataCol pCol_1 = pCols->cols[1]; - SDataCol pCol_2 = pCols->cols[2]; - SDataCol pCol_3 = pCols->cols[3]; - SCellVal sVal = {0}; - tdGetColDataOfRow(&sVal, &pCol_1, 0, pCols->bitmapMode); - ASSERT(sVal.valType == 0); - int32_t val1 = *(int32_t *)sVal.val; - ASSERT(val1 == 7); - tdGetColDataOfRow(&sVal, &pCol_2, 0, pCols->bitmapMode); - ASSERT(sVal.valType == 0); - int64_t val2 = *(int64_t *)sVal.val; - ASSERT(val2 == 77777); - tdGetColDataOfRow(&sVal, &pCol_3, 0, pCols->bitmapMode); - ASSERT(sVal.valType == 0); - int16_t val3 = *(int16_t *)sVal.val; - ASSERT(val3 == 777); - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 9bb0834547..ab2efa4791 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -15,8 +15,6 @@ #include "vnd.h" -int32_t gForceCommit = 1; - static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -184,8 +182,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } // commit if need - if ((gForceCommit == 1) || vnodeShouldCommit(pVnode)) { - gForceCommit = 0; + if (vnodeShouldCommit(pVnode)) { vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change vnodeCommit(pVnode); @@ -722,7 +719,6 @@ static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const } static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - gForceCommit = 1; SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; SSubmitRsp submitRsp = {0}; SSubmitMsgIter msgIter = {0}; From 8fbf5f3dba20d9089251c3783712c4eb29ae5c7d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 19:40:39 +0800 Subject: [PATCH 18/78] other: adjust debug logs --- source/common/src/trow.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index b1ca406a4a..c8a28d7f28 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -523,10 +523,8 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SCellVal sVal = {0}; if (pRowCol->colId == pDataCol->colId) { if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) { - ASSERT(0); return terrno; } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; From b83197398eeee6e6f8d7b8506701a1ee1c675ba1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 19:47:14 +0800 Subject: [PATCH 19/78] fix msg decode issue --- source/common/src/tmsg.c | 18 ++++++++---------- source/dnode/mnode/impl/src/mndSma.c | 5 +++-- source/dnode/mnode/impl/src/mndStb.c | 6 ++++++ source/libs/catalog/src/ctgCache.c | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 127d4ac273..31b8eb71bf 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2652,7 +2652,8 @@ int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { } int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); - for (int32_t i = 0; i < numOfMeta; ++i) { + if (tEncodeI32(&encoder, numOfIndex) < 0) return -1; + for (int32_t i = 0; i < numOfIndex; ++i) { STableIndexRsp *pIndexRsp = taosArrayGet(pRsp->pIndexRsp, i); if (tEncodeCStr(&encoder, pIndexRsp->tbName) < 0) return -1; if (tEncodeCStr(&encoder, pIndexRsp->dbFName) < 0) return -1; @@ -2660,12 +2661,10 @@ int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tEncodeI32(&encoder, pIndexRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pIndexRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; - if (num > 0) { - for (int32_t i = 0; i < num; ++i) { - STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, i); - if (tSerializeSTableIndexInfo(&encoder, pInfo) < 0) return -1; - } - } + for (int32_t i = 0; i < num; ++i) { + STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, i); + if (tSerializeSTableIndexInfo(&encoder, pInfo) < 0) return -1; + } } tEndEncode(&encoder); @@ -2693,9 +2692,8 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; - int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + int32_t numOfMeta = 0; if (tDecodeI32(&decoder, &numOfMeta) < 0) return -1; - pRsp->pMetaRsp = taosArrayInit(numOfMeta, sizeof(STableMetaRsp)); if (pRsp->pMetaRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2708,7 +2706,7 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { taosArrayPush(pRsp->pMetaRsp, &tableMetaRsp); } - int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + int32_t numOfIndex = 0; if (tDecodeI32(&decoder, &numOfIndex) < 0) return -1; pRsp->pIndexRsp = taosArrayInit(numOfIndex, sizeof(STableIndexRsp)); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 9cb7991d33..3f0d20348b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -532,6 +532,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea SStreamObj streamObj = {0}; tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN); + tstrncpy(streamObj.targetDb, streamObj.sourceDb, TSDB_DB_FNAME_LEN); streamObj.createTime = taosGetTimestampMs(); streamObj.updateTime = streamObj.createTime; streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); @@ -913,7 +914,7 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool } strcpy(rsp->dbFName, pStb->db); - strcpy(rsp->tbName, pStb->name); + strcpy(rsp->tbName, pStb->name + strlen(pStb->db) + 1); rsp->suid = pStb->uid; rsp->version = pStb->smaVer; mndReleaseStb(pMnode, pStb); @@ -1127,4 +1128,4 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index e099359d47..6f89c97f83 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1715,6 +1715,12 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t bool exist = false; char tbFName[TSDB_TABLE_FNAME_LEN]; STableIndexRsp indexRsp = {0}; + indexRsp.pIndex = taosArrayInit(10, sizeof(STableIndexInfo)); + if (NULL == indexRsp.pIndex) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); int32_t code = mndGetTableSma(pMnode, tbFName, &indexRsp, &exist); if (code || !exist) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index b938dff932..724b54d250 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1445,7 +1445,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFNa ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); if (suid) { - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, &cache)); } return TSDB_CODE_SUCCESS; From fc32f716287910a0e7f3e91dd190a5fd0425adb4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 13 Jun 2022 19:59:30 +0800 Subject: [PATCH 20/78] opt: optimize group by tag --- source/libs/executor/inc/executorInt.h | 3 - source/libs/executor/inc/executorimpl.h | 8 +-- source/libs/executor/src/executorimpl.c | 92 +++++++++++++++++++++++- source/libs/executor/src/groupoperator.c | 6 +- source/libs/executor/src/scanoperator.c | 40 ++--------- 5 files changed, 100 insertions(+), 49 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 88f308710e..df54161720 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -27,10 +27,7 @@ typedef struct { int32_t bytes; } SGroupKeys, SStateKeys; -int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList); uint64_t calcGroupId(char* pData, int32_t len); -void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex); -int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals); #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 94c6512e77..3e24e43233 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -336,12 +336,6 @@ typedef struct STableScanInfo { int32_t dataBlockLoadFlag; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. - SArray* pGroupCols; - SArray* pGroupColVals; // current group column values, SArray - char* keyBuf; // group by keys for hash - int32_t groupKeyLen; // total group by column width - SHashObj* pGroupSet; // quick locate the window object for each result - SSampleExecInfo sample; // sample execution info int32_t curTWinIdx; } STableScanInfo; @@ -789,7 +783,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SArray* groupKyes, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 2c067bf488..13ac74199e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4592,6 +4592,85 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI return TSDB_CODE_SUCCESS; } +int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey){ + if(groupKey == NULL) { + return TDB_CODE_SUCCESS; + } + + pTableListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (pTableListInfo->map == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t keyLen = 0; + void *keyBuf = NULL; + int32_t numOfGroupCols = taosArrayGetSize(groupKey); + for (int32_t j = 0; j < numOfGroupCols; ++j) { + SColumn* pCol = taosArrayGet(groupKey, j); + keyLen += pCol->bytes; // actual data + null_flag + } + + int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + keyLen += nullFlagSize; + + keyBuf = taosMemoryCalloc(1, keyLen); + if (keyBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for(int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++){ + STableKeyInfo *info = taosArrayGet(pTableListInfo->pTableList, i); + SMetaReader mr = {0}; + metaReaderInit(&mr, pHandle->meta, 0); + metaGetTableEntryByUid(&mr, info->uid); + + char* isNull = (char*)keyBuf; + char* pStart = (char*)keyBuf + sizeof(int8_t) * numOfGroupCols; + for (int32_t j = 0; j < numOfGroupCols; ++j) { + SColumn* pCol = taosArrayGet(groupKey, j); + + if(strcmp(pCol->name, "tbname") == 0){ + isNull[i] = 0; + memcpy(pStart, mr.me.name, strlen(mr.me.name)); + pStart += strlen(mr.me.name); + }else{ + STagVal tagVal = {0}; + tagVal.cid = pCol->colId; + const char* p = metaGetTableTagVal(&mr.me, pCol->type, &tagVal); + if(p == NULL){ + isNull[j] = 1; + continue; + } + isNull[i] = 0; + if (pCol->type == TSDB_DATA_TYPE_JSON) { +// int32_t dataLen = getJsonValueLen(pkey->pData); +// memcpy(pStart, (pkey->pData), dataLen); +// pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pCol->type)) { + memcpy(pStart, tagVal.pData, tagVal.nData); + pStart += tagVal.nData; + ASSERT(tagVal.nData <= pCol->bytes); + } else { + memcpy(pStart, &(tagVal.i64), pCol->bytes); + pStart += pCol->bytes; + } + } + } + + int32_t len = (int32_t) (pStart - (char*)keyBuf); + uint64_t* groupId = taosHashGet(pTableListInfo->map, keyBuf, len); + if (groupId) { + taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), groupId, sizeof(uint64_t)); + } else { + uint64_t tmpId = calcGroupId(keyBuf, len); + taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &tmpId, sizeof(uint64_t)); + } + + metaReaderClear(&mr); + } + taosMemoryFree(keyBuf); + return TDB_CODE_SUCCESS; +} + SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, SNode* pTagCond) { int32_t type = nodeType(pPhyNode); @@ -4605,15 +4684,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pDataReader == NULL && terrno != 0) { return NULL; } - SArray* groupKyes = extractPartitionColInfo(pTableScanNode->pPartitionKeys); + int32_t code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); if (code) { tsdbCleanupReadHandle(pDataReader); return NULL; } - + + SArray* groupKyes = extractPartitionColInfo(pTableScanNode->pPartitionKeys); + code = generateGroupIdMap(pTableListInfo, pHandle, groupKyes); //todo for json + if (code){ + tsdbCleanupReadHandle(pDataReader); + return NULL; + } + SOperatorInfo* pOperator = - createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, groupKyes, pTaskInfo); + createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 132f93a6a5..0f9879ecc9 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -37,7 +37,7 @@ static void destroyGroupOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->pGroupColVals); } -int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { +static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { *pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); if ((*pGroupColVals) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -118,7 +118,7 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo return true; } -void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex) { +static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex) { SColumnDataAgg* pColAgg = NULL; size_t numOfGroupCols = taosArrayGetSize(pGroupCols); @@ -150,7 +150,7 @@ void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* } } -int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { +static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { ASSERT(pKey != NULL); size_t numOfGroupCols = taosArrayGetSize(pGroupColVals); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8fb4878cd9..b6ee9843b4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -391,22 +391,16 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { longjmp(pOperator->pTaskInfo->env, code); } - recordNewGroupKeys(pTableScanInfo->pGroupCols, pTableScanInfo->pGroupColVals, pBlock, 0); - int32_t len = buildGroupKeys(pTableScanInfo->keyBuf, pTableScanInfo->pGroupColVals); - - uint64_t* groupId = taosHashGet(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len); - if (groupId) { - pBlock->info.groupId = *groupId; - } else if (len != 0) { - pBlock->info.groupId = calcGroupId(pTableScanInfo->keyBuf, len); - taosHashPut(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len, &pBlock->info.groupId, sizeof(uint64_t)); - } - // current block is filter out according to filter condition, continue load the next block if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) { continue; } + uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); + if (groupId) { + pBlock->info.groupId = *groupId; + } + pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows; pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; @@ -530,21 +524,13 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) { tsdbCleanupReadHandle(pTableScanInfo->dataReader); - taosArrayDestroy(pTableScanInfo->pGroupCols); - for (int i = 0; i < taosArrayGetSize(pTableScanInfo->pGroupColVals); i++) { - SGroupKeys key = *(SGroupKeys*)taosArrayGet(pTableScanInfo->pGroupColVals, i); - taosMemoryFree(key.pData); - } - taosArrayDestroy(pTableScanInfo->pGroupColVals); - taosMemoryFree(pTableScanInfo->keyBuf); - taosHashCleanup(pTableScanInfo->pGroupSet); if (pTableScanInfo->pColMatchInfo != NULL) { taosArrayDestroy(pTableScanInfo->pColMatchInfo); } } SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, - SReadHandle* readHandle, SArray* groupKyes, SExecTaskInfo* pTaskInfo) { + SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -591,18 +577,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pOperator->numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; - // for table group - pInfo->pGroupCols = groupKyes; - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pInfo->pGroupSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); - if (pInfo->pGroupSet == NULL) { - goto _error; - } - code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, groupKyes); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, NULL, NULL, getTableScannerExecInfo); @@ -992,7 +966,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan SScanPhysiNode* pScanPhyNode = &pTableScanNode->scan; SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; - SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, NULL, pTaskInfo); + SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info; From 2488382dd6de0f5c45a7c7df99dee4019cd6c730 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 13 Jun 2022 20:49:54 +0800 Subject: [PATCH 21/78] fix: avoid invalid value --- source/libs/index/src/indexFilter.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 1cc2679b06..90aafb1097 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -183,6 +183,7 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { memcpy(param->colName, r->literal, strlen(r->literal)); // sprintf(param->colName, "%s_%s", l->colName, r->literal); param->colValType = r->typeData; + param->status = SFLT_COARSE_INDEX; return 0; // memcpy(param->colName, l->colName, sizeof(l->colName)); } @@ -254,11 +255,13 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx return code; } SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); + if (NULL == paramList) { SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR) { + if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR && + (((SOperatorNode *)(node->pLeft))->opType == OP_TYPE_JSON_GET_VALUE)) { SNode *interNode = (node->pLeft); SIF_ERR_JRET(sifInitJsonParam(interNode, ¶mList[0], ctx)); if (nParam > 1) { @@ -525,8 +528,8 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { if (node->opType == OP_TYPE_JSON_GET_VALUE) { return code; } - SIFParam *params = NULL; + SIFParam *params = NULL; SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); if (params[0].status == SFLT_NOT_INDEX || (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { From 45ef3db78abd908abbc57a7bfbc56871381413f2 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 20:53:42 +0800 Subject: [PATCH 22/78] add first function partial/merge --- source/libs/function/inc/builtinsimpl.h | 2 + source/libs/function/src/builtins.c | 4 +- source/libs/function/src/builtinsimpl.c | 82 +++++++++++++++++++++---- 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index f3bcad8ede..af9b8d4a10 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -93,8 +93,10 @@ int32_t diffFunction(SqlFunctionCtx *pCtx); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); +int32_t firstFunctionMerge(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx); int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getFirstLastInfoSize(int32_t resBytes); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a2849d601e..180ffd1848 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1749,7 +1749,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = firstFunction, - .finalizeFunc = firstLastFinalize, + .finalizeFunc = firstLastPartialFinalize, .combineFunc = firstCombine, }, { @@ -1759,7 +1759,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateFirstLastMerge, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, - .processFunc = firstFunction, + .processFunc = firstFunctionMerge, .finalizeFunc = firstLastFinalize, .combineFunc = firstCombine, }, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 44418685b0..1bb27780bc 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -72,6 +72,12 @@ typedef struct STopBotRes { STopBotResItem* pItems; } STopBotRes; +typedef struct SFirstLastRes { + bool hasResult; + int32_t bytes; + char buf[]; +} SFirstLastRes; + typedef struct SStddevRes { double result; int64_t count; @@ -2245,12 +2251,12 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) } int32_t getFirstLastInfoSize(int32_t resBytes) { - return resBytes + sizeof(int64_t); + return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t); } bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); - pEnv->calcMemSize = pNode->node.resType.bytes + sizeof(int64_t); + pEnv->calcMemSize = sizeof(SFirstLastRes) + pNode->node.resType.bytes + sizeof(int64_t); return true; } @@ -2274,12 +2280,13 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); + SFirstLastRes *pInfo = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; int32_t bytes = pInputCol->info.bytes; + pInfo->bytes = bytes; // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { @@ -2297,7 +2304,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (blockDataOrder == TSDB_ORDER_ASC) { // filter according to current result firstly if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(buf + bytes); + TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); if (ts < startKey) { return TSDB_CODE_SUCCESS; } @@ -2313,9 +2320,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) > cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; + pInfo->hasResult = true; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; @@ -2326,7 +2334,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { // in case of descending order time stamp serial, which usually happens as the results of the nest query, // all data needs to be check. if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(buf + bytes); + TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); if (ts < endKey) { return TSDB_CODE_SUCCESS; } @@ -2342,9 +2350,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) > cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; + pInfo->hasResult = true; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; break; @@ -2422,6 +2431,39 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } +static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput) { + pOutput->bytes = pInput->bytes; + TSKEY *tsIn = (TSKEY*)(pInput->buf + pInput->bytes); + TSKEY *tsOut = (TSKEY*)(pOutput->buf + pInput->bytes); + if (pOutput->hasResult) { + if (*tsIn > *tsOut) { + return; + } + } + *tsOut = *tsIn; + memcpy(pOutput->buf, pInput->buf, pOutput->bytes); + pOutput->hasResult = true; + return; +} + +int32_t firstFunctionMerge(SqlFunctionCtx *pCtx) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + + SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + int32_t start = pInput->startRowIndex; + char* data = colDataGetData(pCol, start); + SFirstLastRes* pInputInfo = (SFirstLastRes *)varDataVal(data); + + firstLastTransferInfo(pInputInfo, pInfo); + + SET_VAL(GET_RES_INFO(pCtx), 1, 1); + + return TSDB_CODE_SUCCESS; +} + int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); @@ -2435,6 +2477,24 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); + SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t resultBytes = getFirstLastInfoSize(pRes->bytes); + char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + + memcpy(varDataVal(res), pRes, resultBytes); + varDataSetLen(res, resultBytes); + + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + colDataAppend(pCol, pBlock->info.rows, res, false); + + taosMemoryFree(res); + return 1; +} + int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); char* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); From 4240a38c1783804d4ee7aa0404fca0e305e8ccdc Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 20:56:33 +0800 Subject: [PATCH 23/78] fix bugs --- source/libs/function/src/builtinsimpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1bb27780bc..280f0217b9 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2471,8 +2471,8 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; - char* in = GET_ROWCELL_INTERBUF(pResInfo); - colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); + SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); + colDataAppend(pCol, pBlock->info.rows, pRes->buf, pResInfo->isNullRes); return pResInfo->numOfRes; } From 763e8bd6aef99264dc6608df5881738be11d9790 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 21:17:55 +0800 Subject: [PATCH 24/78] enable first function split --- source/libs/function/src/builtins.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 180ffd1848..3bfb60ec7c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -956,7 +956,7 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { // first(col_list) will be rewritten as first(col) - if (1 != LIST_LENGTH(pFunc->pParameterList)) { + if (2 != LIST_LENGTH(pFunc->pParameterList)) { //input has two params c0,ts, is this a bug? return TSDB_CODE_SUCCESS; } @@ -1739,6 +1739,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = firstFunction, .finalizeFunc = firstLastFinalize, + .pPartialFunc = "_first_partial", + .pMergeFunc = "_first_merge", .combineFunc = firstCombine, }, { From d4a38245a20d6296716200b253a7b8ec5a51abf3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 21:21:46 +0800 Subject: [PATCH 25/78] add some check --- source/libs/function/src/builtinsimpl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 280f0217b9..032333228a 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2432,6 +2432,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput) { + if (!pInput->hasResult) { + return; + } pOutput->bytes = pInput->bytes; TSKEY *tsIn = (TSKEY*)(pInput->buf + pInput->bytes); TSKEY *tsOut = (TSKEY*)(pOutput->buf + pInput->bytes); From 4481bb4e5cc4fdbe8aa7923124e5c552b5baafc5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 13 Jun 2022 21:29:00 +0800 Subject: [PATCH 26/78] fix: error in schemaless with same timestamp --- source/client/src/clientSml.c | 26 +++++++++++++------------- source/client/test/smlTest.cpp | 26 +++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index aaa13ed266..0acfaf00f2 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1297,8 +1297,8 @@ static int32_t smlKvTimeArrayCompare(const void* key1, const void* key2) { static int32_t smlKvTimeHashCompare(const void* key1, const void* key2) { SHashObj *s1 = *(SHashObj **)key1; SHashObj *s2 = *(SHashObj **)key2; - SSmlKv *kv1 = (SSmlKv *)taosHashGet(s1, TS, TS_LEN); - SSmlKv *kv2 = (SSmlKv *)taosHashGet(s2, TS, TS_LEN); + SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN); + SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN); ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); if (kv1->i < kv2->i) { @@ -1316,17 +1316,17 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col if(p == NULL){ taosArrayPush(oneTable->cols, &cols); }else{ // to make the sort stable for update data - SArray *sa = (SArray *)p; - SSmlKv *cur = (SSmlKv*)taosArrayGet(sa, 0); - SSmlKv *dCur = (SSmlKv*)taosArrayGet(cols, 0); + SArray *sa = *(SArray **)p; + SSmlKv *cur = (SSmlKv*)taosArrayGetP(sa, 0); + SSmlKv *dCur = (SSmlKv*)taosArrayGetP(cols, 0); if(cur->i > dCur->i){ taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); }else{ ASSERT(cur->i == dCur->i); int32_t index = TARRAY_ELEM_IDX(oneTable->cols, p) + 1; for(; index < taosArrayGetSize(oneTable->cols); index++){ - SArray *tmp = (SArray *)taosArrayGet(oneTable->cols, index); - SSmlKv *curTs = (SSmlKv*)taosArrayGet(tmp, 0); + SArray *tmp = (SArray *)taosArrayGetP(oneTable->cols, index); + SSmlKv *curTs = (SSmlKv*)taosArrayGetP(tmp, 0); if(curTs->i > dCur->i){ break; } @@ -1351,22 +1351,22 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col if(p == NULL){ taosArrayPush(oneTable->cols, &kvHash); }else{ // to make the sort stable for update data - SHashObj *sa = (SHashObj *)p; - SSmlKv *cur = (SSmlKv *)taosHashGet(sa, TS, TS_LEN); - SSmlKv *dCur = (SSmlKv*)taosArrayGet(cols, 0); + SHashObj *sa = *(SHashObj **)p; + SSmlKv *cur = *(SSmlKv **)taosHashGet(sa, TS, TS_LEN); + SSmlKv *dCur = (SSmlKv*)taosArrayGetP(cols, 0); if(cur->i > dCur->i){ taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); }else{ ASSERT(cur->i == dCur->i); int32_t index = TARRAY_ELEM_IDX(oneTable->cols, p) + 1; for(; index < taosArrayGetSize(oneTable->cols); index++){ - SHashObj *tmp = (SHashObj *)taosArrayGet(oneTable->cols, index); - SSmlKv *curTs = (SSmlKv *)taosHashGet(tmp, TS, TS_LEN); + SHashObj *tmp = (SHashObj *)taosArrayGetP(oneTable->cols, index); + SSmlKv *curTs = *(SSmlKv **)taosHashGet(tmp, TS, TS_LEN); if(curTs->i > dCur->i){ break; } } - taosArrayInsert(oneTable->cols, index, &cols); + taosArrayInsert(oneTable->cols, index, &kvHash); } } return TSDB_CODE_SUCCESS; diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 4ad73a6424..fffb03d9a5 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1260,4 +1260,28 @@ TEST(testCase, sml_16368_Test) { pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); ASSERT_EQ(taos_errno(pRes), 0); taos_free_result(pRes); -}*/ +} + +TEST(testCase, sml_dup_time_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists dup_time schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + //"test_ms,t0=t c0=f 1626006833641", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000" + }; + pRes = taos_query(taos, "use dup_time"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + ASSERT_EQ(taos_errno(pRes), 0); + taos_free_result(pRes); +} +*/ From 0111549155f08d1c03bd04674c9bda5290a09a4f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Jun 2022 08:41:52 +0800 Subject: [PATCH 27/78] index not exist as success --- source/libs/catalog/src/catalog.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6aa86aa4ba..a3576f8738 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -392,8 +392,12 @@ int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - int32_t code = 0; - CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); + int32_t code = ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL); + if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { + code = 0; + goto _return; + } + CTG_ERR_JRET(code); SArray* pInfo = NULL; CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); From e8ab28bee17e379634764012ecc091b67f3245ed Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Jun 2022 09:02:57 +0800 Subject: [PATCH 28/78] fix release crash issue --- source/libs/catalog/src/ctgCache.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 724b54d250..62890b8326 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -311,6 +311,9 @@ _return: ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_CACHE_STAT_INC(tbMetaMissNum, 1); + + *pDb = NULL; + *pTb = NULL; return TSDB_CODE_SUCCESS; } @@ -427,8 +430,6 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); - dbCache = NULL; - tbCache = NULL; ctgAcquireStbMetaFromCache(pCtg, dbFName, ctx->tbInfo.suid, &dbCache, &tbCache); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); From 877cb761f1b4fdc316078b47ffa0abbee1e17783 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 09:43:22 +0800 Subject: [PATCH 29/78] refactor: mnode worker --- include/dnode/mnode/mnode.h | 1 + source/dnode/mgmt/mgmt_mnode/inc/mmInt.h | 14 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 218 ++++++++++---------- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 5 +- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 88 ++++---- 5 files changed, 161 insertions(+), 165 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index ab090940f2..83183422c2 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -83,6 +83,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); */ int32_t mndProcessRpcMsg(SRpcMsg *pMsg); int32_t mndProcessSyncMsg(SRpcMsg *pMsg); +int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg); /** * @brief Generate machine code diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index 22691300bc..2443493d0b 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -53,20 +53,18 @@ void mmRelease(SMnodeMgmt *pMgmt); SArray *mmGetMsgHandles(); int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); -int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg); // mmWorker.c int32_t mmStartWorker(SMnodeMgmt *pMgmt); void mmStopWorker(SMnodeMgmt *pMgmt); -int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutRpcMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc); +int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index c47988e1ef..f151a09b94 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -20,11 +20,6 @@ void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *pInfo) { mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->grant); } -void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { - pInfo->isMnode = 1; - mndGetLoad(pMgmt->pMnode, &pInfo->load); -} - int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { SMonMmInfo mmInfo = {0}; mmGetMonitorInfo(pMgmt, &mmInfo); @@ -50,6 +45,11 @@ int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } +void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { + pInfo->isMnode = 1; + mndGetLoad(pMgmt->pMnode, &pInfo->load); +} + int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { SMonMloadInfo mloads = {0}; mmGetMnodeLoads(pMgmt, &mloads); @@ -129,114 +129,114 @@ SArray *mmGetMsgHandles() { SArray *pArray = taosArrayInit(64, sizeof(SMgmtHandle)); if (pArray == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_QNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_QNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_QNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_QNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_QNODE_LIST, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_BNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_BNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_META, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SMA, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SMA, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_TABLE_INDEX, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_QNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_QNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_QNODE_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_BNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_BNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_TABLE_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutMsgToMonitorQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY, mmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY, mmPutMsgToSyncQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 0f3c06cb3a..cb942e8d1c 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -43,7 +43,6 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - pOption->replica = 1; pOption->selfIndex = 0; @@ -54,8 +53,8 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu } static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { - pOption->deploy = false; pOption->standby = false; + pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; @@ -105,7 +104,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->path = pInput->path; pMgmt->name = pInput->name; pMgmt->msgCb = pInput->msgCb; - pMgmt->msgCb.putToQueueFp = (PutToQueueFp)mmPutRpcMsgToQueue; + pMgmt->msgCb.putToQueueFp = (PutToQueueFp)mmPutMsgToQueue; pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 53943b61b0..6da9bfc528 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -26,7 +26,7 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { tmsgSendRsp(&rsp); } -static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; dTrace("msg:%p, get from mnode queue", pMsg); @@ -53,11 +53,10 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { taosFreeQitem(pMsg); } -static void mmProcessSyncQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +static void mmProcessSyncMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; - dTrace("msg:%p, get from mnode-sync queue", pMsg); - pMsg->info.node = pMgmt->pMnode; + dTrace("msg:%p, get from mnode-sync queue", pMsg); SMsgHead *pHead = pMsg->pCont; pHead->contLen = ntohl(pHead->contLen); @@ -70,64 +69,63 @@ static void mmProcessSyncQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { taosFreeQitem(pMsg); } -static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { - dTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); - taosWriteQitem(pWorker->queue, pMsg); - return 0; +static inline int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pMsg) { + if (mmAcquire(pMgmt) == 0) { + dTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); + taosWriteQitem(pWorker->queue, pMsg); + mmRelease(pMgmt); + return 0; + } else { + dTrace("msg:%p, failed to put into %s queue since %s, type:%s", pMsg, pWorker->name, terrstr(), + TMSG_INFO(pMsg->msgType)); + return -1; + } } -int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg); +inline int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); } -int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg); +inline int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); } -int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg); +inline int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); } -int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - mndPreprocessQueryMsg(pMgmt->pMnode, pMsg); - - return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); +inline int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + if (mndPreprocessQueryMsg(pMgmt->pMnode, pMsg) != 0) { + dError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); + return -1; + } + return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); } -int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->monitorWorker, pMsg); +inline int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->monitorWorker, pMsg); } -int32_t mmPutRpcMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { +inline int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) return -1; - memcpy(pMsg, pRpc, sizeof(SRpcMsg)); + dTrace("msg:%p, is created", pMsg); + memcpy(pMsg, pRpc, sizeof(SRpcMsg)); switch (qtype) { case WRITE_QUEUE: - dTrace("msg:%p, is created and will put into vnode-write queue", pMsg); - taosWriteQitem(pMgmt->writeWorker.queue, pMsg); - return 0; + return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); case QUERY_QUEUE: - dTrace("msg:%p, is created and will put into vnode-query queue", pMsg); - taosWriteQitem(pMgmt->queryWorker.queue, pMsg); - return 0; - + return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); case READ_QUEUE: - dTrace("msg:%p, is created and will put into vnode-read queue", pMsg); - taosWriteQitem(pMgmt->readWorker.queue, pMsg); - return 0; + return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); case SYNC_QUEUE: - if (mmAcquire(pMgmt) == 0) { - dTrace("msg:%p, is created and will put into vnode-sync queue", pMsg); - taosWriteQitem(pMgmt->syncWorker.queue, pMsg); - mmRelease(pMgmt); - return 0; - } else { - return -1; - } + return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); default: terrno = TSDB_CODE_INVALID_PARA; + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pMsg->pCont); return -1; } } @@ -137,7 +135,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = tsNumOfMnodeQueryThreads, .max = tsNumOfMnodeQueryThreads, .name = "mnode-query", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) { @@ -149,7 +147,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = tsNumOfMnodeReadThreads, .max = tsNumOfMnodeReadThreads, .name = "mnode-read", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->readWorker, &rCfg) != 0) { @@ -161,7 +159,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "mnode-write", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->writeWorker, &wCfg) != 0) { @@ -173,7 +171,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "mnode-sync", - .fp = (FItem)mmProcessSyncQueue, + .fp = (FItem)mmProcessSyncMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) { @@ -185,7 +183,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "mnode-monitor", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { From a04065769564d50b2d43e3df31fb566e638af790 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 14 Jun 2022 09:45:15 +0800 Subject: [PATCH 30/78] test: reopen one case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 5149994228..b40d9a7fbb 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -103,7 +103,7 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py python3 ./test.py -f 7-tmq/subscribeDb0.py -# python3 ./test.py -f 7-tmq/subscribeDb1.py +python3 ./test.py -f 7-tmq/subscribeDb1.py python3 ./test.py -f 7-tmq/subscribeStb.py python3 ./test.py -f 7-tmq/subscribeStb0.py python3 ./test.py -f 7-tmq/subscribeStb1.py From 6888c1680c5c39515bf1f626a193f6af0757215b Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 14 Jun 2022 09:48:57 +0800 Subject: [PATCH 31/78] add test case for tail about CI --- tests/system-test/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 5149994228..cda0e37237 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -95,6 +95,7 @@ python3 ./test.py -f 2-query/unique.py python3 ./test.py -f 2-query/stateduration.py python3 ./test.py -f 2-query/function_stateduration.py python3 ./test.py -f 2-query/statecount.py +python3 ./test.py -f 2-query/tail.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py From 3887e940f14c12e77f556f2fd04654417e8652be Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 10:24:05 +0800 Subject: [PATCH 32/78] fix(sync): when receive snapshot rsp from a dropped replica --- source/libs/sync/src/syncMain.c | 21 ++++++++++++--------- source/libs/sync/src/syncSnapshot.c | 6 ++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 94bc755f4e..12d09ce26c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1968,8 +1968,9 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE bool isDrop; if (IamInNew || (!IamInNew && ths->state != TAOS_SYNC_STATE_LEADER)) { - syncNodeUpdateConfig(ths, &newSyncCfg, pEntry->index, &isDrop); + syncNodeUpdateConfig(ths, &newSyncCfg, pEntry->index, &isDrop); + // change isStandBy to normal if (!isDrop) { if (ths->state == TAOS_SYNC_STATE_LEADER) { @@ -1978,14 +1979,16 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE syncNodeBecomeFollower(ths, "config change"); } } - - if (gRaftDetailLog) { - char* sOld = syncCfg2Str(&oldSyncCfg); - char* sNew = syncCfg2Str(&newSyncCfg); - sInfo("==config change== 0x11 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); - taosMemoryFree(sOld); - taosMemoryFree(sNew); - } + } else { + syncNodeBecomeFollower(ths, "config change"); + } + + if (gRaftDetailLog) { + char* sOld = syncCfg2Str(&oldSyncCfg); + char* sNew = syncCfg2Str(&newSyncCfg); + sInfo("==config change== 0x11 old:%s new:%s isDrop:%d index:%ld IamInNew:%d \n", sOld, sNew, isDrop, pEntry->index, IamInNew); + taosMemoryFree(sOld); + taosMemoryFree(sNew); } // always call FpReConfigCb diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 38742220fe..9c43c36af3 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -755,6 +755,12 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // sender receives ack, set seq = ack + 1, send msg from seq // if ack == SYNC_SNAPSHOT_SEQ_END, stop sender int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { + // if already drop replica, do not process + if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + sInfo("recv SyncSnapshotRsp maybe replica already dropped"); + return 0; + } + // get sender SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, &(pMsg->srcId)); ASSERT(pSender != NULL); From 01521a2541af1aa7cd62eb790163ce89407f5807 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 10:41:15 +0800 Subject: [PATCH 33/78] refactor: mnode file --- source/dnode/mgmt/mgmt_mnode/inc/mmInt.h | 6 +- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 84 ++++++++------------- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 24 +++--- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 8 +- 5 files changed, 47 insertions(+), 77 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index 2443493d0b..5dd1a5b3b8 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -34,16 +34,14 @@ typedef struct SMnodeMgmt { SSingleWorker writeWorker; SSingleWorker syncWorker; SSingleWorker monitorWorker; - SReplica replicas[TSDB_MAX_REPLICA]; - int8_t replica; bool stopped; int32_t refCount; TdThreadRwlock lock; } SMnodeMgmt; // mmFile.c -int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed); -int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pMsg, bool deployed); +int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed); +int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed); // mmInt.c int32_t mmAcquire(SMnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 478d6abd52..e24ebcb9e2 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "mmInt.h" -int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { +int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; int32_t len = 0; int32_t maxLen = 4096; @@ -52,45 +52,36 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { } *pDeployed = deployed->valueint; - cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes"); - if (mnodes != NULL) { - if (!mnodes || mnodes->type != cJSON_Array) { - dError("failed to read %s since nodes not found", file); + cJSON *id = cJSON_GetObjectItem(root, "id"); + if (id) { + if (id->type != cJSON_Number) { + dError("failed to read %s since id not found", file); goto _OVER; } - - pMgmt->replica = cJSON_GetArraySize(mnodes); - if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) { - dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica); - goto _OVER; - } - - for (int32_t i = 0; i < pMgmt->replica; ++i) { - cJSON *node = cJSON_GetArrayItem(mnodes, i); - if (node == NULL) break; - - SReplica *pReplica = &pMgmt->replicas[i]; - - cJSON *id = cJSON_GetObjectItem(node, "id"); - if (!id || id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); - goto _OVER; - } + if (pReplica) { pReplica->id = id->valueint; + } + } - cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn"); - if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); - goto _OVER; - } + cJSON *fqdn = cJSON_GetObjectItem(root, "fqdn"); + if (fqdn) { + if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { + dError("failed to read %s since fqdn not found", file); + goto _OVER; + } + if (pReplica) { tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); + } + } - cJSON *port = cJSON_GetObjectItem(node, "port"); - if (!port || port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); - goto _OVER; - } - pReplica->port = port->valueint; + cJSON *port = cJSON_GetObjectItem(root, "port"); + if (port) { + if (port->type != cJSON_Number) { + dError("failed to read %s since port not found", file); + goto _OVER; + } + if (pReplica) { + pReplica->port = (uint16_t)port->valueint; } } @@ -106,7 +97,7 @@ _OVER: return code; } -int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pMsg, bool deployed) { +int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) { char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP); @@ -124,26 +115,11 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pMsg, bool deployed) { char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - - int8_t replica = (pMsg != NULL ? pMsg->replica : pMgmt->replica); - if (replica > 0) { - len += snprintf(content + len, maxLen - len, " \"mnodes\": [{\n"); - for (int32_t i = 0; i < replica; ++i) { - SReplica *pReplica = &pMgmt->replicas[i]; - if (pMsg != NULL) { - pReplica = &pMsg->replicas[i]; - } - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); - if (i < replica - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }],\n"); - } - } + if (pReplica != NULL && pReplica->id > 0) { + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u\n,", pReplica->port); } - len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); len += snprintf(content + len, maxLen - len, "}\n"); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index f151a09b94..124876d718 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -90,7 +90,7 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SMnodeMgmt mgmt = {0}; mgmt.path = pInput->path; mgmt.name = pInput->name; - if (mmWriteFile(&mgmt, &createReq, deployed) != 0) { + if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index cb942e8d1c..7e73e53481 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -27,7 +27,7 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) { static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { SMnodeMgmt mgmt = {0}; mgmt.path = pInput->path; - if (mmReadFile(&mgmt, required) != 0) { + if (mmReadFile(&mgmt, NULL, required) != 0) { return -1; } @@ -52,23 +52,17 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu tstrncpy(pReplica->fqdn, tsLocalFqdn, TSDB_FQDN_LEN); } -static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { +static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) { pOption->standby = false; pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - if (pMgmt->replica > 0) { + if (pReplica->id > 0) { pOption->standby = true; pOption->replica = 1; pOption->selfIndex = 0; - SReplica *pReplica = &pOption->replicas[0]; - for (int32_t i = 0; i < pMgmt->replica; ++i) { - if (pMgmt->replicas[i].id != pMgmt->pData->dnodeId) continue; - pReplica->id = pMgmt->replicas[i].id; - pReplica->port = pMgmt->replicas[i].port; - memcpy(pReplica->fqdn, pMgmt->replicas[i].fqdn, TSDB_FQDN_LEN); - } + memcpy(&pOption->replicas[0], pReplica, sizeof(SReplica)); } } @@ -108,8 +102,9 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); - bool deployed = false; - if (mmReadFile(pMgmt, &deployed) != 0) { + bool deployed = false; + SReplica replica = {0}; + if (mmReadFile(pMgmt, &replica, &deployed) != 0) { dError("failed to read file since %s", terrstr()); mmClose(pMgmt); return -1; @@ -122,7 +117,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { mmBuildOptionForDeploy(pMgmt, pInput, &option); } else { dInfo("mnode start to open"); - mmBuildOptionForOpen(pMgmt, &option); + mmBuildOptionForOpen(pMgmt, &replica, &option); } pMgmt->pMnode = mndOpen(pMgmt->path, &option); @@ -140,8 +135,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } tmsgReportStartup("mnode-worker", "initialized"); - if (!deployed || pMgmt->replica > 0) { - pMgmt->replica = 0; + if (!deployed || replica.id > 0) { deployed = true; if (mmWriteFile(pMgmt, NULL, deployed) != 0) { dError("failed to write mnode file since %s", terrstr()); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 528beb280b..498af36786 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -123,10 +123,12 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); } else { dInfo("node:%s, has been created", pWrapper->name); - (void)dmOpenNode(pWrapper); - (void)dmStartNode(pWrapper); - pWrapper->required = true; + code = dmOpenNode(pWrapper); + if (code != 0) { + code = dmStartNode(pWrapper); + } pWrapper->deployed = true; + pWrapper->required = true; pWrapper->proc.ptype = pDnode->ptype; } From b5bc9aa2bae4dc9df60577b0b6d0cacec53f0107 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Jun 2022 10:44:34 +0800 Subject: [PATCH 34/78] fix rpc retry error --- source/libs/transport/src/transCli.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d1debc6af6..4852dcfb54 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -983,6 +983,9 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { SEpSet epSet = {0}; tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); pCtx->epSet = epSet; + if (!transEpSetIsEqual(&epSet, &pCtx->epSet)) { + pCtx->retryCount = 0; + } } addConnToPool(pThrd->pool, pConn); tTrace("use remote epset, current in use: %d, retry count:%d, try limit: %d", pEpSet->inUse, pCtx->retryCount + 1, From 589b30422f180cc990a371cc4d0fdd36843d26d4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 10:51:00 +0800 Subject: [PATCH 35/78] add last function splitting --- include/libs/function/functionMgt.h | 2 ++ source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 22 ++++++++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index e787639897..b5641f2d07 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -140,6 +140,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_BOTTOM_MERGE, FUNCTION_TYPE_FIRST_PARTIAL, FUNCTION_TYPE_FIRST_MERGE, + FUNCTION_TYPE_LAST_PARTIAL, + FUNCTION_TYPE_LAST_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index af9b8d4a10..7db1396603 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -95,6 +95,7 @@ bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); int32_t firstFunctionMerge(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx); +int32_t lastFunctionMerge(SqlFunctionCtx *pCtx); int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 0a6d99c52d..7a06304487 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1776,6 +1776,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = firstLastFinalize, .combineFunc = lastCombine, }, + { + .name = "_last_partial", + .type = FUNCTION_TYPE_LAST_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastPartial, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunction, + .finalizeFunc = firstLastPartialFinalize, + .combineFunc = lastCombine, + }, + { + .name = "_last_merge", + .type = FUNCTION_TYPE_LAST_MERGE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastMerge, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunctionMerge, + .finalizeFunc = firstLastFinalize, + .combineFunc = lastCombine, + }, { .name = "twa", .type = FUNCTION_TYPE_TWA, From a3f12736407af1bbe8ed75435fde928b70167ef9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 10:58:36 +0800 Subject: [PATCH 36/78] refactor: mnode sync --- include/dnode/mnode/mnode.h | 4 +--- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 4 +++- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 15 ++++----------- source/dnode/mnode/impl/inc/mndInt.h | 14 ++++++-------- source/dnode/mnode/impl/src/mndDnode.c | 4 ++-- source/dnode/mnode/impl/src/mndMain.c | 4 +--- source/dnode/mnode/impl/src/mndSync.c | 18 +++++++++--------- 8 files changed, 27 insertions(+), 38 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 83183422c2..99b704826d 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -32,9 +32,7 @@ typedef struct { int32_t dnodeId; bool standby; bool deploy; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; + SReplica replica; SMsgCb msgCb; } SMnodeOpt; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index e24ebcb9e2..27a35ae17a 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -86,12 +86,14 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { } code = 0; - dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); _OVER: if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); + if (code == 0) { + dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + } terrno = code; return code; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 124876d718..e91f4b8cf4 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -126,7 +126,7 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SArray *mmGetMsgHandles() { int32_t code = -1; - SArray *pArray = taosArrayInit(64, sizeof(SMgmtHandle)); + SArray *pArray = taosArrayInit(128, sizeof(SMgmtHandle)); if (pArray == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 7e73e53481..bc67b8442e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -43,13 +43,9 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - pOption->replica = 1; - pOption->selfIndex = 0; - - SReplica *pReplica = &pOption->replicas[0]; - pReplica->id = 1; - pReplica->port = tsServerPort; - tstrncpy(pReplica->fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pOption->replica.id = 1; + pOption->replica.port = tsServerPort; + tstrncpy(pOption->replica.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); } static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) { @@ -57,12 +53,9 @@ static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SM pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - if (pReplica->id > 0) { pOption->standby = true; - pOption->replica = 1; - pOption->selfIndex = 0; - memcpy(&pOption->replicas[0], pReplica, sizeof(SReplica)); + pOption->replica = *pReplica; } } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 4869a19856..16220f101a 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -76,11 +76,12 @@ typedef struct { } STelemMgmt; typedef struct { - sem_t syncSem; - int64_t sync; - bool standby; - int32_t errCode; - int32_t transId; + sem_t syncSem; + int64_t sync; + bool standby; + SReplica replica; + int32_t errCode; + int32_t transId; } SSyncMgmt; typedef struct { @@ -98,9 +99,6 @@ typedef struct SMnode { bool stopped; bool restored; bool deploy; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; char *path; int64_t checkTime; SSdb *pSdb; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 73eea70195..c936c0f93d 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -95,8 +95,8 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { dnodeObj.id = 1; dnodeObj.createdTime = taosGetTimestampMs(); dnodeObj.updateTime = dnodeObj.createdTime; - dnodeObj.port = pMnode->replicas[0].port; - memcpy(&dnodeObj.fqdn, pMnode->replicas[0].fqdn, TSDB_FQDN_LEN); + dnodeObj.port = tsServerPort; + memcpy(&dnodeObj.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 7a73d3c360..892650fa18 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -289,11 +289,9 @@ static int32_t mndExecSteps(SMnode *pMnode) { } static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { - pMnode->replica = pOption->replica; - pMnode->selfIndex = pOption->selfIndex; - memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); pMnode->msgCb = pOption->msgCb; pMnode->selfDnodeId = pOption->dnodeId; + pMnode->syncMgmt.replica = pOption->replica; pMnode->syncMgmt.standby = pOption->standby; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index e0b4cc6a57..e58012b1b7 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -188,15 +188,15 @@ int32_t mndInitSync(SMnode *pMnode) { syncInfo.isStandBy = pMgmt->standby; syncInfo.snapshotEnable = true; - SSyncCfg *pCfg = &syncInfo.syncCfg; - pCfg->replicaNum = pMnode->replica; - pCfg->myIndex = pMnode->selfIndex; - mInfo("start to open mnode sync, replica:%d myindex:%d standby:%d", pCfg->replicaNum, pCfg->myIndex, pMgmt->standby); - for (int32_t i = 0; i < pMnode->replica; ++i) { - SNodeInfo *pNode = &pCfg->nodeInfo[i]; - tstrncpy(pNode->nodeFqdn, pMnode->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); - pNode->nodePort = pMnode->replicas[i].port; - mInfo("index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); + mInfo("start to open mnode sync, standby:%d", pMgmt->standby); + if (pMgmt->standby || pMgmt->replica.id > 0) { + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->replicaNum = 1; + pCfg->myIndex = 0; + SNodeInfo *pNode = &pCfg->nodeInfo[0]; + tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = pMgmt->replica.port; + mInfo("fqdn:%s port:%u", pNode->nodeFqdn, pNode->nodePort); } tsem_init(&pMgmt->syncSem, 0, 0); From 6a4fc4577126119b4ffc3d2f0285862a8b656647 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 11:00:22 +0800 Subject: [PATCH 37/78] fix bugs --- source/libs/function/src/builtinsimpl.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index f2c5df8fc2..fe52e9c7c5 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2355,12 +2355,13 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); + SFirstLastRes *pInfo = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; int32_t bytes = pInputCol->info.bytes; + pInfo->bytes = bytes; // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { @@ -2385,9 +2386,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; } @@ -2403,9 +2404,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; pResInfo->numOfRes = 1; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); } From 6f27ee975e758cf4af935b0d554c740412f9c2b7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 11:04:36 +0800 Subject: [PATCH 38/78] enable last splitting --- source/libs/function/src/builtins.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 7a06304487..1c3f274ee9 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1774,6 +1774,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = lastFunction, .finalizeFunc = firstLastFinalize, + .pPartialFunc = "_last_partial", + .pMergeFunc = "_last_merge", .combineFunc = lastCombine, }, { From d93f4c4a83d906e729cbf8fc344cdfa4c5b14254 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:10:36 +0800 Subject: [PATCH 39/78] refactor: mnode worker --- source/dnode/mgmt/mgmt_mnode/inc/mmInt.h | 4 -- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 19 -------- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 48 +++++++++++++++------ 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index 5dd1a5b3b8..c5c3d76f1e 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -43,10 +43,6 @@ typedef struct SMnodeMgmt { int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed); int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed); -// mmInt.c -int32_t mmAcquire(SMnodeMgmt *pMgmt); -void mmRelease(SMnodeMgmt *pMgmt); - // mmHandle.c SArray *mmGetMsgHandles(); int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index bc67b8442e..b7124dfaa5 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -164,22 +164,3 @@ SMgmtFunc mmGetMgmtFunc() { return mgmtFunc; } - -int32_t mmAcquire(SMnodeMgmt *pMgmt) { - int32_t code = 0; - - taosThreadRwlockRdlock(&pMgmt->lock); - if (pMgmt->stopped) { - code = -1; - } else { - atomic_add_fetch_32(&pMgmt->refCount, 1); - } - taosThreadRwlockUnlock(&pMgmt->lock); - return code; -} - -void mmRelease(SMnodeMgmt *pMgmt) { - taosThreadRwlockRdlock(&pMgmt->lock); - atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosThreadRwlockUnlock(&pMgmt->lock); -} \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 6da9bfc528..17581fcef7 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -16,6 +16,25 @@ #define _DEFAULT_SOURCE #include "mmInt.h" +static inline int32_t mmAcquire(SMnodeMgmt *pMgmt) { + int32_t code = 0; + + taosThreadRwlockRdlock(&pMgmt->lock); + if (pMgmt->stopped) { + code = -1; + } else { + atomic_add_fetch_32(&pMgmt->refCount, 1); + } + taosThreadRwlockUnlock(&pMgmt->lock); + return code; +} + +static inline void mmRelease(SMnodeMgmt *pMgmt) { + taosThreadRwlockRdlock(&pMgmt->lock); + atomic_sub_fetch_32(&pMgmt->refCount, 1); + taosThreadRwlockUnlock(&pMgmt->lock); +} + static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, @@ -107,27 +126,30 @@ inline int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { } inline int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - if (pMsg == NULL) return -1; - dTrace("msg:%p, is created", pMsg); - - memcpy(pMsg, pRpc, sizeof(SRpcMsg)); + SSingleWorker *pWorker = NULL; switch (qtype) { case WRITE_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); + pWorker = &pMgmt->writeWorker; + break; case QUERY_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); + pWorker = &pMgmt->queryWorker; + break; case READ_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); + pWorker = &pMgmt->readWorker; + break; case SYNC_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); + pWorker = &pMgmt->syncWorker; + break; default: terrno = TSDB_CODE_INVALID_PARA; - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pMsg->pCont); - return -1; } + + if (pWorker == NULL) return -1; + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); + if (pMsg == NULL) return -1; + dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name); + + return mmPutMsgToWorker(pMgmt, pWorker, pMsg); } int32_t mmStartWorker(SMnodeMgmt *pMgmt) { From 22f86ee52978c26d91ff137c1b552f7a4d7fa389 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 14 Jun 2022 11:15:10 +0800 Subject: [PATCH 40/78] feat(stream): partition by --- source/libs/executor/src/groupoperator.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 132f93a6a5..f663f5798a 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -582,6 +582,15 @@ int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { return offset; } +static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { + void *ite = NULL; + while( (ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL ) { + taosArrayDestroy( ((SDataGroupInfo *)ite)->pPageList); + } + taosHashClear(pInfo->pGroupSet); + clearDiskbasedBuf(pInfo->pBuf); +} + static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SPartitionOperatorInfo* pInfo = pOperator->info; @@ -591,6 +600,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { pInfo->pGroupIter = taosHashIterate(pInfo->pGroupSet, pInfo->pGroupIter); if (pInfo->pGroupIter == NULL) { doSetOperatorCompleted(pOperator); + clearPartitionOperator(pInfo); return NULL; } From 63927d15fab259f84e78e710ce1011fb3aff8ad6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:15:17 +0800 Subject: [PATCH 41/78] fix: compile err in windows --- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 17581fcef7..f02f90962b 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -101,19 +101,19 @@ static inline int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker } } -inline int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); } -inline int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); } -inline int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); } -inline int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { if (mndPreprocessQueryMsg(pMgmt->pMnode, pMsg) != 0) { dError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); return -1; @@ -121,11 +121,11 @@ inline int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); } -inline int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->monitorWorker, pMsg); } -inline int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { +int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SSingleWorker *pWorker = NULL; switch (qtype) { case WRITE_QUEUE: From 313d2f0e4dbcd7d647d9a8bbd511e2151f9c08bc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:18:30 +0800 Subject: [PATCH 42/78] test: enable mnode/basic3.sim --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1c694bf7fb..db8a055362 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim -#./test.sh -f tsim/mnode/basic3.sim +./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim From 5ffb01b2e69a2641f2ca51e28d6bea339ad58ac2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 14 Jun 2022 11:24:45 +0800 Subject: [PATCH 43/78] fix: error in schemaless with same timestamp --- source/client/src/clientSml.c | 45 +--- source/libs/parser/src/parInsertData.c | 4 + source/util/src/talgo.c | 12 +- source/util/test/taosbsearchTest.cpp | 328 +++++++++++++++++++++++++ 4 files changed, 350 insertions(+), 39 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 0acfaf00f2..c10311acee 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1312,27 +1312,11 @@ static int32_t smlKvTimeHashCompare(const void* key1, const void* key2) { static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){ if(dataFormat){ - void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GE); + void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT); if(p == NULL){ taosArrayPush(oneTable->cols, &cols); - }else{ // to make the sort stable for update data - SArray *sa = *(SArray **)p; - SSmlKv *cur = (SSmlKv*)taosArrayGetP(sa, 0); - SSmlKv *dCur = (SSmlKv*)taosArrayGetP(cols, 0); - if(cur->i > dCur->i){ - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); - }else{ - ASSERT(cur->i == dCur->i); - int32_t index = TARRAY_ELEM_IDX(oneTable->cols, p) + 1; - for(; index < taosArrayGetSize(oneTable->cols); index++){ - SArray *tmp = (SArray *)taosArrayGetP(oneTable->cols, index); - SSmlKv *curTs = (SSmlKv*)taosArrayGetP(tmp, 0); - if(curTs->i > dCur->i){ - break; - } - } - taosArrayInsert(oneTable->cols, index, &cols); - } + }else{ + taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); } return TSDB_CODE_SUCCESS; } @@ -1347,29 +1331,14 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } - void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GE); + void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT); if(p == NULL){ taosArrayPush(oneTable->cols, &kvHash); - }else{ // to make the sort stable for update data - SHashObj *sa = *(SHashObj **)p; - SSmlKv *cur = *(SSmlKv **)taosHashGet(sa, TS, TS_LEN); - SSmlKv *dCur = (SSmlKv*)taosArrayGetP(cols, 0); - if(cur->i > dCur->i){ - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); - }else{ - ASSERT(cur->i == dCur->i); - int32_t index = TARRAY_ELEM_IDX(oneTable->cols, p) + 1; - for(; index < taosArrayGetSize(oneTable->cols); index++){ - SHashObj *tmp = (SHashObj *)taosArrayGetP(oneTable->cols, index); - SSmlKv *curTs = *(SSmlKv **)taosHashGet(tmp, TS, TS_LEN); - if(curTs->i > dCur->i){ - break; - } - } - taosArrayInsert(oneTable->cols, index, &kvHash); - } + }else{ + taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash); } return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } static SSmlSTableMeta* smlBuildSTableMeta(){ diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 1960073f29..2b505d4bf7 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -283,6 +283,8 @@ void sortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf) { if (!dataBuf->ordered) { char* pBlockData = pBlocks->data; + + // todo. qsort is unstable, if timestamp is same, should get the last one qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); int32_t i = 0; @@ -350,6 +352,8 @@ int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKey if (!dataBuf->ordered) { pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + + // todo. qsort is unstable, if timestamp is same, should get the last one qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar); pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index 866ac5757b..5353cd9bfe 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -175,7 +175,13 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size c = compar(key, p); if (c == 0) { - break; + if (flags == TD_GT){ + lidx = midx + 1; + } else if(flags == TD_LT){ + ridx = midx - 1; + }else{ + break; + } } else if (c < 0) { ridx = midx - 1; } else { @@ -189,6 +195,10 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size return (c <= 0) ? p : (midx + 1 < nmemb ? p + size : NULL); } else if (flags == TD_LE) { return (c >= 0) ? p : (midx > 0 ? p - size : NULL); + } else if (flags == TD_GT) { + return (c < 0) ? p : (midx + 1 < nmemb ? p + size : NULL); + } else if (flags == TD_LT) { + return (c > 0) ? p : (midx > 0 ? p - size : NULL); } else { ASSERT(0); } diff --git a/source/util/test/taosbsearchTest.cpp b/source/util/test/taosbsearchTest.cpp index 0b250c9ecc..b8fa3a4476 100644 --- a/source/util/test/taosbsearchTest.cpp +++ b/source/util/test/taosbsearchTest.cpp @@ -248,6 +248,170 @@ TEST(testCase, taosbsearch_greater_or_equal) { ASSERT_EQ(pRet, nullptr); } +TEST(testCase, taosbsearch_greater) { + // For equal test + int key = 3; + void *pRet = NULL; + + pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + // 1 element + int array1[1] = {5}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 5); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + key = 5; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + // 2 element + int array2[2] = {3, 6}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + // 3 element + int array3[3] = {3, 6, 8}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + // 4 element + int array4[4] = {3, 6, 8, 11}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + // 5 element + int array5[5] = {3, 6, 8, 11, 15}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 15); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 15); + + key = 15; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + key = 17; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); +} + TEST(testCase, taosbsearch_less_or_equal) { // For equal test int key = 3; @@ -411,4 +575,168 @@ TEST(testCase, taosbsearch_less_or_equal) { key = 17; pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); ASSERT_EQ(*(int *)pRet, 15); +} + +TEST(testCase, taosbsearch_less) { + // For equal test + int key = 3; + void *pRet = NULL; + + pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + // 1 element + int array1[1] = {5}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 5); + + key = 5; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + // 2 element + int array2[2] = {3, 6}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + // 3 element + int array3[3] = {3, 6, 8}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 8); + + // 4 element + int array4[4] = {3, 6, 8, 11}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 11); + + // 5 element + int array5[5] = {3, 6, 8, 11, 15}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 15; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 17; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 15); } \ No newline at end of file From 0dde9cc9f95c9d6aebabc8a3b4b2feb2be12ffc3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 11:25:11 +0800 Subject: [PATCH 44/78] fix bugs --- source/libs/function/src/builtinsimpl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fe52e9c7c5..b5009b7d41 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2390,6 +2390,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); + pInfo->hasResult = true; pResInfo->numOfRes = 1; } break; @@ -2407,6 +2408,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; + pInfo->hasResult = true; pResInfo->numOfRes = 1; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); } From 4e7aac8171a99b41c3c020e54046bbe95dc1d0d0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:34:51 +0800 Subject: [PATCH 45/78] fix: crash in mnode --- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index f02f90962b..ee2df5f089 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -147,8 +147,9 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { if (pWorker == NULL) return -1; SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) return -1; - dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name); + memcpy(pMsg, pRpc, sizeof(SRpcMsg)); + dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name); return mmPutMsgToWorker(pMgmt, pWorker, pMsg); } From d44c6a810bb288839d18fb68d88070d807a2a2ed Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 14 Jun 2022 11:54:55 +0800 Subject: [PATCH 46/78] docs: refine keywords section and add delete keywords --- docs/en/12-taos-sql/12-keywords.md | 311 +++++++++++++++++++++++++---- docs/zh/12-taos-sql/12-keywords.md | 310 ++++++++++++++++++++++++---- 2 files changed, 536 insertions(+), 85 deletions(-) diff --git a/docs/en/12-taos-sql/12-keywords.md b/docs/en/12-taos-sql/12-keywords.md index ed0c96b4e4..6d40deb5a6 100644 --- a/docs/en/12-taos-sql/12-keywords.md +++ b/docs/en/12-taos-sql/12-keywords.md @@ -4,50 +4,275 @@ title: Keywords There are about 200 keywords reserved by TDengine, they can't be used as the name of database, STable or table with either upper case, lower case or mixed case. -**Keywords List** +## Keywords List -| | | | | | -| ----------- | ---------- | --------- | ---------- | ------------ | -| ABORT | CREATE | IGNORE | NULL | STAR | -| ACCOUNT | CTIME | IMMEDIATE | OF | STATE | -| ACCOUNTS | DATABASE | IMPORT | OFFSET | STATEMENT | -| ADD | DATABASES | IN | OR | STATE_WINDOW | -| AFTER | DAYS | INITIALLY | ORDER | STORAGE | -| ALL | DBS | INSERT | PARTITIONS | STREAM | -| ALTER | DEFERRED | INSTEAD | PASS | STREAMS | -| AND | DELIMITERS | INT | PLUS | STRING | -| AS | DESC | INTEGER | PPS | SYNCDB | -| ASC | DESCRIBE | INTERVAL | PRECISION | TABLE | -| ATTACH | DETACH | INTO | PREV | TABLES | -| BEFORE | DISTINCT | IS | PRIVILEGE | TAG | -| BEGIN | DIVIDE | ISNULL | QTIME | TAGS | -| BETWEEN | DNODE | JOIN | QUERIES | TBNAME | -| BIGINT | DNODES | KEEP | QUERY | TIMES | -| BINARY | DOT | KEY | QUORUM | TIMESTAMP | -| BITAND | DOUBLE | KILL | RAISE | TINYINT | -| BITNOT | DROP | LE | REM | TOPIC | -| BITOR | EACH | LIKE | REPLACE | TOPICS | -| BLOCKS | END | LIMIT | REPLICA | TRIGGER | -| BOOL | EQ | LINEAR | RESET | TSERIES | -| BY | EXISTS | LOCAL | RESTRICT | UMINUS | -| CACHE | EXPLAIN | LP | ROW | UNION | -| CACHELAST | FAIL | LSHIFT | RP | UNSIGNED | -| CASCADE | FILE | LT | RSHIFT | UPDATE | -| CHANGE | FILL | MATCH | SCORES | UPLUS | -| CLUSTER | FLOAT | MAXROWS | SELECT | USE | -| COLON | FOR | MINROWS | SEMI | USER | -| COLUMN | FROM | MINUS | SESSION | USERS | -| COMMA | FSYNC | MNODES | SET | USING | -| COMP | GE | MODIFY | SHOW | VALUES | -| COMPACT | GLOB | MODULES | SLASH | VARIABLE | -| CONCAT | GRANTS | NCHAR | SLIDING | VARIABLES | -| CONFLICT | GROUP | NE | SLIMIT | VGROUPS | -| CONNECTION | GT | NONE | SMALLINT | VIEW | -| CONNECTIONS | HAVING | NOT | SOFFSET | VNODES | -| CONNS | ID | NOTNULL | STable | WAL | -| COPY | IF | NOW | STableS | WHERE | -| _C0 | _QSTART | _QSTOP | _QDURATION | _WSTART | -| _WSTOP | _WDURATION | _ROWTS | +### A + +- ABORT +- ACCOUNT +- ACCOUNTS +- ADD +- AFTER +- ALL +- ALTER +- AND +- AS +- ASC +- ATTACH + +### B + +- BEFORE +- BEGIN +- BETWEEN +- BIGINT +- BINARY +- BITAND +- BITNOT +- BITOR +- BLOCKS +- BOOL +- BY + +### C + +- CACHE +- CACHELAST +- CASCADE +- CHANGE +- CLUSTER +- COLON +- COLUMN +- COMMA +- COMP +- COMPACT +- CONCAT +- CONFLICT +- CONNECTION +- CONNECTIONS +- CONNS +- COPY +- CREATE +- CTIME + +### D + +- DATABASE +- DATABASES +- DAYS +- DBS +- DEFERRED +- DELETE +- DELIMITERS +- DESC +- DESCRIBE +- DETACH +- DISTINCT +- DIVIDE +- DNODE +- DNODES +- DOT +- DOUBLE +- DROP + +### E + +- END +- EQ +- EXISTS +- EXPLAIN + +### F + +- FAIL +- FILE +- FILL +- FLOAT +- FOR +- FROM +- FSYNC + +### G + +- GE +- GLOB +- GRANTS +- GROUP +- GT + +### H + +- HAVING + +### I + +- ID +- IF +- IGNORE +- IMMEDIA +- IMPORT +- IN +- INITIAL +- INSERT +- INSTEAD +- INT +- INTEGER +- INTERVA +- INTO +- IS +- ISNULL + +### J + +- JOIN + +### K + +- KEEP +- KEY +- KILL + +### L + +- LE +- LIKE +- LIMIT +- LINEAR +- LOCAL +- LP +- LSHIFT +- LT + +### M + +- MATCH +- MAXROWS +- MINROWS +- MINUS +- MNODES +- MODIFY +- MODULES + +### N + +- NE +- NONE +- NOT +- NOTNULL +- NOW +- NULL + +### O + +- OF +- OFFSET +- OR +- ORDER + +### P + +- PARTITION +- PASS +- PLUS +- PPS +- PRECISION +- PREV +- PRIVILEGE + +### Q + +- QTIME +- QUERIE +- QUERY +- QUORUM + +### R + +- RAISE +- REM +- REPLACE +- REPLICA +- RESET +- RESTRIC +- ROW +- RP +- RSHIFT + +### S + +- SCORES +- SELECT +- SEMI +- SESSION +- SET +- SHOW +- SLASH +- SLIDING +- SLIMIT +- SMALLIN +- SOFFSET +- STable +- STableS +- STAR +- STATE +- STATEMEN +- STATE_WI +- STORAGE +- STREAM +- STREAMS +- STRING +- SYNCDB + +### T + +- TABLE +- TABLES +- TAG +- TAGS +- TBNAME +- TIMES +- TIMESTAMP +- TINYINT +- TOPIC +- TOPICS +- TRIGGER +- TSERIES + +### U + +- UMINUS +- UNION +- UNSIGNED +- UPDATE +- UPLUS +- USE +- USER +- USERS +- USING + +### V + +- VALUES +- VARIABLE +- VARIABLES +- VGROUPS +- VIEW +- VNODES + +### W + +- WAL +- WHERE + +### _ + +- _C0 +- _QSTART +- _QSTOP +- _QDURATION +- _WSTART +- _WSTOP +- _WDURATION ## Explanations ### TBNAME diff --git a/docs/zh/12-taos-sql/12-keywords.md b/docs/zh/12-taos-sql/12-keywords.md index 5c68e5da7e..202f223b45 100644 --- a/docs/zh/12-taos-sql/12-keywords.md +++ b/docs/zh/12-taos-sql/12-keywords.md @@ -45,48 +45,274 @@ title: TDengine 参数限制与保留关键字 目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable 名、数据列名及标签列名等。这些关键字列表如下： -| 关键字列表 | | | | | -| ----------- | ---------- | --------- | ---------- | ------------ | -| ABORT | CREATE | IGNORE | NULL | STAR | -| ACCOUNT | CTIME | IMMEDIATE | OF | STATE | -| ACCOUNTS | DATABASE | IMPORT | OFFSET | STATEMENT | -| ADD | DATABASES | IN | OR | STATE_WINDOW | -| AFTER | DAYS | INITIALLY | ORDER | STORAGE | -| ALL | DBS | INSERT | PARTITIONS | STREAM | -| ALTER | DEFERRED | INSTEAD | PASS | STREAMS | -| AND | DELIMITERS | INT | PLUS | STRING | -| AS | DESC | INTEGER | PPS | SYNCDB | -| ASC | DESCRIBE | INTERVAL | PRECISION | TABLE | -| ATTACH | DETACH | INTO | PREV | TABLES | -| BEFORE | DISTINCT | IS | PRIVILEGE | TAG | -| BEGIN | DIVIDE | ISNULL | QTIME | TAGS | -| BETWEEN | DNODE | JOIN | QUERIES | TBNAME | -| BIGINT | DNODES | KEEP | QUERY | TIMES | -| BINARY | DOT | KEY | QUORUM | TIMESTAMP | -| BITAND | DOUBLE | KILL | RAISE | TINYINT | -| BITNOT | DROP | LE | REM | TOPIC | -| BITOR | EACH | LIKE | REPLACE | TOPICS | -| BLOCKS | END | LIMIT | REPLICA | TRIGGER | -| BOOL | EQ | LINEAR | RESET | TSERIES | -| BY | EXISTS | LOCAL | RESTRICT | UMINUS | -| CACHE | EXPLAIN | LP | ROW | UNION | -| CACHELAST | FAIL | LSHIFT | RP | UNSIGNED | -| CASCADE | FILE | LT | RSHIFT | UPDATE | -| CHANGE | FILL | MATCH | SCORES | UPLUS | -| CLUSTER | FLOAT | MAXROWS | SELECT | USE | -| COLON | FOR | MINROWS | SEMI | USER | -| COLUMN | FROM | MINUS | SESSION | USERS | -| COMMA | FSYNC | MNODES | SET | USING | -| COMP | GE | MODIFY | SHOW | VALUES | -| COMPACT | GLOB | MODULES | SLASH | VARIABLE | -| CONCAT | GRANTS | NCHAR | SLIDING | VARIABLES | -| CONFLICT | GROUP | NE | SLIMIT | VGROUPS | -| CONNECTION | GT | NONE | SMALLINT | VIEW | -| CONNECTIONS | HAVING | NOT | SOFFSET | VNODES | -| CONNS | ID | NOTNULL | STABLE | WAL | -| COPY | IF | NOW | STABLES | WHERE | -| _C0 | _QSTART | _QSTOP | _QDURATION | _WSTART | -| _WSTOP | _WDURATION | _ROWTS | +### A + +- ABORT +- ACCOUNT +- ACCOUNTS +- ADD +- AFTER +- ALL +- ALTER +- AND +- AS +- ASC +- ATTACH + +### B + +- BEFORE +- BEGIN +- BETWEEN +- BIGINT +- BINARY +- BITAND +- BITNOT +- BITOR +- BLOCKS +- BOOL +- BY + +### C + +- CACHE +- CACHELAST +- CASCADE +- CHANGE +- CLUSTER +- COLON +- COLUMN +- COMMA +- COMP +- COMPACT +- CONCAT +- CONFLICT +- CONNECTION +- CONNECTIONS +- CONNS +- COPY +- CREATE +- CTIME + +### D + +- DATABASE +- DATABASES +- DAYS +- DBS +- DEFERRED +- DELETE +- DELIMITERS +- DESC +- DESCRIBE +- DETACH +- DISTINCT +- DIVIDE +- DNODE +- DNODES +- DOT +- DOUBLE +- DROP + +### E + +- END +- EQ +- EXISTS +- EXPLAIN + +### F + +- FAIL +- FILE +- FILL +- FLOAT +- FOR +- FROM +- FSYNC + +### G + +- GE +- GLOB +- GRANTS +- GROUP +- GT + +### H + +- HAVING + +### I + +- ID +- IF +- IGNORE +- IMMEDIA +- IMPORT +- IN +- INITIAL +- INSERT +- INSTEAD +- INT +- INTEGER +- INTERVA +- INTO +- IS +- ISNULL + +### J + +- JOIN + +### K + +- KEEP +- KEY +- KILL + +### L + +- LE +- LIKE +- LIMIT +- LINEAR +- LOCAL +- LP +- LSHIFT +- LT + +### M + +- MATCH +- MAXROWS +- MINROWS +- MINUS +- MNODES +- MODIFY +- MODULES + +### N + +- NE +- NONE +- NOT +- NOTNULL +- NOW +- NULL + +### O + +- OF +- OFFSET +- OR +- ORDER + +### P + +- PARTITION +- PASS +- PLUS +- PPS +- PRECISION +- PREV +- PRIVILEGE + +### Q + +- QTIME +- QUERIE +- QUERY +- QUORUM + +### R + +- RAISE +- REM +- REPLACE +- REPLICA +- RESET +- RESTRIC +- ROW +- RP +- RSHIFT + +### S + +- SCORES +- SELECT +- SEMI +- SESSION +- SET +- SHOW +- SLASH +- SLIDING +- SLIMIT +- SMALLIN +- SOFFSET +- STable +- STableS +- STAR +- STATE +- STATEMEN +- STATE_WI +- STORAGE +- STREAM +- STREAMS +- STRING +- SYNCDB + +### T + +- TABLE +- TABLES +- TAG +- TAGS +- TBNAME +- TIMES +- TIMESTAMP +- TINYINT +- TOPIC +- TOPICS +- TRIGGER +- TSERIES + +### U + +- UMINUS +- UNION +- UNSIGNED +- UPDATE +- UPLUS +- USE +- USER +- USERS +- USING + +### V + +- VALUES +- VARIABLE +- VARIABLES +- VGROUPS +- VIEW +- VNODES + +### W + +- WAL +- WHERE + +### _ + +- _C0 +- _QSTART +- _QSTOP +- _QDURATION +- _WSTART +- _WSTOP +- _WDURATION + ## 特殊说明 ### TBNAME From fe6d46d2c0a94d236897aeb1977e81e82cc66047 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 12:41:33 +0800 Subject: [PATCH 47/78] test: disable mnode basic3 --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index db8a055362..1c694bf7fb 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim -./test.sh -f tsim/mnode/basic3.sim +#./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim From 0b58c2b850a8e5896a6532763978d0e441bf2956 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 14 Jun 2022 13:03:12 +0800 Subject: [PATCH 48/78] test: modify sleep length --- tests/system-test/7-tmq/subscribeDb1.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index 44c24eb616..ed92a429ae 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -437,12 +437,12 @@ class TDTestCase: event.wait() tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 20 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - time.sleep(5) + time.sleep(3) tdLog.info("pkill consume processor") if (platform.system().lower() == 'windows'): os.system("TASKKILL /F /IM tmq_sim.exe") @@ -465,7 +465,7 @@ class TDTestCase: totalConsumeRows += resultList[i] if totalConsumeRows >= expectrowcnt or totalConsumeRows <= 0: - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.info("act consume rows: %d, expect consume rows between %d and 0"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") time.sleep(15) From 979420801eb0acda4cb3449b82a6332eadf9c87e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 13:07:27 +0800 Subject: [PATCH 49/78] fix(sync) if FpEqMsg return error, do not enqueue msg --- source/libs/sync/src/syncMain.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f3faa60d34..7934087ff3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1658,7 +1658,13 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); if (pSyncNode->FpEqMsg != NULL) { - pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + if (code != 0) { + sError("vgId:%d sync enqueue timer msg error, code:%d", pSyncNode->vgId, code); + rpcFreeCont(rpcMsg.pCont); + return; + } + } else { sTrace("syncNodeEqHeartbeatTimer pSyncNode->FpEqMsg is NULL"); } From a2039d8bb2dda45763804be5a267ebb1cd0d2693 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 13:16:49 +0800 Subject: [PATCH 50/78] fix(sync) if FpEqMsg return error, do not enqueue msg --- source/libs/sync/src/syncMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7934087ff3..e7f11b9a7d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1670,8 +1670,12 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } syncTimeoutDestroy(pSyncMsg); + if (gSyncEnv != NULL) { taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pHeartbeatTimer); + } else { + sError("sync env is already stop"); + } } else { sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 "", From 110bf979044f8aa012ad57a93bfbd07049c23506 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 13:22:24 +0800 Subject: [PATCH 51/78] fix: mnode not start --- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 2 +- tests/script/jenkins/basic.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 498af36786..119d521827 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -124,7 +124,7 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { } else { dInfo("node:%s, has been created", pWrapper->name); code = dmOpenNode(pWrapper); - if (code != 0) { + if (code == 0) { code = dmStartNode(pWrapper); } pWrapper->deployed = true; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1c694bf7fb..db8a055362 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim -#./test.sh -f tsim/mnode/basic3.sim +./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim From 545345de7b7e67dce5c3135dfc5424619dd334c1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Jun 2022 13:23:47 +0800 Subject: [PATCH 52/78] remove debug flag --- source/client/inc/clientInt.h | 2 +- source/libs/catalog/src/ctgDbg.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d72aa39e4c..6aa83e9575 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -45,7 +45,7 @@ extern "C" { #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms -#define SYNC_ON_TOP_OF_ASYNC 0 +#define SYNC_ON_TOP_OF_ASYNC 1 enum { RES_TYPE__QUERY = 1, diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index cd61abf6cc..9de1ea22be 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.apiEnable = true, .lockEnable = true}; +SCtgDebug gCTGDebug = {0}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); From cb85ee6d3c944148f0616757dbf304e95098eff0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 14 Jun 2022 13:36:54 +0800 Subject: [PATCH 53/78] opt: optimize generating groupid in partition/group by tag --- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 24 ++++++++---------------- source/libs/executor/src/scanoperator.c | 25 +++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 3e24e43233..ab60acab53 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -821,7 +821,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, SArray* pTableIdList, +SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 13ac74199e..e4c072a396 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4558,7 +4558,6 @@ static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SRead static int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STableListInfo* pListInfo, SNode* pTagCond); -static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo); static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* createSortInfo(SNodeList* pNodeList); @@ -4725,12 +4724,17 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else { qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); } - SArray* tableIdList = extractTableIdList(pTableListInfo); + + SArray* groupKyes = extractPartitionColInfo(pTableScanNode->pPartitionKeys); + int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKyes); //todo for json + if (code){ + tsdbCleanupReadHandle(pDataReader); + return NULL; + } SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pDataReader, pHandle, tableIdList, pTableScanNode, pTaskInfo, &twSup); + createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup); - taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; @@ -5183,18 +5187,6 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa return code; } -SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) { - SArray* tableIdList = taosArrayInit(4, sizeof(uint64_t)); - - // Transfer the Array of STableKeyInfo into uid list. - for (int32_t i = 0; i < taosArrayGetSize(pTableGroupInfo->pTableList); ++i) { - STableKeyInfo* pkeyInfo = taosArrayGet(pTableGroupInfo->pTableList, i); - taosArrayPush(tableIdList, &pkeyInfo->uid); - } - - return tableIdList; -} - tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, SNode* pTagCond) { int32_t code = diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b6ee9843b4..d30e4ef6db 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -886,6 +886,11 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->pRes->info.groupId = groupId; } + uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t)); + if (groupIdPre) { + pInfo->pRes->info.groupId = *groupIdPre; + } + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); if (!pColMatchInfo->output) { @@ -953,11 +958,24 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } } -SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, SArray* pTableIdList, +static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) { + SArray* tableIdList = taosArrayInit(4, sizeof(uint64_t)); + + // Transfer the Array of STableKeyInfo into uid list. + for (int32_t i = 0; i < taosArrayGetSize(pTableGroupInfo->pTableList); ++i) { + STableKeyInfo* pkeyInfo = taosArrayGet(pTableGroupInfo->pTableList, i); + taosArrayPush(tableIdList, &pkeyInfo->uid); + } + + return tableIdList; +} + +SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _error; @@ -988,10 +1006,13 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan // set the extract column id to streamHandle tqReadHandleSetColIdList((STqReadHandle*)pHandle->reader, pColIds); - int32_t code = tqReadHandleSetTbUidList(pHandle->reader, pTableIdList); + SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList); + int32_t code = tqReadHandleSetTbUidList(pHandle->reader, tableIdList); if (code != 0) { + taosArrayDestroy(tableIdList); goto _error; } + taosArrayDestroy(tableIdList); pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); if (pInfo->pBlockLists == NULL) { From 0237c68e272da34a38439b5c2707a69e2ece361a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Jun 2022 13:44:13 +0800 Subject: [PATCH 54/78] fix: invalid read/write --- source/dnode/vnode/src/meta/metaQuery.c | 11 ++++ tests/script/tsim/stable/tag_filter.sim | 75 +++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 1d8082084f..a6339125c4 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -663,12 +663,23 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { void * entryKey = NULL, *entryVal = NULL; int32_t nEntryKey, nEntryVal; + bool first = true; while (1) { valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal); if (valid < 0) { break; } STagIdxKey *p = entryKey; + if (p->type != pCursor->type) { + if (first) { + valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); + if (valid < 0) break; + continue; + } else { + break; + } + } + first = false; if (p != NULL) { int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); if (cmp == 0) { diff --git a/tests/script/tsim/stable/tag_filter.sim b/tests/script/tsim/stable/tag_filter.sim index 04b8f21de9..1f400eb803 100644 --- a/tests/script/tsim/stable/tag_filter.sim +++ b/tests/script/tsim/stable/tag_filter.sim @@ -73,5 +73,80 @@ if $rows != 6 then endi +print ========== prepare stbBin and ctbBin +sql create table db.stbBin (ts timestamp, c1 int, c2 binary(4)) tags(t1 binary(16)) + +sql create table db.ctbBin using db.stbBin tags("a") +sql insert into db.ctbBin values(now, 1, "2") + +sql create table db.ctbBin1 using db.stbBin tags("b") +sql insert into db.ctbBin1 values(now, 2, "2") + +sql create table db.ctbBin2 using db.stbBin tags("c") +sql insert into db.ctbBin2 values(now, 3, "2") + +sql create table db.ctbBin3 using db.stbBin tags("d") +sql insert into db.ctbBin3 values(now, 4, "2") + +sql select * from db.stbBin where t1 = "a" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbBin where t1 < "a" +if $rows != 0 then + return -=1 +endi + +sql select * from db.stbBin where t1 < "b" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbBin where t1 between "a" and "e" +if $rows != 4 then + return -1 +endi + + +print ========== prepare stbNc and ctbNc +sql create table db.stbNc (ts timestamp, c1 int, c2 binary(4)) tags(t1 nchar(16)) + + +sql create table db.ctbNc using db.stbNc tags("a") +sql insert into db.ctbNc values(now, 1, "2") + +sql create table db.ctbNc1 using db.stbNc tags("b") +sql insert into db.ctbNc1 values(now, 2, "2") + +sql create table db.ctbNc2 using db.stbNc tags("c") +sql insert into db.ctbNc2 values(now, 3, "2") + +sql create table db.ctbNc3 using db.stbNc tags("d") +sql insert into db.ctbNc3 values(now, 4, "2") + +sql select * from db.stbNc where t1 = "a" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbNc where t1 < "a" +if $rows != 0 then + return -=1 +endi + +sql select * from db.stbNc where t1 < "b" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbNc where t1 between "a" and "e" +if $rows != 4 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From 2a6dfbb0d7185d53eb1519ce9e66d583346d775c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 13:48:33 +0800 Subject: [PATCH 55/78] enh(query): avg function use pre-agg results --- source/libs/function/src/builtinsimpl.c | 154 +++++++++++++----------- 1 file changed, 83 insertions(+), 71 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index b5009b7d41..3510a59254 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -642,8 +642,8 @@ bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { int32_t avgFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; - // Only the pre-computing information loaded and actual data does not loaded SInputColumnInfoData* pInput = &pCtx->input; + SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; int32_t type = pInput->pData[0]->info.type; SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -660,95 +660,107 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { goto _avg_over; } - switch (type) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t* plist = (int8_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; + if (pInput->colDataAggIsSet) { + numOfElem = numOfRows - pAgg->numOfNull; + ASSERT(numOfElem >= 0); + + pAvgRes->count += numOfElem; + if (IS_INTEGER_TYPE(type)) { + pAvgRes->sum.isum += pAgg->sum; + } else if (IS_FLOAT_TYPE(type)) { + pAvgRes->sum.dsum += GET_DOUBLE_VAL((const char*)&(pAgg->sum)); + } + } else { // computing based on the true data block + switch (type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t* plist = (int8_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; } - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.isum += plist[i]; + break; } - break; - } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t* plist = (int16_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t* plist = (int16_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; + } + break; + } + + case TSDB_DATA_TYPE_INT: { + int32_t* plist = (int32_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; } - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.isum += plist[i]; + break; } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t* plist = (int32_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; + case TSDB_DATA_TYPE_BIGINT: { + int64_t* plist = (int64_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; } - - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.isum += plist[i]; + break; } - break; - } + case TSDB_DATA_TYPE_FLOAT: { + float* plist = (float*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } - case TSDB_DATA_TYPE_BIGINT: { - int64_t* plist = (int64_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.dsum += plist[i]; } - - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.isum += plist[i]; + break; } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float* plist = (float*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; + case TSDB_DATA_TYPE_DOUBLE: { + double* plist = (double*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.dsum += plist[i]; } - - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.dsum += plist[i]; + break; } - break; + + default: + break; } - - case TSDB_DATA_TYPE_DOUBLE: { - double* plist = (double*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; - } - - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.dsum += plist[i]; - } - break; - } - - default: - break; } _avg_over: From ad11eef979b82ad517e8154e9ba7e8006fff9f51 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 13:52:07 +0800 Subject: [PATCH 56/78] enh: show db status such as creating and dropping --- source/dnode/mnode/impl/src/mndDb.c | 25 ++++++++++++++----------- source/dnode/mnode/impl/src/mndMnode.c | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index b4237466d4..cfe363fdf0 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1366,7 +1366,7 @@ char *buildRetension(SArray *pRetension) { } static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, - bool sysDb) { + bool sysDb, ESdbStatus objStatus) { int32_t cols = 0; int32_t bytes = pShow->pMeta->pSchemas[cols].bytes; @@ -1379,6 +1379,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in } char *status = "ready"; + if (objStatus == SDB_STATUS_CREATING) status = "creating"; + if (objStatus == SDB_STATUS_DROPPING) status = "dropping"; char statusB[24] = {0}; STR_WITH_SIZE_TO_VARSTR(statusB, status, strlen(status)); @@ -1503,8 +1505,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)statusB, false); -// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); + // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + // colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); char *p = buildRetension(pDb->cfg.pRetensions); @@ -1548,29 +1550,30 @@ static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, v } static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - SDbObj *pDb = NULL; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SDbObj *pDb = NULL; + ESdbStatus objStatus = 0; // Append the information_schema database into the result. if (!pShow->sysDbRsp) { SDbObj infoschemaDb = {0}; setInformationSchemaDbCfg(&infoschemaDb); - dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true); + dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0); numOfRows += 1; SDbObj perfschemaDb = {0}; setPerfSchemaDbCfg(&perfschemaDb); - dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true); + dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0); numOfRows += 1; pShow->sysDbRsp = true; } while (numOfRows < rowsCapacity) { - pShow->pIter = sdbFetch(pSdb, SDB_DB, pShow->pIter, (void **)&pDb); + pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus); if (pShow->pIter == NULL) { break; } @@ -1578,7 +1581,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc int32_t numOfTables = 0; sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL); - dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false); + dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus); numOfRows++; sdbRelease(pSdb, pDb); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 95a2c3d935..f9749a969d 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -659,7 +659,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int32_t numOfRows = 0; int32_t cols = 0; SMnodeObj *pObj = NULL; - ESdbStatus objStatus; + ESdbStatus objStatus = 0; char *pWrite; int64_t curMs = taosGetTimestampMs(); From 8dea8dbbebeca2cf8b61d739d3cadbfe7b8c6b7f Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 14 Jun 2022 13:40:48 +0800 Subject: [PATCH 57/78] docs: move delete data from developer guide to sql reference guide --- .../05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} | 0 docs/en/12-taos-sql/{07-function.md => 10-function.md} | 0 docs/en/12-taos-sql/{08-interval.md => 12-interval.md} | 0 docs/en/12-taos-sql/{09-limit.md => 14-limit.md} | 0 docs/en/12-taos-sql/{10-json.md => 16-json.md} | 0 docs/en/12-taos-sql/{11-escape.md => 18-escape.md} | 0 docs/en/12-taos-sql/{12-keywords.md => 20-keywords.md} | 0 .../05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} | 0 docs/zh/12-taos-sql/{07-function.md => 10-function.md} | 0 docs/zh/12-taos-sql/{08-interval.md => 12-interval.md} | 0 docs/zh/12-taos-sql/{09-limit.md => 14-limit.md} | 0 docs/zh/12-taos-sql/{10-json.md => 16-json.md} | 0 docs/zh/12-taos-sql/{11-escape.md => 18-escape.md} | 0 docs/zh/12-taos-sql/{12-keywords.md => 20-keywords.md} | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename docs/en/{07-develop/05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} (100%) rename docs/en/12-taos-sql/{07-function.md => 10-function.md} (100%) rename docs/en/12-taos-sql/{08-interval.md => 12-interval.md} (100%) rename docs/en/12-taos-sql/{09-limit.md => 14-limit.md} (100%) rename docs/en/12-taos-sql/{10-json.md => 16-json.md} (100%) rename docs/en/12-taos-sql/{11-escape.md => 18-escape.md} (100%) rename docs/en/12-taos-sql/{12-keywords.md => 20-keywords.md} (100%) rename docs/zh/{07-develop/05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} (100%) rename docs/zh/12-taos-sql/{07-function.md => 10-function.md} (100%) rename docs/zh/12-taos-sql/{08-interval.md => 12-interval.md} (100%) rename docs/zh/12-taos-sql/{09-limit.md => 14-limit.md} (100%) rename docs/zh/12-taos-sql/{10-json.md => 16-json.md} (100%) rename docs/zh/12-taos-sql/{11-escape.md => 18-escape.md} (100%) rename docs/zh/12-taos-sql/{12-keywords.md => 20-keywords.md} (100%) diff --git a/docs/en/07-develop/05-delete-data.mdx b/docs/en/12-taos-sql/08-delete-data.mdx similarity index 100% rename from docs/en/07-develop/05-delete-data.mdx rename to docs/en/12-taos-sql/08-delete-data.mdx diff --git a/docs/en/12-taos-sql/07-function.md b/docs/en/12-taos-sql/10-function.md similarity index 100% rename from docs/en/12-taos-sql/07-function.md rename to docs/en/12-taos-sql/10-function.md diff --git a/docs/en/12-taos-sql/08-interval.md b/docs/en/12-taos-sql/12-interval.md similarity index 100% rename from docs/en/12-taos-sql/08-interval.md rename to docs/en/12-taos-sql/12-interval.md diff --git a/docs/en/12-taos-sql/09-limit.md b/docs/en/12-taos-sql/14-limit.md similarity index 100% rename from docs/en/12-taos-sql/09-limit.md rename to docs/en/12-taos-sql/14-limit.md diff --git a/docs/en/12-taos-sql/10-json.md b/docs/en/12-taos-sql/16-json.md similarity index 100% rename from docs/en/12-taos-sql/10-json.md rename to docs/en/12-taos-sql/16-json.md diff --git a/docs/en/12-taos-sql/11-escape.md b/docs/en/12-taos-sql/18-escape.md similarity index 100% rename from docs/en/12-taos-sql/11-escape.md rename to docs/en/12-taos-sql/18-escape.md diff --git a/docs/en/12-taos-sql/12-keywords.md b/docs/en/12-taos-sql/20-keywords.md similarity index 100% rename from docs/en/12-taos-sql/12-keywords.md rename to docs/en/12-taos-sql/20-keywords.md diff --git a/docs/zh/07-develop/05-delete-data.mdx b/docs/zh/12-taos-sql/08-delete-data.mdx similarity index 100% rename from docs/zh/07-develop/05-delete-data.mdx rename to docs/zh/12-taos-sql/08-delete-data.mdx diff --git a/docs/zh/12-taos-sql/07-function.md b/docs/zh/12-taos-sql/10-function.md similarity index 100% rename from docs/zh/12-taos-sql/07-function.md rename to docs/zh/12-taos-sql/10-function.md diff --git a/docs/zh/12-taos-sql/08-interval.md b/docs/zh/12-taos-sql/12-interval.md similarity index 100% rename from docs/zh/12-taos-sql/08-interval.md rename to docs/zh/12-taos-sql/12-interval.md diff --git a/docs/zh/12-taos-sql/09-limit.md b/docs/zh/12-taos-sql/14-limit.md similarity index 100% rename from docs/zh/12-taos-sql/09-limit.md rename to docs/zh/12-taos-sql/14-limit.md diff --git a/docs/zh/12-taos-sql/10-json.md b/docs/zh/12-taos-sql/16-json.md similarity index 100% rename from docs/zh/12-taos-sql/10-json.md rename to docs/zh/12-taos-sql/16-json.md diff --git a/docs/zh/12-taos-sql/11-escape.md b/docs/zh/12-taos-sql/18-escape.md similarity index 100% rename from docs/zh/12-taos-sql/11-escape.md rename to docs/zh/12-taos-sql/18-escape.md diff --git a/docs/zh/12-taos-sql/12-keywords.md b/docs/zh/12-taos-sql/20-keywords.md similarity index 100% rename from docs/zh/12-taos-sql/12-keywords.md rename to docs/zh/12-taos-sql/20-keywords.md From 601e454a242bd511f2e2914f3f946490715cb45f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 13:58:40 +0800 Subject: [PATCH 58/78] enh(tmq): put offset store into vnode --- include/common/tmsg.h | 29 +++- include/common/tmsgdef.h | 3 +- source/client/src/clientImpl.c | 54 +++--- source/client/src/tmq.c | 172 +++++++++++++++++++- source/common/src/tdatablock.c | 4 +- source/common/src/tmsg.c | 29 ++++ source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/vnode/src/inc/tq.h | 36 ++-- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tq/tq.c | 35 +++- source/dnode/vnode/src/tq/tqOffset.c | 115 +++++++++++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 15 +- 12 files changed, 415 insertions(+), 79 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d69849349c..be9eda3825 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1494,9 +1494,9 @@ typedef struct { int32_t code; } STaskDropRsp; -#define STREAM_TRIGGER_AT_ONCE 1 -#define STREAM_TRIGGER_WINDOW_CLOSE 2 -#define STREAM_TRIGGER_MAX_DELAY 3 +#define STREAM_TRIGGER_AT_ONCE 1 +#define STREAM_TRIGGER_WINDOW_CLOSE 2 +#define STREAM_TRIGGER_MAX_DELAY 3 typedef struct { char name[TSDB_TABLE_FNAME_LEN]; @@ -2297,6 +2297,29 @@ int32_t tDecodeSMqOffset(SDecoder* decoder, SMqOffset* pOffset); int32_t tEncodeSMqCMCommitOffsetReq(SEncoder* encoder, const SMqCMCommitOffsetReq* pReq); int32_t tDecodeSMqCMCommitOffsetReq(SDecoder* decoder, SMqCMCommitOffsetReq* pReq); +// tqOffset +enum { + TMQ_OFFSET__SNAPSHOT = 1, + TMQ_OFFSET__LOG, +}; + +typedef struct { + int8_t type; + union { + struct { + int64_t uid; + int64_t ts; + }; + struct { + int64_t version; + }; + }; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; +} STqOffset; + +int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset); +int32_t tDecodeSTqOffset(SDecoder* pDecoder, STqOffset* pOffset); + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char stb[TSDB_TABLE_FNAME_LEN]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 743f10bd55..534194e1cf 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -140,7 +140,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "consumer-recover", SMqConsumerRecoverMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "do-rebalance", SMqDoRebalanceMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DROP_CGROUP, "drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mq-tmr", SMTimerReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "trans-tmr", NULL, NULL) @@ -176,6 +176,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", SVDropStbReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_DELETE, "vnode-mq-vg-delete", SMqVDeleteReq, SMqVDeleteRsp) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_COMMIT_OFFSET, "vnode-commit-offset", STqOffset, STqOffset) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_TASK, "vnode-drop-task", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TOPIC, "vnode-create-topic", NULL, NULL) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index dc83f7bc43..f48383c43a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -205,7 +205,7 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { SRetrieveTableRsp* pRsp = NULL; int32_t code = qExecCommand(pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, false); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true); } return code; @@ -232,9 +232,7 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { return TSDB_CODE_SUCCESS; } -static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { - return pRequest->pTscObj->pAppInfo; -} +static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { return pRequest->pTscObj->pAppInfo; } void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { SRetrieveTableRsp* pRsp = NULL; @@ -258,7 +256,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { } pRequest->body.queryFp(pRequest->body.param, pRequest, 0); -// pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); + // pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); } int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { @@ -401,7 +399,7 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod SQueryResult res = {.code = 0, .numOfRows = 0}; int32_t code = schedulerAsyncExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, - pRequest->metric.start, schdExecCallback, &res); + pRequest->metric.start, schdExecCallback, &res); pRequest->body.resInfo.execRes = res.res; @@ -457,7 +455,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList return pRequest->code; } - if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type || TDMT_VND_CREATE_TABLE == pRequest->type) { + if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type || + TDMT_VND_CREATE_TABLE == pRequest->type) { pRequest->body.resInfo.numOfRows = res.numOfRows; if (pRequest->body.queryJob != 0) { @@ -470,9 +469,9 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList return pRequest->code; } -int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet *epset) { - int32_t code = 0; - SArray* pArray = NULL; +int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { + int32_t code = 0; + SArray* pArray = NULL; SSubmitRsp* pRsp = (SSubmitRsp*)res; if (pRsp->nBlocks <= 0) { return TSDB_CODE_SUCCESS; @@ -502,7 +501,7 @@ _return: return code; } -int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet *epset) { +int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { int32_t code = 0; SArray* pArray = NULL; SArray* pTbArray = (SArray*)res; @@ -540,15 +539,15 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { return TSDB_CODE_SUCCESS; } - SCatalog* pCatalog = NULL; - SAppInstInfo* pAppInfo = getAppInfo(pRequest); + SCatalog* pCatalog = NULL; + SAppInstInfo* pAppInfo = getAppInfo(pRequest); int32_t code = catalogGetHandle(pAppInfo->clusterId, &pCatalog); if (code) { return code; } - SEpSet epset = getEpSet_s(&pAppInfo->mgmtEp); + SEpSet epset = getEpSet_s(&pAppInfo->mgmtEp); SQueryExecRes* pRes = &pRequest->body.resInfo.execRes; switch (pRes->msgType) { @@ -566,8 +565,8 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { break; } default: - tscError("0x%"PRIx64", invalid exec result for request type %d, reqId:0x%"PRIx64, pRequest->self, - pRequest->type, pRequest->requestId); + tscError("0x%" PRIx64 ", invalid exec result for request type %d, reqId:0x%" PRIx64, pRequest->self, + pRequest->type, pRequest->requestId); code = TSDB_CODE_APP_ERROR; } @@ -575,13 +574,13 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { } void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { - SRequestObj* pRequest = (SRequestObj*) param; + SRequestObj* pRequest = (SRequestObj*)param; pRequest->code = code; STscObj* pTscObj = pRequest->pTscObj; if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) { - tscDebug("0x%"PRIx64" client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code), - pRequest->retry, pRequest->requestId); + tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, + pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); pRequest->prevCode = code; doAsyncQuery(pRequest, true); return; @@ -589,7 +588,7 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { if (code == TSDB_CODE_SUCCESS) { code = handleQueryExecRsp(pRequest); - ASSERT(pRequest->code == TSDB_CODE_SUCCESS); + ASSERT(pRequest->code == TSDB_CODE_SUCCESS); pRequest->code = code; } @@ -697,16 +696,17 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) { schedulerAsyncExecJob(pAppInfo->pTransporter, pNodeList, pRequest->body.pDag, &pRequest->body.queryJob, pRequest->sqlstr, pRequest->metric.start, schedulerExecCb, pRequest); } else { - tscError("0x%"PRIx64" failed to create query plan, code:%s 0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); + tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); pRequest->body.queryFp(pRequest->body.param, pRequest, code); } - //todo not to be released here + // todo not to be released here taosArrayDestroy(pNodeList); break; } case QUERY_EXEC_MODE_EMPTY_RESULT: - pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; pRequest->body.queryFp(pRequest->body.param, pRequest, 0); break; default: @@ -1349,14 +1349,14 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 p += sizeof(uint64_t); // check fields - for(int32_t i = 0; i < numOfCols; ++i) { - int16_t type = *(int16_t*) p; + for (int32_t i = 0; i < numOfCols; ++i) { + int16_t type = *(int16_t*)p; p += sizeof(int16_t); - int32_t bytes = *(int32_t*) p; + int32_t bytes = *(int32_t*)p; p += sizeof(int32_t); -// ASSERT(type == pFields[i].type && bytes == pFields[i].bytes); + ASSERT(type == pFields[i].type && bytes == pFields[i].bytes); } int32_t* colLength = (int32_t*)p; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 7d49c4206f..48c85cf265 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -132,6 +132,7 @@ typedef struct { // statistics int64_t pollCnt; // offset + int64_t committedOffset; int64_t currentOffset; // connection info int32_t vgId; @@ -193,6 +194,26 @@ typedef struct { void* userParam; } SMqCommitCbParam; +typedef struct { + tmq_t* tmq; + int8_t automatic; + int8_t async; + int8_t freeOffsets; + int8_t waitingRspNum; + int8_t totalRspNum; + tmq_resp_err_t rspErr; + tmq_commit_cb* userCb; + SArray* successfulOffsets; + SArray* failedOffsets; + void* userParam; + tsem_t rspSem; +} SMqCommitCbParamSet; + +typedef struct { + SMqCommitCbParamSet* params; + STqOffset* pOffset; +} SMqCommitCbParam2; + tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); conf->withTbName = false; @@ -343,6 +364,135 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } +int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { + SMqCommitCbParam2* pParam = (SMqCommitCbParam2*)param; + SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; + // push into array + if (code == 0) { + taosArrayPush(pParamSet->failedOffsets, &pParam->pOffset); + } else { + taosArrayPush(pParamSet->successfulOffsets, &pParam->pOffset); + } + // count down waiting rsp + int8_t waitingRspNum = atomic_sub_fetch_8(&pParam->params->waitingRspNum, 1); + ASSERT(waitingRspNum >= 0); + + if (waitingRspNum == 0) { + // if no more waiting rsp + if (pParamSet->async) { + // call async cb func + if (pParamSet->automatic && pParamSet->tmq->commitCb) { + pParamSet->tmq->commitCb(pParamSet->tmq, pParamSet->rspErr, NULL, pParamSet->tmq->commitCbUserParam); + } else if (!pParamSet->automatic && pParamSet->userCb) { + // sem post + pParamSet->userCb(pParamSet->tmq, pParamSet->rspErr, NULL, pParamSet->userParam); + } + } + + taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); + taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); + } + return 0; +} + +int32_t tmqComitInner2(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { + int32_t code = -1; + + SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); + if (pParamSet == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pParamSet->tmq = tmq; + pParamSet->automatic = automatic; + pParamSet->async = async; + pParamSet->freeOffsets = 1; + pParamSet->userCb = userCb; + pParamSet->userParam = userParam; + tsem_init(&pParamSet->rspSem, 0, 0); + + for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, i); + STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); + if (pOffset == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + int32_t tlen = strlen(tmq->groupId); + memcpy(pOffset->subKey, tmq->groupId, tlen); + pOffset->subKey[tlen] = TMQ_SEPARATOR; + strcpy(pOffset->subKey + tlen + 1, pTopic->topicName); + int32_t len; + int32_t code; + tEncodeSize(tEncodeSTqOffset, pOffset, len, code); + if (code < 0) { + ASSERT(0); + } + void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + tEncodeSTqOffset(&encoder, pOffset); + + // build param + SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); + pParam->params = pParamSet; + pParam->pOffset = pOffset; + + // build send info + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (pMsgSendInfo == NULL) { + // TODO + continue; + } + pMsgSendInfo->msgInfo = (SDataBuf){ + .pData = buf, + .len = len, + .handle = NULL, + }; + + pMsgSendInfo->requestId = generateRequestId(); + pMsgSendInfo->requestObjRefId = 0; + pMsgSendInfo->param = pParam; + pMsgSendInfo->fp = tmqCommitCb2; + pMsgSendInfo->msgType = TDMT_MND_MQ_COMMIT_OFFSET; + // send msg + + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); + } + } + + if (!async) { + tsem_wait(&pParamSet->rspSem); + code = pParamSet->rspErr; + tsem_destroy(&pParamSet->rspSem); + } else { + code = 0; + } + + if (code != 0 && async) { + if (automatic) { + tmq->commitCb(tmq, code, NULL, tmq->commitCbUserParam); + } else { + userCb(tmq, code, NULL, userParam); + } + } + + if (!async) { + taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); + taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); + } + + return 0; +} + int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { SMqCMCommitOffsetReq req; @@ -890,12 +1040,13 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); int64_t offset = pVgEp->offset; - tscDebug("consumer %ld epoch %d vg %d offset og to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld(epoch %d) original offset of vg %d is %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); if (pOffset != NULL) { offset = *pOffset; - tscDebug("consumer %ld epoch %d vg %d found %s", tmq->consumerId, epoch, pVgEp->vgId, vgKey); + tscDebug("consumer %ld(epoch %d) receive offset of vg %d, full key is %s", tmq->consumerId, epoch, pVgEp->vgId, + vgKey); } - tscDebug("consumer %ld epoch %d vg %d offset set to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); SMqClientVg clientVg = { .pollCnt = 0, .currentOffset = offset, @@ -1226,9 +1377,8 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/ - /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/ - if (pollRspWrapper->msg.head.epoch == atomic_load_32(&tmq->epoch)) { - /*printf("epoch match\n");*/ + int32_t consumerEpoch = atomic_load_32(&tmq->epoch); + if (pollRspWrapper->msg.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ pVg->currentOffset = pollRspWrapper->msg.rspOffset; @@ -1243,7 +1393,8 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { taosFreeQitem(pollRspWrapper); return pRsp; } else { - /*printf("epoch mismatch\n");*/ + tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n", pollRspWrapper->msg.head.epoch, + consumerEpoch); taosFreeQitem(pollRspWrapper); } } else { @@ -1263,10 +1414,14 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { SMqRspObj* rspObj; int64_t startTime = taosGetTimestampMs(); +#if 0 + tmqHandleAllDelayedTask(tmq); + tmqPollImpl(tmq, timeout); rspObj = tmqHandleAllRsp(tmq, timeout, false); if (rspObj) { return (TAOS_RES*)rspObj; } +#endif // in no topic status also need process delayed task if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { @@ -1359,8 +1514,7 @@ const char* tmq_get_table_name(TAOS_RES* res) { pRspObj->resIter >= pRspObj->rsp.blockNum) { return NULL; } - const char* name = taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter); - return name; + return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter); } return NULL; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b7a9ef88b6..4d1f5a34b1 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1756,7 +1756,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = 1, + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .pData = (uint8_t*)&pDataBlock->info.groupId, .nData = sizeof(uint64_t)}; @@ -1821,7 +1821,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = 1, + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .pData = (uint8_t*)&pDataBlock->info.groupId, .nData = sizeof(uint64_t)}; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d16ab57ea9..4ebd91b0da 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4746,3 +4746,32 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) { taosMemoryFree(pRsp->pMeta); } } + +int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) { + if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1; + if (pOffset->type == TMQ_OFFSET__SNAPSHOT) { + if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pOffset->ts) < 0) return -1; + } else if (pOffset->type == TMQ_OFFSET__LOG) { + if (tEncodeI64(pEncoder, pOffset->version) < 0) return -1; + } else { + ASSERT(0); + } + if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1; + return 0; +} + +int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) { + if (tDecodeI8(pDecoder, &pOffset->type) < 0) return -1; + if (pOffset->type == TMQ_OFFSET__SNAPSHOT) { + if (tDecodeI64(pDecoder, &pOffset->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pOffset->ts) < 0) return -1; + } else if (pOffset->type == TMQ_OFFSET__LOG) { + if (tDecodeI64(pDecoder, &pOffset->version) < 0) return -1; + } else { + ASSERT(0); + } + if (tDecodeCStrTo(pDecoder, pOffset->subKey) < 0) return -1; + return 0; +} + diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index ee120576c3..750006d05f 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -344,6 +344,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT_RSMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 5a8564bfd1..2ee0673ce5 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -41,7 +41,6 @@ extern "C" { #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) // clang-format on -typedef struct STqOffsetCfg STqOffsetCfg; typedef struct STqOffsetStore STqOffsetStore; // tqRead @@ -127,14 +126,15 @@ typedef struct { } STqHandle; struct STQ { - char* path; - SHashObj* pushMgr; // consumerId -> STqHandle* - SHashObj* handles; // subKey -> STqHandle - SHashObj* pStreamTasks; // taksId -> SStreamTask - SVnode* pVnode; - SWal* pWal; - TDB* pMetaStore; - TTB* pExecStore; + char* path; + SHashObj* pushMgr; // consumerId -> STqHandle* + SHashObj* handles; // subKey -> STqHandle + SHashObj* pStreamTasks; // taksId -> SStreamTask + STqOffsetStore* pOffsetStore; + SVnode* pVnode; + SWal* pWal; + TDB* pMetaStore; + TTB* pExecStore; }; typedef struct { @@ -157,17 +157,19 @@ int32_t tqMetaClose(STQ* pTq); int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle); int32_t tqMetaDeleteHandle(STQ* pTq, const char* key); +typedef struct { + int32_t size; +} STqOffsetHead; + +STqOffsetStore* tqOffsetOpen(); +void tqOffsetClose(STqOffsetStore*); +STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey); +int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset); +int32_t tqOffsetSnapshot(STqOffsetStore* pStore); + // tqSink void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); -// tqOffset -STqOffsetStore* tqOffsetOpen(STqOffsetCfg*); -void tqOffsetClose(STqOffsetStore*); -int64_t tqOffsetFetch(STqOffsetStore* pStore, const char* subscribeKey); -int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t offset); -int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey); -int32_t tqOffsetPersistAll(STqOffsetStore* pStore); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0c2b09a493..300a5f890e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -137,6 +137,7 @@ int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* data); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4a5ea49d79..22685c1e19 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -66,19 +66,23 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { ASSERT(0); } + if (tqOffsetOpen(pTq) < 0) { + ASSERT(0); + } + return pTq; } void tqClose(STQ* pTq) { if (pTq) { - taosMemoryFreeClear(pTq->path); + tqOffsetClose(pTq->pOffsetStore); taosHashCleanup(pTq->handles); taosHashCleanup(pTq->pStreamTasks); taosHashCleanup(pTq->pushMgr); + taosMemoryFree(pTq->path); tqMetaClose(pTq); taosMemoryFree(pTq); } - // TODO } int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp) { @@ -109,6 +113,33 @@ int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con return 0; } +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { + STqOffset offset = {0}; + SDecoder decoder; + tDecoderInit(&decoder, msg, msgLen); + if (tDecodeSTqOffset(&decoder, &offset) < 0) { + ASSERT(0); + return -1; + } + tDecoderClear(&decoder); + + if (offset.type == TMQ_OFFSET__SNAPSHOT) { + tqDebug("receive offset commit msg to %s, offset(type:snapshot) uid: %ld, ts: %ld", offset.subKey, offset.uid, + offset.ts); + } else if (offset.type == TMQ_OFFSET__LOG) { + tqDebug("receive offset commit msg to %s, offset(type:log) version: %ld", offset.subKey, offset.version); + } else { + ASSERT(0); + } + + if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { + ASSERT(0); + return -1; + } + + return 0; +} + int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 4d83a67579..2db49d7cf5 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -16,26 +16,113 @@ #include "tq.h" -enum ETqOffsetPersist { - TQ_OFFSET_PERSIST__LAZY = 1, - TQ_OFFSET_PERSIST__EAGER, -}; - -struct STqOffsetCfg { - int8_t persistPolicy; -}; - struct STqOffsetStore { - STqOffsetCfg cfg; - SHashObj* pHash; // SHashObj + STQ* pTq; + SHashObj* pHash; // SHashObj }; -STqOffsetStore* tqOffsetOpen(STqOffsetCfg* pCfg) { - STqOffsetStore* pStore = taosMemoryMalloc(sizeof(STqOffsetStore)); +STqOffsetStore* tqOffsetOpen(STQ* pTq) { + STqOffsetStore* pStore = taosMemoryCalloc(1, sizeof(STqOffsetStore)); if (pStore == NULL) { return NULL; } - memcpy(&pStore->cfg, pCfg, sizeof(STqOffsetCfg)); pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); + if (pStore->pHash == NULL) { + if (pStore->pHash) taosHashCleanup(pStore->pHash); + return NULL; + } + TdFilePtr pFile = taosOpenFile(pStore->pTq->path, TD_FILE_READ); + if (pFile != NULL) { + STqOffsetHead head = {0}; + int64_t code; + + while (1) { + if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { + if (code < 0) { + break; + } else { + ASSERT(0); + // TODO handle error + } + } + int32_t size = htonl(head.size); + void* memBuf = taosMemoryCalloc(1, size); + if ((code = taosReadFile(pFile, memBuf, size)) != size) { + ASSERT(0); + // TODO handle error + } + STqOffset offset; + SDecoder decoder; + tDecoderInit(&decoder, memBuf, size); + if (tDecodeSTqOffset(&decoder, &offset) < 0) { + ASSERT(0); + } + tDecoderClear(&decoder); + if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { + ASSERT(0); + // TODO + } + } + + taosCloseFile(&pFile); + } return pStore; } + +void tqOffsetClose(STqOffsetStore* pStore) { + tqOffsetSnapshot(pStore); + taosHashCleanup(pStore->pHash); +} + +STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) { + return (STqOffset*)taosHashGet(pStore->pHash, subscribeKey, strlen(subscribeKey)); +} + +int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) { + return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset)); +} + +int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { + // open file + // TODO file name should be with a version + TdFilePtr pFile = taosOpenFile(pStore->pTq->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pFile == NULL) { + ASSERT(0); + return -1; + } + void* pIter = NULL; + while (1) { + pIter = taosHashIterate(pStore->pHash, pIter); + if (pIter == NULL) break; + STqOffset* pOffset = (STqOffset*)pIter; + int32_t bodyLen; + int32_t code; + tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code); + ASSERT(code == 0); + if (code < 0) { + ASSERT(0); + taosHashCancelIterate(pStore->pHash, pIter); + return -1; + } + + int32_t totLen = sizeof(STqOffsetHead) + bodyLen; + void* buf = taosMemoryCalloc(1, totLen); + void* abuf = POINTER_SHIFT(buf, sizeof(STqOffsetHead)); + + ((STqOffsetHead*)buf)->size = htonl(bodyLen); + SEncoder encoder; + tEncoderInit(&encoder, abuf, bodyLen); + tEncodeSTqOffset(&encoder, pOffset); + // write file + int64_t writeLen; + if ((writeLen = taosWriteFile(pFile, buf, totLen)) != bodyLen) { + ASSERT(0); + tqError("write offset incomplete, len %d, write len %ld", bodyLen, writeLen); + taosHashCancelIterate(pStore->pHash, pIter); + return -1; + } + } + // close and rename file + taosCloseFile(&pFile); + return 0; +} diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ab2efa4791..186ab4b528 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -148,17 +148,24 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp case TDMT_VND_MQ_VG_CHANGE: if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { - // TODO: handle error + goto _err; } break; case TDMT_VND_MQ_VG_DELETE: if (tqProcessVgDeleteReq(pVnode->pTq, pMsg->pCont, pMsg->contLen) < 0) { - // TODO: handle error + goto _err; + } + break; + case TDMT_VND_MQ_COMMIT_OFFSET: + if (tqProcessOffsetCommitReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), + pMsg->contLen - sizeof(SMsgHead)) < 0) { + goto _err; } break; case TDMT_STREAM_TASK_DEPLOY: { if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { + goto _err; } } break; case TDMT_VND_ALTER_CONFIRM: @@ -901,8 +908,8 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { vInfo("vgId:%d, alter hashrange msg will be processed", TD_VID(pVnode)); - // todo - // 1. stop work + // todo + // 1. stop work // 2. adjust hash range / compact / remove wals / rename vgroups // 3. reload sync return 0; From b298d45bb7e24cca6ca014d7e9de5c59efac6e2e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 14 Jun 2022 13:46:28 +0800 Subject: [PATCH 59/78] feat(stream): create sub table --- source/common/src/tdatablock.c | 13 ++--- tests/script/jenkins/basic.txt | 1 + tests/script/tsim/stream/partitionby.sim | 68 ++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 tests/script/tsim/stream/partitionby.sim diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b7a9ef88b6..5affcf5100 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1708,6 +1708,7 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { pTag->keyLen = strlen(pTag->key); pTag->type = TSDB_DATA_TYPE_UBIGINT; pTag->u = groupId; + pTag->length = sizeof(uint64_t); taosArrayPush(tags, &pTag); void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); @@ -1756,10 +1757,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = 1, + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; + .i64 = (int64_t)pDataBlock->info.groupId, + }; STag* pTag = NULL; taosArrayClear(tagArray); taosArrayPush(tagArray, &tagVal); @@ -1821,10 +1822,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = 1, + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; + .i64 = (int64_t)pDataBlock->info.groupId, + }; taosArrayClear(tagArray); taosArrayPush(tagArray, &tagVal); STag* pTag = NULL; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index db8a055362..0a03ff58fc 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -78,6 +78,7 @@ # ./test.sh -f tsim/stream/state0.sim # ./test.sh -f tsim/stream/triggerInterval0.sim # ./test.sh -f tsim/stream/triggerSession0.sim +# ./test.sh -f tsim/stream/partitionby.sim # ---- transaction diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim new file mode 100644 index 0000000000..df1e096551 --- /dev/null +++ b/tests/script/tsim/stream/partitionby.sim @@ -0,0 +1,68 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); +sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by ta,tb,tc interval(10s); + +sql insert into ts1 values(1648791213001,1,12,3,1.0); +sql insert into ts2 values(1648791213001,1,12,3,1.0); + +sql insert into ts3 values(1648791213001,1,12,3,1.0); +sql insert into ts4 values(1648791213001,1,12,3,1.0); +$loop_count = 0 + +loop0: +sleep 300 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 4 then +print =====rows=$rows +goto loop0 +endi + + +sql create database test1 vgroups 1; +sql use test1; +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,2,3); +sql create table ts2 using st tags(1,3,4); +sql create table ts3 using st tags(1,4,5); + +sql create stream streams1 trigger at_once into streamt as select _wstartts, count(*) c1, count(a) c2 from st partition by ta,tb,tc interval(10s); + + +sql insert into ts1 values(1648791211000,1,2,3); + +sql insert into ts2 values(1648791211000,1,2,3); + +$loop_count = 0 + +loop0: +sleep 300 +sql select * from streamt; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then +print =====rows=$rows +goto loop0 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 92cf4c4cb613cd046099c5d54390f22adf064632 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 14 Jun 2022 14:38:35 +0800 Subject: [PATCH 60/78] os: fix lz4 error --- source/os/src/osLz4.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/os/src/osLz4.c b/source/os/src/osLz4.c index f4e4951857..7f3c42fb6c 100644 --- a/source/os/src/osLz4.c +++ b/source/os/src/osLz4.c @@ -45,13 +45,13 @@ int32_t BUILDIN_CLZL(uint64_t val) { #else _MyBitScanReverse64(&r, val); #endif - return (int)(r >> 3); + return (int)(63 - r); } int32_t BUILDIN_CLZ(uint32_t val) { unsigned long r = 0; _BitScanReverse(&r, val); - return (int)(r >> 3); + return (int)(31 - r); } int32_t BUILDIN_CTZL(uint64_t val) { @@ -61,13 +61,13 @@ int32_t BUILDIN_CTZL(uint64_t val) { #else _MyBitScanForward64(&r, val); #endif - return (int)(r >> 3); + return (int)(r); } int32_t BUILDIN_CTZ(uint32_t val) { unsigned long r = 0; _BitScanForward(&r, val); - return (int)(r >> 3); + return (int)(r); } #endif From 7435221e3280de9b723bc8108b289e18ad6b2298 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 14 Jun 2022 14:40:01 +0800 Subject: [PATCH 61/78] opt: optimize generating groupid in partition/group by tag --- include/util/taoserror.h | 2 ++ source/libs/executor/src/executorimpl.c | 16 +++++++++++----- source/util/src/terror.c | 1 + 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 03308e395f..8d9951f9e3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -695,6 +695,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150) #define TSDB_CODE_RSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3151) +//index +#define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200) #ifdef __cplusplus } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e4c072a396..f878e9e027 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4690,8 +4690,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - SArray* groupKyes = extractPartitionColInfo(pTableScanNode->pPartitionKeys); - code = generateGroupIdMap(pTableListInfo, pHandle, groupKyes); //todo for json + SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionKeys); + code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); //todo for json + taosArrayDestroy(groupKeys); if (code){ tsdbCleanupReadHandle(pDataReader); return NULL; @@ -4725,8 +4726,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); } - SArray* groupKyes = extractPartitionColInfo(pTableScanNode->pPartitionKeys); - int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKyes); //todo for json + SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionKeys); + int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); //todo for json + taosArrayDestroy(groupKeys); if (code){ tsdbCleanupReadHandle(pDataReader); return NULL; @@ -5059,6 +5061,7 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { } SArray* extractPartitionColInfo(SNodeList* pNodeList) { + if(!pNodeList) return NULL; size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); if (pList == NULL) { @@ -5163,7 +5166,9 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa SArray* res = taosArrayInit(8, sizeof(uint64_t)); code = doFilterTag(pTagCond, &metaArg, res); - if (code != TSDB_CODE_SUCCESS) { + if (code == TSDB_CODE_INDEX_REBUILDING){ // todo + // doFilter(); + } else if (code != TSDB_CODE_SUCCESS) { qError("failed to get tableIds, reason: %s, suid: %" PRIu64 "", tstrerror(code), tableUid); taosArrayDestroy(res); terrno = code; @@ -5171,6 +5176,7 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa } else { qDebug("sucess to get tableIds, size: %d, suid: %" PRIu64 "", (int)taosArrayGetSize(res), tableUid); } + for (int i = 0; i < taosArrayGetSize(res); i++) { STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)}; taosArrayPush(pListInfo->pTableList, &info); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index e122ad0ab6..079d5ef590 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -566,6 +566,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_RM_SKEY_IN_HASH, "Rm tsma skey in cac TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV, "Invalid rsma env") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT, "Invalid rsma state") +TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding") #ifdef TAOS_ERROR_C }; From 5268b5233aa772c4304c11b970d0752b283ab9ba Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 14:40:06 +0800 Subject: [PATCH 62/78] refactor(sync): update replica index in snapshot sender --- source/libs/sync/src/syncMain.c | 50 ++++++++++++++++++----------- source/libs/sync/src/syncSnapshot.c | 8 +++-- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 8be8c14b8f..feced0afdf 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -561,7 +561,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) stub.createTime = taosGetTimestampMs(); stub.rpcMsg = *pMsg; uint64_t seqNum = syncRespMgrAdd(pSyncNode->pSyncRespMgr, &stub); - sDebug("vgId:%d, sync event propose, type:%s seq:%" PRIu64 " handle:%p", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), + sDebug("vgId:%d sync event propose, type:%s seq:%" PRIu64 " handle:%p", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), seqNum, pMsg->info.handle); SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, seqNum, isWeak, pSyncNode->vgId); @@ -1244,7 +1244,13 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l SRaftId oldReplicasId[TSDB_MAX_REPLICA]; memcpy(oldReplicasId, pSyncNode->replicasId, sizeof(oldReplicasId)); SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; - memcpy(oldSenders, pSyncNode->senders, sizeof(oldSenders)); + for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { + oldSenders[i] = (pSyncNode->senders)[i]; + sDebug("vgId:%d sync event save senders %d, %p", pSyncNode->vgId, i, oldSenders[i]); + if (gRaftDetailLog) { + ; + } + } // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; @@ -1286,22 +1292,24 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l // reset new for (int i = 0; i < pSyncNode->replicaNum; ++i) { // reset sender + bool reset = false; for (int j = 0; j < TSDB_MAX_REPLICA; ++j) { if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j])) { char host[128]; uint16_t port; syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); - sDebug("vgId:%d sync event reset sender for %lu, %s:%d", pSyncNode->vgId, (pSyncNode->replicasId)[i].addr, host, - port); + sDebug("vgId:%d sync event reset sender for %lu, newIndex:%d, %s:%d, %p", pSyncNode->vgId, + (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j]); (pSyncNode->senders)[i] = oldSenders[j]; oldSenders[j] = NULL; + reset = true; + + // reset replicaIndex + int32_t oldreplicaIndex = (pSyncNode->senders)[i]->replicaIndex; + (pSyncNode->senders)[i]->replicaIndex = i; + sDebug("vgId:%d sync event udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", pSyncNode->vgId, + oldreplicaIndex, i, host, port, (pSyncNode->senders)[i], reset); } - - // reset replicaIndex - int32_t oldreplicaIndex = (pSyncNode->senders)[i]->replicaIndex; - (pSyncNode->senders)[i]->replicaIndex = i; - - sDebug("vgId:%d sync event udpate replicaIndex from %d to %d", pSyncNode->vgId, oldreplicaIndex, i); } } @@ -1309,7 +1317,7 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if ((pSyncNode->senders)[i] == NULL) { (pSyncNode->senders)[i] = snapshotSenderCreate(pSyncNode, i); - sDebug("vgId:%d sync event create new sender replicaIndex:%d", pSyncNode->vgId, i); + sDebug("vgId:%d sync event create new sender %p replicaIndex:%d", pSyncNode->vgId, (pSyncNode->senders)[i], i); } } @@ -1317,8 +1325,8 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if (oldSenders[i] != NULL) { snapshotSenderDestroy(oldSenders[i]); + sDebug("vgId:%d sync event delete old sender %p replicaIndex:%d", pSyncNode->vgId, oldSenders[i], i); oldSenders[i] = NULL; - sDebug("vgId:%d sync event delete old sender replicaIndex:%d", pSyncNode->vgId, i); } } @@ -1378,8 +1386,8 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { } void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { - sDebug("vgId:%d sync event become follower, isStandBy:%d, %s", pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, - debugStr); + sDebug("vgId:%d sync event become follower, isStandBy:%d, replicaNum:%d, %s", pSyncNode->vgId, + pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, debugStr); // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -1413,8 +1421,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // /\ UNCHANGED <> // void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { - sDebug("vgId:%d sync event become leader, isStandBy:%d, %s", pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, - debugStr); + sDebug("vgId:%d sync event become leader, isStandBy:%d, replicaNum:%d %s", pSyncNode->vgId, + pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, debugStr); // state change pSyncNode->state = TAOS_SYNC_STATE_LEADER; @@ -2028,14 +2036,18 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE // change isStandBy to normal if (!isDrop) { + char tmpbuf[128]; + snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d", oldSyncCfg.replicaNum, newSyncCfg.replicaNum); if (ths->state == TAOS_SYNC_STATE_LEADER) { - syncNodeBecomeLeader(ths, "config change"); + syncNodeBecomeLeader(ths, tmpbuf); } else { - syncNodeBecomeFollower(ths, "config change"); + syncNodeBecomeFollower(ths, tmpbuf); } } } else { - syncNodeBecomeFollower(ths, "config change2"); + char tmpbuf[128]; + snprintf(tmpbuf, sizeof(tmpbuf), "config change2 from %d to %d", oldSyncCfg.replicaNum, newSyncCfg.replicaNum); + syncNodeBecomeFollower(ths, tmpbuf); } if (gRaftDetailLog) { diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index d21a1e96e9..afebfa798e 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -588,6 +588,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // maybe update lastconfig if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) { + int32_t oldReplicaNum = pSyncNode->replicaNum; + // update new config myIndex bool IamInNew = false; SSyncCfg newSyncCfg = pMsg->lastConfig; @@ -614,10 +616,12 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // change isStandBy to normal if (!isDrop) { + char tmpbuf[128]; + snprintf(tmpbuf, sizeof(tmpbuf), "config change3 from %d to %d", oldReplicaNum, newSyncCfg.replicaNum); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - syncNodeBecomeLeader(pSyncNode, "config change3"); + syncNodeBecomeLeader(pSyncNode, tmpbuf); } else { - syncNodeBecomeFollower(pSyncNode, "config change3"); + syncNodeBecomeFollower(pSyncNode, tmpbuf); } } } From fa25fa32e0c56c2f3788493452e9ae77cf8cc755 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 14:54:11 +0800 Subject: [PATCH 63/78] refactor(sync): add test syncRaftIdChec --- source/libs/sync/test/syncRaftIdCheck.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/test/syncRaftIdCheck.cpp b/source/libs/sync/test/syncRaftIdCheck.cpp index f23c0db82b..90560e91e7 100644 --- a/source/libs/sync/test/syncRaftIdCheck.cpp +++ b/source/libs/sync/test/syncRaftIdCheck.cpp @@ -15,14 +15,14 @@ int main(int argc, char** argv) { char host[128]; uint16_t port; syncUtilU642Addr(u64, host, sizeof(host), &port); - printf("%s:%d \n", host, port); + printf("%lu -> %s:%d \n", u64, host, port); } else if (argc == 3) { uint64_t u64; char* host = argv[1]; uint16_t port = atoi(argv[2]); - syncUtilAddr2U64(host, port); - printf("%lu \n", u64); + u64 = syncUtilAddr2U64(host, port); + printf("%s:%d -> %lu \n", host, port, u64); } else { usage(argv[0]); exit(-1); From 4113472e89c69c1bd55036b03016346305409351 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 15:00:28 +0800 Subject: [PATCH 64/78] add avg function translate partial/merge --- include/libs/function/functionMgt.h | 2 + source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 53 +++++++++++++++++++++++++ source/libs/function/src/builtinsimpl.c | 2 + 4 files changed, 58 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index b5641f2d07..f324d8fed1 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -142,6 +142,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_FIRST_MERGE, FUNCTION_TYPE_LAST_PARTIAL, FUNCTION_TYPE_LAST_MERGE, + FUNCTION_TYPE_AVG_PARTIAL, + FUNCTION_TYPE_AVG_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 7db1396603..2f31d57907 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -58,6 +58,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx); int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t avgInvertFunction(SqlFunctionCtx* pCtx); int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); +int32_t getAvgInfoSize(); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1c3f274ee9..4fd0db4819 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -155,6 +155,35 @@ static int32_t translateSum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return TSDB_CODE_SUCCESS; } +static int32_t translateAvgPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = getAvgInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateAvgMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (TSDB_DATA_TYPE_BINARY != paraType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; + + return TSDB_CODE_SUCCESS; +} + static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; @@ -1518,6 +1547,30 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .invertFunc = avgInvertFunction, .combineFunc = avgCombine, }, + { + .name = "_avg_partial", + .type = FUNCTION_TYPE_AVG_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = getAvgFuncEnv, + .initFunc = avgFunctionSetup, + .processFunc = avgFunction, + .finalizeFunc = avgFinalize, + .invertFunc = avgInvertFunction, + .combineFunc = avgCombine, + }, + { + .name = "_avg_merge", + .type = FUNCTION_TYPE_AVG_MERGE, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = getAvgFuncEnv, + .initFunc = avgFunctionSetup, + .processFunc = avgFunction, + .finalizeFunc = avgFinalize, + .invertFunc = avgInvertFunction, + .combineFunc = avgCombine, + }, { .name = "percentile", .type = FUNCTION_TYPE_PERCENTILE, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3510a59254..ee47b1aa7f 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -624,6 +624,8 @@ bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } +int32_t getAvgInfoSize() { return (int32_t)sizeof(SAvgRes); } + bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SAvgRes); return true; From 5881ef26905b46e36cd2f054a3d837f47948a487 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 14:28:41 +0800 Subject: [PATCH 65/78] refactor: convert datablock to submit block --- include/common/tdatablock.h | 3 - source/common/src/tdatablock.c | 167 -------------------------- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqOffset.c | 19 ++- source/dnode/vnode/src/tq/tqSink.c | 169 +++++++++++++++++++++++++++ 5 files changed, 187 insertions(+), 173 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index af46535c94..709462a744 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -234,9 +234,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); -SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId); - static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock); } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4d1f5a34b1..b4ece426b3 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1728,173 +1728,6 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { return rname.childTableName; } -SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId) { - SSubmitReq* ret = NULL; - SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); - if (!tagArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - // cal size - int32_t cap = sizeof(SSubmitReq); - int32_t sz = taosArrayGetSize(pBlocks); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - int32_t rows = pDataBlock->info.rows; - // TODO min - int32_t rowSize = pDataBlock->info.rowSize; - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); - int32_t schemaLen = 0; - - if (createTb) { - SVCreateTbReq createTbReq = {0}; - char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); - createTbReq.name = cname; - createTbReq.flags = 0; - createTbReq.type = TSDB_CHILD_TABLE; - createTbReq.ctb.suid = suid; - - STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; - STag* pTag = NULL; - taosArrayClear(tagArray); - taosArrayPush(tagArray, &tagVal); - tTagNew(tagArray, 1, false, &pTag); - if (pTag == NULL) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - return NULL; - } - createTbReq.ctb.pTag = (uint8_t*)pTag; - - int32_t code; - tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - - tdDestroySVCreateTbReq(&createTbReq); - if (code < 0) { - taosArrayDestroy(tagArray); - return NULL; - } - } - - cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; - } - - // assign data - // TODO - ret = taosMemoryCalloc(1, cap + 46); - ret = POINTER_SHIFT(ret, 46); - ret->header.vgId = vgId; - ret->version = htonl(1); - ret->length = sizeof(SSubmitReq); - ret->numOfBlocks = htonl(sz); - - void* submitBlk = POINTER_SHIFT(ret, sizeof(SSubmitReq)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - - SSubmitBlk* blkHead = submitBlk; - blkHead->numOfRows = htons(pDataBlock->info.rows); - blkHead->sversion = htonl(pTSchema->version); - // TODO - blkHead->suid = htobe64(suid); - // uid is assigned by vnode - blkHead->uid = 0; - - int32_t rows = pDataBlock->info.rows; - /*int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);*/ - /*blkHead->dataLen = htonl(rows * maxLen);*/ - blkHead->dataLen = 0; - - void* blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk)); - - int32_t schemaLen = 0; - if (createTb) { - SVCreateTbReq createTbReq = {0}; - char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); - createTbReq.name = cname; - createTbReq.flags = 0; - createTbReq.type = TSDB_CHILD_TABLE; - createTbReq.ctb.suid = suid; - - STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; - taosArrayClear(tagArray); - taosArrayPush(tagArray, &tagVal); - STag* pTag = NULL; - tTagNew(tagArray, 1, false, &pTag); - if (pTag == NULL) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosMemoryFreeClear(ret); - return NULL; - } - createTbReq.ctb.pTag = (uint8_t*)pTag; - - int32_t code; - tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - if (code < 0) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosMemoryFreeClear(ret); - return NULL; - } - - SEncoder encoder = {0}; - tEncoderInit(&encoder, blockData, schemaLen); - code = tEncodeSVCreateTbReq(&encoder, &createTbReq); - tEncoderClear(&encoder); - tdDestroySVCreateTbReq(&createTbReq); - - if (code < 0) { - taosArrayDestroy(tagArray); - taosMemoryFreeClear(ret); - return NULL; - } - } - blkHead->schemaLen = htonl(schemaLen); - - STSRow* rowData = POINTER_SHIFT(blockData, schemaLen); - - for (int32_t j = 0; j < rows; j++) { - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); - tdSRowResetBuf(&rb, rowData); - - for (int32_t k = 0; k < pTSchema->numOfCols; k++) { - const STColumn* pColumn = &pTSchema->columns[k]; - SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); - if (colDataIsNull_s(pColData, j)) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, pColumn->offset, k); - } else { - void* data = colDataGetData(pColData, j); - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); - } - } - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - blkHead->dataLen += rowLen; - } - int32_t dataLen = blkHead->dataLen; - blkHead->dataLen = htonl(dataLen); - - ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; - blkHead = POINTER_SHIFT(blkHead, schemaLen + dataLen); - /*submitBlk = blkHead;*/ - } - - ret->length = htonl(ret->length); - taosArrayDestroy(tagArray); - return ret; -} - void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { // todo extract method diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 22685c1e19..9f34ae39c0 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -47,7 +47,7 @@ void tqCleanUp() { } STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { - STQ* pTq = taosMemoryMalloc(sizeof(STQ)); + STQ* pTq = taosMemoryCalloc(1, sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 2db49d7cf5..41444b7288 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -17,21 +17,33 @@ #include "tq.h" struct STqOffsetStore { + char* fname; STQ* pTq; SHashObj* pHash; // SHashObj }; +static char* buildFileName(const char* path) { + int32_t len = strlen(path); + char* fname = taosMemoryCalloc(1, len + 20); + snprintf(fname, len + 20, "%s/offset", path); + return fname; +} + STqOffsetStore* tqOffsetOpen(STQ* pTq) { STqOffsetStore* pStore = taosMemoryCalloc(1, sizeof(STqOffsetStore)); if (pStore == NULL) { return NULL; } + pStore->pTq = pTq; + pTq->pOffsetStore = pStore; + pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); if (pStore->pHash == NULL) { if (pStore->pHash) taosHashCleanup(pStore->pHash); return NULL; } - TdFilePtr pFile = taosOpenFile(pStore->pTq->path, TD_FILE_READ); + char* fname = buildFileName(pStore->pTq->path); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); if (pFile != NULL) { STqOffsetHead head = {0}; int64_t code; @@ -65,6 +77,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { } taosCloseFile(&pFile); + taosMemoryFree(fname); } return pStore; } @@ -85,7 +98,8 @@ int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) { int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { // open file // TODO file name should be with a version - TdFilePtr pFile = taosOpenFile(pStore->pTq->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + char* fname = buildFileName(pStore->pTq->path); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pFile == NULL) { ASSERT(0); return -1; @@ -124,5 +138,6 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { } // close and rename file taosCloseFile(&pFile); + taosMemoryFree(fname); return 0; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 5c0bf971fb..391a008440 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -15,6 +15,175 @@ #include "tq.h" +static SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, + const char* stbFullName, int32_t vgId); + +static SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid, + const char* stbFullName, int32_t vgId) { + SSubmitReq* ret = NULL; + SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + // cal size + int32_t cap = sizeof(SSubmitReq); + int32_t sz = taosArrayGetSize(pBlocks); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + int32_t rows = pDataBlock->info.rows; + // TODO min + int32_t rowSize = pDataBlock->info.rowSize; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + int32_t schemaLen = 0; + + if (createTb) { + SVCreateTbReq createTbReq = {0}; + char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + createTbReq.name = cname; + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; + + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .pData = (uint8_t*)&pDataBlock->info.groupId, + .nData = sizeof(uint64_t)}; + STag* pTag = NULL; + taosArrayClear(tagArray); + taosArrayPush(tagArray, &tagVal); + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return NULL; + } + createTbReq.ctb.pTag = (uint8_t*)pTag; + + int32_t code; + tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + + tdDestroySVCreateTbReq(&createTbReq); + if (code < 0) { + taosArrayDestroy(tagArray); + return NULL; + } + } + + cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; + } + + // assign data + // TODO + ret = rpcMallocCont(cap); + ret->header.vgId = vgId; + ret->version = htonl(1); + ret->length = sizeof(SSubmitReq); + ret->numOfBlocks = htonl(sz); + + void* submitBlk = POINTER_SHIFT(ret, sizeof(SSubmitReq)); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + + SSubmitBlk* blkHead = submitBlk; + blkHead->numOfRows = htons(pDataBlock->info.rows); + blkHead->sversion = htonl(pTSchema->version); + // TODO + blkHead->suid = htobe64(suid); + // uid is assigned by vnode + blkHead->uid = 0; + + int32_t rows = pDataBlock->info.rows; + /*int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);*/ + /*blkHead->dataLen = htonl(rows * maxLen);*/ + blkHead->dataLen = 0; + + void* blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk)); + + int32_t schemaLen = 0; + if (createTb) { + SVCreateTbReq createTbReq = {0}; + char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + createTbReq.name = cname; + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; + + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .pData = (uint8_t*)&pDataBlock->info.groupId, + .nData = sizeof(uint64_t)}; + taosArrayClear(tagArray); + taosArrayPush(tagArray, &tagVal); + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + taosMemoryFreeClear(ret); + return NULL; + } + createTbReq.ctb.pTag = (uint8_t*)pTag; + + int32_t code; + tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + taosMemoryFreeClear(ret); + return NULL; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, blockData, schemaLen); + code = tEncodeSVCreateTbReq(&encoder, &createTbReq); + tEncoderClear(&encoder); + tdDestroySVCreateTbReq(&createTbReq); + + if (code < 0) { + taosArrayDestroy(tagArray); + taosMemoryFreeClear(ret); + return NULL; + } + } + blkHead->schemaLen = htonl(schemaLen); + + STSRow* rowData = POINTER_SHIFT(blockData, schemaLen); + + for (int32_t j = 0; j < rows; j++) { + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTSchema->version); + tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); + tdSRowResetBuf(&rb, rowData); + + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pColumn = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, pColumn->offset, k); + } else { + void* data = colDataGetData(pColData, j); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); + } + } + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + blkHead->dataLen += rowLen; + } + int32_t dataLen = blkHead->dataLen; + blkHead->dataLen = htonl(dataLen); + + ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; + blkHead = POINTER_SHIFT(blkHead, schemaLen + dataLen); + /*submitBlk = blkHead;*/ + } + + ret->length = htonl(ret->length); + taosArrayDestroy(tagArray); + return ret; +} + void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pRes = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; From e82289542df8f1cb0c546b3e8db229f0efedaf89 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 15:15:37 +0800 Subject: [PATCH 66/78] avg function add partial/merge --- source/libs/function/inc/builtinsimpl.h | 2 + source/libs/function/src/builtinsimpl.c | 52 +++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 2f31d57907..1bb30934c6 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -55,7 +55,9 @@ int32_t maxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t avgFunction(SqlFunctionCtx* pCtx); +int32_t avgFunctionMerge(SqlFunctionCtx* pCtx); int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t avgInvertFunction(SqlFunctionCtx* pCtx); int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getAvgInfoSize(); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index ee47b1aa7f..fe9b4e1c3a 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -51,6 +51,7 @@ typedef struct SAvgRes { double result; SSumRes sum; int64_t count; + int16_t type; // store the original input type, used in merge function } SAvgRes; typedef struct STuplePos { @@ -649,6 +650,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { int32_t type = pInput->pData[0]->info.type; SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + pAvgRes->type = type; // computing based on the true data block SColumnInfoData* pCol = pInput->pData[0]; @@ -771,6 +773,38 @@ _avg_over: return TSDB_CODE_SUCCESS; } +static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) { + int16_t type = pInput->type; + if (IS_INTEGER_TYPE(type)) { + pOutput->sum.isum += pInput->sum.isum; + } else { + pOutput->sum.dsum += pInput->sum.dsum; + } + + pOutput->count += pInput->count; + + return; +} + +int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + + SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SAvgRes* pInputInfo; + + int32_t start = pInput->startRowIndex; + char* data = colDataGetData(pCol, start); + pInputInfo = (SAvgRes*)varDataVal(data); + + avgTransferInfo(pInputInfo, pInfo); + + SET_VAL(GET_RES_INFO(pCtx), 1, 1); + + return TSDB_CODE_SUCCESS; +} + #define LIST_AVG_N(sumT, T) \ do { \ T* plist = (T*)pCol->pData; \ @@ -872,6 +906,24 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return functionFinalize(pCtx, pBlock); } +int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t resultBytes = getAvgInfoSize(); + char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + + memcpy(varDataVal(res), pInfo, resultBytes); + varDataSetLen(res, resultBytes); + + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + colDataAppend(pCol, pBlock->info.rows, res, false); + + taosMemoryFree(res); + return pResInfo->numOfRes; +} + EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { return FUNC_DATA_REQUIRED_STATIS_LOAD; } From 79d9c04a4c6118069c3065434d7bc71815f73685 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 15:22:39 +0800 Subject: [PATCH 67/78] fix: check error code --- source/dnode/vnode/src/tq/tqOffset.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 41444b7288..8d6cb28065 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -50,7 +50,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { while (1) { if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { - if (code < 0) { + if (code == 0) { break; } else { ASSERT(0); From 18c0e0ac7411e223741f60bfda569bb029884b0e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 15:31:47 +0800 Subject: [PATCH 68/78] fix(query): cast --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c6cdd42584..6c3132a3ca 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1415,7 +1415,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t bytes = *(int32_t*)p; p += sizeof(int32_t); - ASSERT(type == pFields[i].type && bytes == pFields[i].bytes); + /*ASSERT(type == pFields[i].type && bytes == pFields[i].bytes);*/ } int32_t* colLength = (int32_t*)p; From 4ccbc4d85fe1caddfe2c171223afbb7fb06f69b0 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 15:34:55 +0800 Subject: [PATCH 69/78] enable avg function partial/merge --- source/libs/function/src/builtins.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4fd0db4819..4dae296f05 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1546,16 +1546,18 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = avgFinalize, .invertFunc = avgInvertFunction, .combineFunc = avgCombine, + .pPartialFunc = "_avg_partial", + .pMergeFunc = "_avg_merge" }, { .name = "_avg_partial", .type = FUNCTION_TYPE_AVG_PARTIAL, .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateInNumOutDou, + .translateFunc = translateAvgPartial, .getEnvFunc = getAvgFuncEnv, .initFunc = avgFunctionSetup, .processFunc = avgFunction, - .finalizeFunc = avgFinalize, + .finalizeFunc = avgPartialFinalize, .invertFunc = avgInvertFunction, .combineFunc = avgCombine, }, @@ -1563,10 +1565,10 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "_avg_merge", .type = FUNCTION_TYPE_AVG_MERGE, .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateInNumOutDou, + .translateFunc = translateAvgMerge, .getEnvFunc = getAvgFuncEnv, .initFunc = avgFunctionSetup, - .processFunc = avgFunction, + .processFunc = avgFunctionMerge, .finalizeFunc = avgFinalize, .invertFunc = avgInvertFunction, .combineFunc = avgCombine, From 300a3a2c0221f362839730358bfdbc8d92028be5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 15:46:16 +0800 Subject: [PATCH 70/78] enh: drop sma while drop stb and db --- source/dnode/mnode/impl/inc/mndSma.h | 2 + source/dnode/mnode/impl/src/mndDb.c | 1 + source/dnode/mnode/impl/src/mndSma.c | 139 ++++++++++++++++++++++----- source/dnode/mnode/impl/src/mndStb.c | 33 ++++--- 4 files changed, 137 insertions(+), 38 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 4a80f619d3..5d13f8b74d 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,6 +26,8 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); +int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb); +int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index cfe363fdf0..43263af573 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -935,6 +935,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER; + if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; SUserObj *pUser = mndAcquireUser(pMnode, pDb->createUser); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index c19b558f19..cc57076c0a 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -38,10 +38,10 @@ static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma); static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSpSmatb); static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew); static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups); -static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq); -static int32_t mndProcessMDropSmaReq(SRpcMsg *pReq); +static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq); +static int32_t mndProcessDropSmaReq(SRpcMsg *pReq); static int32_t mndProcessGetSmaReq(SRpcMsg *pReq); -static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq); +static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq); static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); @@ -56,8 +56,8 @@ int32_t mndInitSma(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndSmaActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessMCreateSmaReq); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessCreateSmaReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessDropSmaReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetSmaReq); @@ -79,7 +79,6 @@ static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) @@ -100,6 +99,7 @@ static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { SDB_SET_INT32(pRaw, dataPos, pSma->tagsFilterLen, _OVER) SDB_SET_INT32(pRaw, dataPos, pSma->sqlLen, _OVER) SDB_SET_INT32(pRaw, dataPos, pSma->astLen, _OVER) + if (pSma->exprLen > 0) { SDB_SET_BINARY(pRaw, dataPos, pSma->expr, pSma->exprLen, _OVER) } @@ -115,6 +115,7 @@ static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { SDB_SET_RESERVE(pRaw, dataPos, TSDB_SMA_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) + terrno = 0; _OVER: @@ -193,6 +194,7 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, TSDB_SMA_RESERVE_SIZE, _OVER) + terrno = 0; _OVER: @@ -383,6 +385,25 @@ static int32_t mndSetCreateSmaVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, S return 0; } +static int32_t mndSetUpdateSmaStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { + SStbObj stbObj = {0}; + taosRLockLatch(&pStb->lock); + memcpy(&stbObj, pStb, sizeof(SStbObj)); + taosRUnLockLatch(&pStb->lock); + stbObj.pColumns = NULL; + stbObj.pTags = NULL; + stbObj.updateTime = taosGetTimestampMs(); + stbObj.lock = 0; + stbObj.smaVer++; + + SSdbRaw *pCommitRaw = mndStbActionEncode(&stbObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + static int32_t mndSetCreateSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; @@ -457,7 +478,6 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, pSma->schemaTag.pSchema[0].flags = 0; snprintf(pSma->schemaTag.pSchema[0].name, TSDB_COL_NAME_LEN, "groupId"); - int32_t smaContLen = 0; void *pSmaReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &smaContLen); if (pSmaReq == NULL) return -1; @@ -559,6 +579,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndSetCreateSmaVgroupRedoLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupCommitLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; + if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER; if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, STREAM_TRIGGER_AT_ONCE, 0, pTrans) != 0) goto _OVER; @@ -599,7 +620,7 @@ static int32_t mndCheckCreateSmaReq(SMCreateSmaReq *pCreate) { return 0; } -static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq) { +static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SStbObj *pStb = NULL; @@ -781,13 +802,17 @@ static int32_t mndSetDropSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SD } static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *pSma) { - int32_t code = -1; - SVgObj *pVgroup = NULL; - STrans *pTrans = NULL; + int32_t code = -1; + SVgObj *pVgroup = NULL; + SStbObj *pStb = NULL; + STrans *pTrans = NULL; pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); if (pVgroup == NULL) goto _OVER; + pStb = mndAcquireStb(pMnode, pSma->stb); + if (pStb == NULL) goto _OVER; + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; @@ -798,6 +823,7 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p if (mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; + if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER; if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -807,10 +833,78 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p _OVER: mndTransDrop(pTrans); mndReleaseVgroup(pMnode, pVgroup); + mndReleaseStb(pMnode, pStb); return code; } -static int32_t mndProcessMDropSmaReq(SRpcMsg *pReq) { +int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + SSmaObj *pSma = NULL; + void *pIter = NULL; + SVgObj *pVgroup = NULL; + int32_t code = -1; + + while (1) { + pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); + if (pIter == NULL) break; + + if (pSma->stbUid == pStb->uid) { + pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); + if (pVgroup == NULL) goto _OVER; + if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; + if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; + if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; + if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER; + mndReleaseVgroup(pMnode, pVgroup); + pVgroup = NULL; + } + + sdbRelease(pSdb, pSma); + } + + code = 0; + +_OVER: + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pSma); + mndReleaseVgroup(pMnode, pVgroup); + return code; +} + +int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + SSmaObj *pSma = NULL; + void *pIter = NULL; + SVgObj *pVgroup = NULL; + int32_t code = -1; + + while (1) { + pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); + if (pIter == NULL) break; + + if (pSma->dbUid == pDb->uid) { + pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); + if (pVgroup == NULL) goto _OVER; + if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; + if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; + if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; + mndReleaseVgroup(pMnode, pVgroup); + pVgroup = NULL; + } + + sdbRelease(pSdb, pSma); + } + + code = 0; + +_OVER: + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pSma); + mndReleaseVgroup(pMnode, pVgroup); + return code; +} + +static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SUserObj *pUser = NULL; @@ -900,10 +994,10 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp } static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIndexRsp *rsp, bool *exist) { - int32_t code = 0; - SSmaObj *pSma = NULL; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + int32_t code = 0; + SSmaObj *pSma = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; STableIndexInfo info; while (1) { @@ -922,14 +1016,14 @@ static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIn info.dstTbUid = pSma->dstTbUid; info.dstVgId = pSma->dstVgId; - SVgObj* pVg = mndAcquireVgroup(pMnode, pSma->dstVgId); + SVgObj *pVg = mndAcquireVgroup(pMnode, pSma->dstVgId); if (pVg == NULL) { code = -1; sdbRelease(pSdb, pSma); return code; } info.epSet = mndGetVgroupEpset(pMnode, pVg); - + info.expr = taosMemoryMalloc(pSma->exprLen + 1); if (info.expr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -953,7 +1047,7 @@ static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIn sdbRelease(pSdb, pSma); } - + return code; } @@ -1005,10 +1099,10 @@ _OVER: static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { STableIndexReq indexReq = {0}; - SMnode *pMnode = pReq->info.node; - int32_t code = -1; + SMnode *pMnode = pReq->info.node; + int32_t code = -1; STableIndexRsp rsp = {0}; - bool exist = false; + bool exist = false; if (tDeserializeSTableIndexReq(pReq->pCont, pReq->contLen, &indexReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -1055,7 +1149,6 @@ _OVER: return code; } - static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index ad0c913fb8..cb0a893436 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -23,6 +23,7 @@ #include "mndPerfSchema.h" #include "mndScheduler.h" #include "mndShow.h" +#include "mndSma.h" #include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" @@ -36,9 +37,9 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew); -static int32_t mndProcessMCreateStbReq(SRpcMsg *pReq); -static int32_t mndProcessMAlterStbReq(SRpcMsg *pReq); -static int32_t mndProcessMDropStbReq(SRpcMsg *pReq); +static int32_t mndProcessCreateStbReq(SRpcMsg *pReq); +static int32_t mndProcessAlterStbReq(SRpcMsg *pReq); +static int32_t mndProcessDropStbReq(SRpcMsg *pReq); static int32_t mndProcessTableMetaReq(SRpcMsg *pReq); static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); @@ -54,9 +55,9 @@ int32_t mndInitStb(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndStbActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcessMCreateStbReq); - mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcessMAlterStbReq); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_STB, mndProcessMDropStbReq); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcessCreateStbReq); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcessAlterStbReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_STB, mndProcessDropStbReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndTransProcessRsp); @@ -318,6 +319,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { pOld->updateTime = pNew->updateTime; pOld->tagVer = pNew->tagVer; pOld->colVer = pNew->colVer; + pOld->smaVer = pNew->smaVer; pOld->nextColId = pNew->nextColId; pOld->ttl = pNew->ttl; pOld->numOfColumns = pNew->numOfColumns; @@ -361,7 +363,7 @@ SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { return mndAcquireDb(pMnode, db); } -static FORCE_INLINE int schemaExColIdCompare(const void *colId, const void *pSchema) { +static FORCE_INLINE int32_t schemaExColIdCompare(const void *colId, const void *pSchema) { if (*(col_id_t *)colId < ((SSchema *)pSchema)->colId) { return -1; } else if (*(col_id_t *)colId > ((SSchema *)pSchema)->colId) { @@ -395,14 +397,14 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.pRSmaParam.xFilesFactor = pStb->xFilesFactor; req.pRSmaParam.delay = pStb->delay; if (pStb->ast1Len > 0) { - if (mndConvertRsmaTask(&req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, pStb->pAst1, pStb->uid, STREAM_TRIGGER_AT_ONCE, 0, - req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { + if (mndConvertRsmaTask(&req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, pStb->pAst1, pStb->uid, + STREAM_TRIGGER_AT_ONCE, 0, req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } } if (pStb->ast2Len > 0) { - if (mndConvertRsmaTask(&req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, pStb->pAst2, pStb->uid, STREAM_TRIGGER_AT_ONCE, 0, - req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { + if (mndConvertRsmaTask(&req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, pStb->pAst2, pStb->uid, + STREAM_TRIGGER_AT_ONCE, 0, req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } } @@ -761,7 +763,7 @@ int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p return 0; } -static int32_t mndProcessMCreateStbReq(SRpcMsg *pReq) { +static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SStbObj *pStb = NULL; @@ -1296,7 +1298,7 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, int32_t *pLen) { - int ret; + int32_t ret; SEncoder ec = {0}; uint32_t contLen = 0; SMAlterStbRsp alterRsp = {0}; @@ -1415,7 +1417,7 @@ _OVER: return code; } -static int32_t mndProcessMAlterStbReq(SRpcMsg *pReq) { +static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SDbObj *pDb = NULL; @@ -1545,6 +1547,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndDropSmasByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -1554,7 +1557,7 @@ _OVER: return code; } -static int32_t mndProcessMDropStbReq(SRpcMsg *pReq) { +static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SUserObj *pUser = NULL; From a17c6e3c33b895865e120ba146b86c7c208f20ae Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 15:47:56 +0800 Subject: [PATCH 71/78] fix bugs --- source/libs/function/src/builtinsimpl.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fe9b4e1c3a..5ee9dbdee6 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -774,8 +774,8 @@ _avg_over: } static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) { - int16_t type = pInput->type; - if (IS_INTEGER_TYPE(type)) { + pOutput->type = pInput->type; + if (IS_INTEGER_TYPE(pOutput->type)) { pOutput->sum.isum += pInput->sum.isum; } else { pOutput->sum.dsum += pInput->sum.dsum; @@ -792,11 +792,10 @@ int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) { ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - SAvgRes* pInputInfo; int32_t start = pInput->startRowIndex; char* data = colDataGetData(pCol, start); - pInputInfo = (SAvgRes*)varDataVal(data); + SAvgRes* pInputInfo = (SAvgRes*)varDataVal(data); avgTransferInfo(pInputInfo, pInfo); @@ -889,8 +888,8 @@ int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; - int32_t type = pInput->pData[0]->info.type; SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t type = pAvgRes->type; if (IS_INTEGER_TYPE(type)) { pAvgRes->result = pAvgRes->sum.isum / ((double)pAvgRes->count); @@ -3292,11 +3291,10 @@ int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx) { ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - SSpreadInfo* pInputInfo; int32_t start = pInput->startRowIndex; char* data = colDataGetData(pCol, start); - pInputInfo = (SSpreadInfo*)varDataVal(data); + SSpreadInfo* pInputInfo = (SSpreadInfo*)varDataVal(data); spreadTransferInfo(pInputInfo, pInfo); From c9509acf222bc3777a4bca3fea29d1c1cf1ffa0a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 16:01:32 +0800 Subject: [PATCH 72/78] refactor: mnode pre process msg --- include/dnode/mnode/mnode.h | 20 +++++++++++++++++--- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 3 ++- source/dnode/mnode/impl/src/mndQuery.c | 7 +++---- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 99b704826d..21d8405f58 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -59,7 +59,13 @@ void mndClose(SMnode *pMnode); * @param pMnode The mnode object. */ int32_t mndStart(SMnode *pMnode); -void mndStop(SMnode *pMnode); + +/** + * @brief Stop mnode + * + * @param pMnode The mnode object. + */ +void mndStop(SMnode *pMnode); /** * @brief Get mnode monitor info. @@ -71,17 +77,25 @@ void mndStop(SMnode *pMnode); * @return int32_t 0 for success, -1 for failure. */ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pCluster, SMonVgroupInfo *pVgroup, SMonGrantInfo *pGrant); + +/** + * @brief Get mnode loads for status msg. + * + * @param pMnode The mnode object. + * @param pLoad + * @return int32_t 0 for success, -1 for failure. + */ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); /** - * @brief Process the read, write, sync request. + * @brief Process the rpc, sync request. * * @param pMsg The request msg. * @return int32_t 0 for success, -1 for failure. */ int32_t mndProcessRpcMsg(SRpcMsg *pMsg); int32_t mndProcessSyncMsg(SRpcMsg *pMsg); -int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg); +int32_t mndPreProcessMsg(SRpcMsg *pMsg); /** * @brief Generate machine code diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index ee2df5f089..7cd7da1aa9 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -114,7 +114,8 @@ int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - if (mndPreprocessQueryMsg(pMgmt->pMnode, pMsg) != 0) { + pMsg->info.node = pMgmt->pMnode; + if (mndPreProcessMsg(pMsg) != 0) { dError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); return -1; } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 12b39e5b78..5374f48e47 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -18,11 +18,10 @@ #include "mndMnode.h" #include "qworker.h" -int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg) { - if (TDMT_VND_QUERY != pMsg->msgType) { - return 0; - } +int32_t mndPreProcessMsg(SRpcMsg *pMsg) { + if (TDMT_VND_QUERY != pMsg->msgType) return 0; + SMnode *pMnode = pMsg->info.node; return qWorkerPreprocessQueryMsg(pMnode->pQuery, pMsg); } From 72bc32af1e16ef1af3496e7ac4db30aa0747519c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 16:17:37 +0800 Subject: [PATCH 73/78] ehn: remove sync ref --- source/dnode/mnode/impl/src/mndMain.c | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 3b2eafced8..bd82701ae4 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -393,11 +393,6 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { return TAOS_SYNC_OTHER_ERROR; } - if (mndAcquireSyncRef(pMnode) != 0) { - mError("failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); - return TAOS_SYNC_OTHER_ERROR; - } - char logBuf[512] = {0}; char *syncNodeStr = sync2SimpleStr(pMgmt->sync); snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); @@ -501,7 +496,6 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { } } - mndReleaseSyncRef(pMnode); return code; } @@ -754,24 +748,3 @@ void mndSetStop(SMnode *pMnode) { } bool mndGetStop(SMnode *pMnode) { return pMnode->stopped; } - -int32_t mndAcquireSyncRef(SMnode *pMnode) { - int32_t code = 0; - taosThreadRwlockRdlock(&pMnode->lock); - if (pMnode->stopped) { - terrno = TSDB_CODE_APP_NOT_READY; - code = -1; - } else { - int32_t ref = atomic_add_fetch_32(&pMnode->syncRef, 1); - // mTrace("mnode sync is acquired, ref:%d", ref); - } - taosThreadRwlockUnlock(&pMnode->lock); - return code; -} - -void mndReleaseSyncRef(SMnode *pMnode) { - taosThreadRwlockRdlock(&pMnode->lock); - int32_t ref = atomic_sub_fetch_32(&pMnode->syncRef, 1); - // mTrace("mnode sync is released, ref:%d", ref); - taosThreadRwlockUnlock(&pMnode->lock); -} From 04593fa44ec4e37d13832ec3e8ce4efe07da24f5 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 16:38:33 +0800 Subject: [PATCH 74/78] fix(stream): build table name --- source/client/src/tmq.c | 12 ++++++++---- source/common/src/tdatablock.c | 1 + source/dnode/vnode/src/tq/tqSink.c | 20 +++++++++++--------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 48c85cf265..10c6c3623a 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -199,8 +199,8 @@ typedef struct { int8_t automatic; int8_t async; int8_t freeOffsets; - int8_t waitingRspNum; - int8_t totalRspNum; + int32_t waitingRspNum; + int32_t totalRspNum; tmq_resp_err_t rspErr; tmq_commit_cb* userCb; SArray* successfulOffsets; @@ -373,8 +373,9 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { } else { taosArrayPush(pParamSet->successfulOffsets, &pParam->pOffset); } + // count down waiting rsp - int8_t waitingRspNum = atomic_sub_fetch_8(&pParam->params->waitingRspNum, 1); + int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); ASSERT(waitingRspNum >= 0); if (waitingRspNum == 0) { @@ -395,7 +396,8 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { return 0; } -int32_t tmqComitInner2(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { +int32_t tmqCommitInner2(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_t automatic, int8_t async, + tmq_commit_cb* userCb, void* userParam) { int32_t code = -1; SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); @@ -466,6 +468,8 @@ int32_t tmqComitInner2(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); + pParamSet->waitingRspNum++; + pParamSet->totalRspNum++; } } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b4ece426b3..5a2aaed74e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1708,6 +1708,7 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { pTag->keyLen = strlen(pTag->key); pTag->type = TSDB_DATA_TYPE_UBIGINT; pTag->u = groupId; + pTag->length = sizeof(uint64_t); taosArrayPush(tags, &pTag); void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 391a008440..0cca1d2e10 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -46,11 +46,12 @@ static SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSche createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; - STag* pTag = NULL; + STagVal tagVal = { + .cid = pDataBlock->info.numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, + }; + STag* pTag = NULL; taosArrayClear(tagArray); taosArrayPush(tagArray, &tagVal); tTagNew(tagArray, 1, false, &pTag); @@ -110,10 +111,11 @@ static SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSche createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; + STagVal tagVal = { + .cid = pDataBlock->info.numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, + }; taosArrayClear(tagArray); taosArrayPush(tagArray, &tagVal); STag* pTag = NULL; From cd94eabc73440242b7d81ae4f5ab239b7409c146 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 16:42:05 +0800 Subject: [PATCH 75/78] add stddev function translate partial/merge --- include/libs/function/functionMgt.h | 2 + source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 53 +++++++++++++++++++++++++ source/libs/function/src/builtinsimpl.c | 2 + 4 files changed, 58 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index f324d8fed1..a73b9b47f4 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -144,6 +144,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_LAST_MERGE, FUNCTION_TYPE_AVG_PARTIAL, FUNCTION_TYPE_AVG_MERGE, + FUNCTION_TYPE_STDDEV_PARTIAL, + FUNCTION_TYPE_STDDEV_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 1bb30934c6..0d08f97c77 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -68,6 +68,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx); int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t stddevInvertFunction(SqlFunctionCtx* pCtx); int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); +int32_t getStddevInfoSize(); bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool leastSQRFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4dae296f05..0099f9e8c4 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -184,6 +184,35 @@ static int32_t translateAvgMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t le return TSDB_CODE_SUCCESS; } +static int32_t translateStddevPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = getStddevInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateStddevMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (TSDB_DATA_TYPE_BINARY != paraType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; + + return TSDB_CODE_SUCCESS; +} + static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; @@ -1523,6 +1552,30 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .invertFunc = stddevInvertFunction, .combineFunc = stddevCombine, }, + { + .name = "_stddev_partial", + .type = FUNCTION_TYPE_STDDEV_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateStddevPartial, + .getEnvFunc = getStddevFuncEnv, + .initFunc = stddevFunctionSetup, + .processFunc = stddevFunction, + .finalizeFunc = stddevFinalize, + .invertFunc = stddevInvertFunction, + .combineFunc = stddevCombine, + }, + { + .name = "_stddev_merge", + .type = FUNCTION_TYPE_STDDEV_MERGE, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateStddevMerge, + .getEnvFunc = getStddevFuncEnv, + .initFunc = stddevFunctionSetup, + .processFunc = stddevFunction, + .finalizeFunc = stddevFinalize, + .invertFunc = stddevInvertFunction, + .combineFunc = stddevCombine, + }, { .name = "leastsquares", .type = FUNCTION_TYPE_LEASTSQUARES, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5ee9dbdee6..648a60c299 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1488,6 +1488,8 @@ int32_t maxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { return minMaxCombine(pDestCtx, pSourceCtx, 0); } +int32_t getStddevInfoSize() { return (int32_t)sizeof(SStddevRes); } + bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SStddevRes); return true; From 68d33b1656abd0f3267f791634da99ba31298e96 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 16:53:59 +0800 Subject: [PATCH 76/78] add stddev function partial/merge --- source/libs/function/src/builtinsimpl.c | 56 ++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 648a60c299..c6ab9f859d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -89,6 +89,7 @@ typedef struct SStddevRes { double dsum; int64_t isum; }; + int16_t type; } SStddevRes; typedef struct SLeastSQRInfo { @@ -1513,6 +1514,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { int32_t type = pInput->pData[0]->info.type; SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + pStddevRes->type = type; // computing based on the true data block SColumnInfoData* pCol = pInput->pData[0]; @@ -1629,6 +1631,39 @@ _stddev_over: return TSDB_CODE_SUCCESS; } +static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) { + pOutput->type = pInput->type; + if (IS_INTEGER_TYPE(pOutput->type)) { + pOutput->quadraticISum += pInput->quadraticISum; + pOutput->isum += pInput->isum; + } else { + pOutput->quadraticDSum += pInput->quadraticDSum; + pOutput->dsum += pInput->dsum; + } + + pOutput->count += pInput->count; + + return; +} + +int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + + SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + int32_t start = pInput->startRowIndex; + char* data = colDataGetData(pCol, start); + SStddevRes* pInputInfo = (SStddevRes*)varDataVal(data); + + stddevTransferInfo(pInputInfo, pInfo); + + SET_VAL(GET_RES_INFO(pCtx), 1, 1); + + return TSDB_CODE_SUCCESS; +} + #define LIST_STDDEV_SUB_N(sumT, T) \ do { \ T* plist = (T*)pCol->pData; \ @@ -1694,9 +1729,10 @@ int32_t stddevInvertFunction(SqlFunctionCtx* pCtx) { int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; - int32_t type = pInput->pData[0]->info.type; SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t type = pStddevRes->type; double avg; + if (IS_INTEGER_TYPE(type)) { avg = pStddevRes->isum / ((double)pStddevRes->count); pStddevRes->result = sqrt(pStddevRes->quadraticISum / ((double)pStddevRes->count) - avg * avg); @@ -1708,6 +1744,24 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return functionFinalize(pCtx, pBlock); } +int32_t stddevPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t resultBytes = getStddevInfoSize(); + char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + + memcpy(varDataVal(res), pInfo, resultBytes); + varDataSetLen(res, resultBytes); + + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + colDataAppend(pCol, pBlock->info.rows, res, false); + + taosMemoryFree(res); + return pResInfo->numOfRes; +} + int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SStddevRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); From 7ed7fd0fd681689f86515ed7f4f34f8d1ce61fcf Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 17:00:32 +0800 Subject: [PATCH 77/78] enable stddev splitting --- source/libs/function/inc/builtinsimpl.h | 2 ++ source/libs/function/src/builtins.c | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 0d08f97c77..f3060243ed 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -65,7 +65,9 @@ int32_t getAvgInfoSize(); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t stddevFunction(SqlFunctionCtx* pCtx); +int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx); int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t stddevPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t stddevInvertFunction(SqlFunctionCtx* pCtx); int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getStddevInfoSize(); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 0099f9e8c4..4ffdf44cd6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1551,6 +1551,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = stddevFinalize, .invertFunc = stddevInvertFunction, .combineFunc = stddevCombine, + .pPartialFunc = "_stddev_partial", + .pMergeFunc = "_stddev_merge" }, { .name = "_stddev_partial", @@ -1560,7 +1562,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getStddevFuncEnv, .initFunc = stddevFunctionSetup, .processFunc = stddevFunction, - .finalizeFunc = stddevFinalize, + .finalizeFunc = stddevPartialFinalize, .invertFunc = stddevInvertFunction, .combineFunc = stddevCombine, }, @@ -1571,7 +1573,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateStddevMerge, .getEnvFunc = getStddevFuncEnv, .initFunc = stddevFunctionSetup, - .processFunc = stddevFunction, + .processFunc = stddevFunctionMerge, .finalizeFunc = stddevFinalize, .invertFunc = stddevInvertFunction, .combineFunc = stddevCombine, From 1248b5fbb345b1b0f80b27e9fa7e52dad5f7db80 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 16:57:42 +0800 Subject: [PATCH 78/78] enh(stream): forbid indefinite row functions for streams --- source/libs/function/src/builtins.c | 33 +++++++++++++++-------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1c3f274ee9..ed3d838b8c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -393,7 +393,8 @@ static int32_t translateTopBotImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t pValue->notReserved = true; // set result type - pFunc->node.resType = (SDataType){.bytes = getTopBotInfoSize(pValue->datum.i) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + pFunc->node.resType = + (SDataType){.bytes = getTopBotInfoSize(pValue->datum.i) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; } else { if (1 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -956,11 +957,11 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { // first(col_list) will be rewritten as first(col) - if (2 != LIST_LENGTH(pFunc->pParameterList)) { //input has two params c0,ts, is this a bug? + if (2 != LIST_LENGTH(pFunc->pParameterList)) { // input has two params c0,ts, is this a bug? return TSDB_CODE_SUCCESS; } - SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); uint8_t paraType = ((SExprNode*)pPara)->resType.type; int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes; if (isPartial) { @@ -969,8 +970,8 @@ static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32 "The parameters of first/last can only be columns"); } - pFunc->node.resType = (SDataType){.bytes = getFirstLastInfoSize(paraBytes) + VARSTR_HEADER_SIZE, - .type = TSDB_DATA_TYPE_BINARY}; + pFunc->node.resType = + (SDataType){.bytes = getFirstLastInfoSize(paraBytes) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; } else { if (TSDB_DATA_TYPE_BINARY != paraType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); @@ -1521,7 +1522,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "percentile", .type = FUNCTION_TYPE_PERCENTILE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_REPEAT_SCAN_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_REPEAT_SCAN_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translatePercentile, .getEnvFunc = getPercentileFuncEnv, .initFunc = percentileFunctionSetup, @@ -1682,7 +1683,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "elapsed", .type = FUNCTION_TYPE_ELAPSED, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .dataRequiredFunc = statisDataRequired, .translateFunc = translateElapsed, .getEnvFunc = getElapsedFuncEnv, @@ -1803,7 +1804,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "twa", .type = FUNCTION_TYPE_TWA, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateInNumOutDou, .getEnvFunc = getTwaFuncEnv, .initFunc = twaFunctionSetup, @@ -1889,7 +1890,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "diff", .type = FUNCTION_TYPE_DIFF, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateDiff, .getEnvFunc = getDiffFuncEnv, .initFunc = diffFunctionSetup, @@ -1899,7 +1900,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "statecount", .type = FUNCTION_TYPE_STATE_COUNT, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateStateCount, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -1909,7 +1910,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "stateduration", .type = FUNCTION_TYPE_STATE_DURATION, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateStateDuration, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -1919,7 +1920,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "csum", .type = FUNCTION_TYPE_CSUM, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateCsum, .getEnvFunc = getCsumFuncEnv, .initFunc = functionSetup, @@ -1929,7 +1930,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "mavg", .type = FUNCTION_TYPE_MAVG, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateMavg, .getEnvFunc = getMavgFuncEnv, .initFunc = mavgFunctionSetup, @@ -1939,7 +1940,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "sample", .type = FUNCTION_TYPE_SAMPLE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateSample, .getEnvFunc = getSampleFuncEnv, .initFunc = sampleFunctionSetup, @@ -1949,7 +1950,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "tail", .type = FUNCTION_TYPE_TAIL, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateTail, .getEnvFunc = getTailFuncEnv, .initFunc = tailFunctionSetup, @@ -1959,7 +1960,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "unique", .type = FUNCTION_TYPE_UNIQUE, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateUnique, .getEnvFunc = getUniqueFuncEnv, .initFunc = uniqueFunctionSetup,