From abcd8a8f5bef55e434b1baec0e0871b42f8e348a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Jun 2022 16:11:56 +0800 Subject: [PATCH 001/124] q --- source/client/test/smlTest.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 832564e0db..25cbb42ee9 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -913,7 +913,8 @@ TEST(testCase, smlParseTelnetLine_error_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + SSmlHandle *info = smlBuildSmlInfo(pTscObj, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); int32_t ret = 0; @@ -982,7 +983,8 @@ TEST(testCase, smlParseTelnetLine_json_error_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + SSmlHandle *info = smlBuildSmlInfo(pTscObj, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(info, nullptr); int32_t ret = 0; @@ -1069,7 +1071,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { " },\n" "]", }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(taos_errno(pRes), 0); taos_free_result(pRes); } @@ -1106,7 +1108,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { " },\n" "]", }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT_NE(taos_errno(pRes), 0); taos_free_result(pRes); } @@ -1257,7 +1259,7 @@ TEST(testCase, sml_16368_Test) { "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": {\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}},\n" "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": {\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]" }; - pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); + pRes = taos_schemaless_insert(taos, (char**)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); ASSERT_EQ(taos_errno(pRes), 0); taos_free_result(pRes); } @@ -1283,5 +1285,5 @@ TEST(testCase, sml_dup_time_Test) { pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); ASSERT_EQ(taos_errno(pRes), 0); taos_free_result(pRes); -} -*/ +}*/ + From 986d5c77525fb87e468ea5060d210caa83f0312a Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 12 Jul 2022 14:45:00 +0800 Subject: [PATCH 002/124] test:add test case for udf coltrol --- tests/system-test/0-others/udf_cfg1.py | 305 +++++++++++ tests/system-test/0-others/udf_cfg2.py | 667 +++++++++++++++++++++++++ tests/system-test/fulltest.sh | 2 + 3 files changed, 974 insertions(+) create mode 100644 tests/system-test/0-others/udf_cfg1.py create mode 100644 tests/system-test/0-others/udf_cfg2.py diff --git a/tests/system-test/0-others/udf_cfg1.py b/tests/system-test/0-others/udf_cfg1.py new file mode 100644 index 0000000000..fc9265617d --- /dev/null +++ b/tests/system-test/0-others/udf_cfg1.py @@ -0,0 +1,305 @@ +from distutils.log import error +import taos +import sys +import time +import os +import platform + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +import subprocess + +class TDTestCase: + updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, + "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, + "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143 ,"udf":0} + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def prepare_udf_so(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + print(projPath) + + if platform.system().lower() == 'windows': + self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + if (not tdDnodes.dnodes[0].remoteIP == ""): + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + else: + self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf2 = self.libudf2.replace('\r','').replace('\n','') + + + def prepare_data(self): + + tdSql.execute("drop database if exists db ") + tdSql.execute("create database if not exists db duration 300") + tdSql.execute("use db") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdSql.execute("create table tb (ts timestamp , num1 int , num2 int, num3 double , num4 binary(30))") + tdSql.execute( + f'''insert into tb values + ( '2020-04-21 01:01:01.000', NULL, 1, 1, "binary1" ) + ( '2020-10-21 01:01:01.000', 1, 1, 1.11, "binary1" ) + ( '2020-12-31 01:01:01.000', 2, 22222, 22, "binary1" ) + ( '2021-01-01 01:01:06.000', 3, 33333, 33, "binary1" ) + ( '2021-05-07 01:01:10.000', 4, 44444, 44, "binary1" ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, "binary1" ) + ( '2021-09-30 01:01:16.000', 5, 55555, 55, "binary1" ) + ( '2022-02-01 01:01:20.000', 6, 66666, 66, "binary1" ) + ( '2022-10-28 01:01:26.000', 0, 00000, 00, "binary1" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -88, "binary1" ) + ( '2022-12-31 01:01:36.000', 9, -9999999, -99, "binary1" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, "binary1" ) + ''' + ) + + # udf functions with join + ts_start = 1652517451000 + tdSql.execute("create stable st (ts timestamp , c1 int , c2 int ,c3 double ,c4 double ) tags(ind int)") + tdSql.execute("create table sub1 using st tags(1)") + tdSql.execute("create table sub2 using st tags(2)") + + for i in range(10): + ts = ts_start + i *1000 + tdSql.execute(" insert into sub1 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0)) + tdSql.execute(" insert into sub2 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0)) + + + def create_udf_function(self): + + for i in range(5): + # create scalar functions + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + + # create aggregate functions + + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) + + functions = tdSql.getResult("show functions") + function_nums = len(functions) + if function_nums == 2: + tdLog.info("create two udf functions success ") + + # drop functions + + tdSql.execute("drop function udf1") + tdSql.execute("drop function udf2") + + functions = tdSql.getResult("show functions") + for function in functions: + if "udf1" in function[0] or "udf2" in function[0]: + tdLog.info("drop udf functions failed ") + tdLog.exit("drop udf functions failed") + + tdLog.info("drop two udf functions success ") + + # create scalar functions + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + + # create aggregate functions + + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) + + functions = tdSql.getResult("show functions") + function_nums = len(functions) + if function_nums == 2: + tdLog.info("create two udf functions success ") + + def basic_udf_query(self): + + # scalar functions + + tdSql.execute("use db ") + tdSql.error("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb") + tdSql.error("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") + + # aggregate functions + tdSql.error("select udf2(num1) ,udf2(num2), udf2(num3) from tb") + + # Arithmetic compute + tdSql.error("select udf2(num1)+100 ,udf2(num2)-100, udf2(num3)*100 ,udf2(num3)/100 from tb") + + tdSql.error("select udf2(c1) ,udf2(c6) from stb1 ") + + tdSql.error("select udf2(c1)+100 ,udf2(c6)-100 ,udf2(c1)*100 ,udf2(c6)/100 from stb1 ") + + # # bug for crash when query sub table + tdSql.error("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1") + + tdSql.error("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from stb1 ") + + # regular table with aggregate functions + + tdSql.error("select udf1(num1) , count(num1) from tb;") + tdSql.error("select udf1(num1) , avg(num1) from tb;") + tdSql.error("select udf1(num1) , twa(num1) from tb;") + tdSql.error("select udf1(num1) , irate(num1) from tb;") + tdSql.error("select udf1(num1) , sum(num1) from tb;") + tdSql.error("select udf1(num1) , stddev(num1) from tb;") + tdSql.error("select udf1(num1) , mode(num1) from tb;") + tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") + # stable + tdSql.error("select udf1(c1) , count(c1) from stb1;") + tdSql.error("select udf1(c1) , avg(c1) from stb1;") + tdSql.error("select udf1(c1) , twa(c1) from stb1;") + tdSql.error("select udf1(c1) , irate(c1) from stb1;") + tdSql.error("select udf1(c1) , sum(c1) from stb1;") + tdSql.error("select udf1(c1) , stddev(c1) from stb1;") + tdSql.error("select udf1(c1) , mode(c1) from stb1;") + tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") + + # regular table with select functions + + tdSql.error("select udf1(num1) , max(num1) from tb;") + + + tdSql.error("select udf1(num1) , min(num1) from tb;") + tdSql.error("select udf1(num1) , first(num1) from tb;") + tdSql.error("select udf1(num1) , last(num1) from tb;") + tdSql.error("select udf1(num1) , top(num1,1) from tb;") + tdSql.error("select udf1(num1) , bottom(num1,1) from tb;") + + + # stable + tdSql.error("select udf1(c1) , max(c1) from stb1;") + tdSql.error("select udf1(c1) , min(c1) from stb1;") + tdSql.error("select udf1(c1) , first(c1) from stb1;") + tdSql.error("select udf1(c1) , last(c1) from stb1;") + tdSql.error("select udf1(c1) , top(c1 ,1) from stb1;") + tdSql.error("select udf1(c1) , bottom(c1,1) from stb1;") + + + # regular table with compute functions + + tdSql.error("select udf1(num1) , abs(num1) from tb;") + + # stable with compute functions + tdSql.error("select udf1(c1) , abs(c1) from stb1;") + + # nest query + tdSql.error("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 order by ts;") + tdSql.error("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") + + # bug fix for crash + # order by udf function result + for _ in range(50): + tdSql.error("select udf2(c1) from stb1 group by 1-udf1(c1)") + print(tdSql.queryResult) + + # udf functions with filter + + tdSql.error("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") + tdSql.error("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + + tdSql.error("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.error("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + + # udf functions with group by + tdSql.error("select udf1(c1) from ct1 group by c1") + tdSql.error("select udf1(c1) from stb1 group by c1") + tdSql.error("select c1,c2, udf1(c1,c2) from ct1 group by c1,c2") + tdSql.error("select c1,c2, udf1(c1,c2) from stb1 group by c1,c2") + tdSql.error("select udf2(c1) from ct1 group by c1") + tdSql.error("select udf2(c1) from stb1 group by c1") + tdSql.error("select c1,c2, udf2(c1,c6) from ct1 group by c1,c2") + tdSql.error("select c1,c2, udf2(c1,c6) from stb1 group by c1,c2") + tdSql.error("select udf2(c1) from stb1 group by udf1(c1)") + tdSql.error("select udf2(c1) from stb1 group by floor(c1)") + + # udf mix with order by + tdSql.error("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + + print(" env is ok for all ") + self.prepare_udf_so() + self.prepare_data() + self.create_udf_function() + self.basic_udf_query() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/udf_cfg2.py b/tests/system-test/0-others/udf_cfg2.py new file mode 100644 index 0000000000..07f83b1455 --- /dev/null +++ b/tests/system-test/0-others/udf_cfg2.py @@ -0,0 +1,667 @@ +from distutils.log import error +import taos +import sys +import time +import os +import platform + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +import subprocess + +class TDTestCase: + updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, + "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, + "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143 ,"udf":1} + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def prepare_udf_so(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + print(projPath) + + if platform.system().lower() == 'windows': + self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + if (not tdDnodes.dnodes[0].remoteIP == ""): + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + else: + self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf2 = self.libudf2.replace('\r','').replace('\n','') + + + def prepare_data(self): + + tdSql.execute("drop database if exists db ") + tdSql.execute("create database if not exists db duration 300") + tdSql.execute("use db") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdSql.execute("create table tb (ts timestamp , num1 int , num2 int, num3 double , num4 binary(30))") + tdSql.execute( + f'''insert into tb values + ( '2020-04-21 01:01:01.000', NULL, 1, 1, "binary1" ) + ( '2020-10-21 01:01:01.000', 1, 1, 1.11, "binary1" ) + ( '2020-12-31 01:01:01.000', 2, 22222, 22, "binary1" ) + ( '2021-01-01 01:01:06.000', 3, 33333, 33, "binary1" ) + ( '2021-05-07 01:01:10.000', 4, 44444, 44, "binary1" ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, "binary1" ) + ( '2021-09-30 01:01:16.000', 5, 55555, 55, "binary1" ) + ( '2022-02-01 01:01:20.000', 6, 66666, 66, "binary1" ) + ( '2022-10-28 01:01:26.000', 0, 00000, 00, "binary1" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -88, "binary1" ) + ( '2022-12-31 01:01:36.000', 9, -9999999, -99, "binary1" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, "binary1" ) + ''' + ) + + # udf functions with join + ts_start = 1652517451000 + tdSql.execute("create stable st (ts timestamp , c1 int , c2 int ,c3 double ,c4 double ) tags(ind int)") + tdSql.execute("create table sub1 using st tags(1)") + tdSql.execute("create table sub2 using st tags(2)") + + for i in range(10): + ts = ts_start + i *1000 + tdSql.execute(" insert into sub1 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0)) + tdSql.execute(" insert into sub2 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0)) + + + def create_udf_function(self): + + for i in range(5): + # create scalar functions + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + + # create aggregate functions + + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) + + functions = tdSql.getResult("show functions") + function_nums = len(functions) + if function_nums == 2: + tdLog.info("create two udf functions success ") + + # drop functions + + tdSql.execute("drop function udf1") + tdSql.execute("drop function udf2") + + functions = tdSql.getResult("show functions") + for function in functions: + if "udf1" in function[0] or "udf2" in function[0]: + tdLog.info("drop udf functions failed ") + tdLog.exit("drop udf functions failed") + + tdLog.info("drop two udf functions success ") + + # create scalar functions + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + + # create aggregate functions + + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) + + functions = tdSql.getResult("show functions") + function_nums = len(functions) + if function_nums == 2: + tdLog.info("create two udf functions success ") + + def basic_udf_query(self): + + # scalar functions + + tdSql.execute("use db ") + tdSql.query("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(0,2,1) + tdSql.checkData(0,3,88) + tdSql.checkData(0,4,1.000000000) + tdSql.checkData(0,5,88) + tdSql.checkData(0,6,"binary1") + tdSql.checkData(0,7,88) + + tdSql.checkData(3,0,3) + tdSql.checkData(3,1,88) + tdSql.checkData(3,2,33333) + tdSql.checkData(3,3,88) + tdSql.checkData(3,4,33.000000000) + tdSql.checkData(3,5,88) + tdSql.checkData(3,6,"binary1") + tdSql.checkData(3,7,88) + + tdSql.checkData(11,0,None) + tdSql.checkData(11,1,None) + tdSql.checkData(11,2,None) + tdSql.checkData(11,3,None) + tdSql.checkData(11,4,None) + tdSql.checkData(11,5,None) + tdSql.checkData(11,6,"binary1") + tdSql.checkData(11,7,88) + + tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(0,2,None) + tdSql.checkData(0,3,None) + tdSql.checkData(0,4,None) + tdSql.checkData(0,5,None) + tdSql.checkData(0,6,None) + tdSql.checkData(0,7,None) + + tdSql.checkData(20,0,8) + tdSql.checkData(20,1,88) + tdSql.checkData(20,2,88888) + tdSql.checkData(20,3,88) + tdSql.checkData(20,4,888) + tdSql.checkData(20,5,88) + tdSql.checkData(20,6,88) + tdSql.checkData(20,7,88) + + + # aggregate functions + tdSql.query("select udf2(num1) ,udf2(num2), udf2(num3) from tb") + tdSql.checkData(0,0,15.362291496) + tdSql.checkData(0,1,10000949.553189287) + tdSql.checkData(0,2,168.633425216) + + # Arithmetic compute + tdSql.query("select udf2(num1)+100 ,udf2(num2)-100, udf2(num3)*100 ,udf2(num3)/100 from tb") + tdSql.checkData(0,0,115.362291496) + tdSql.checkData(0,1,10000849.553189287) + tdSql.checkData(0,2,16863.342521576) + tdSql.checkData(0,3,1.686334252) + + tdSql.query("select udf2(c1) ,udf2(c6) from stb1 ") + tdSql.checkData(0,0,25.514701644) + tdSql.checkData(0,1,265.247614504) + + tdSql.query("select udf2(c1)+100 ,udf2(c6)-100 ,udf2(c1)*100 ,udf2(c6)/100 from stb1 ") + tdSql.checkData(0,0,125.514701644) + tdSql.checkData(0,1,165.247614504) + tdSql.checkData(0,2,2551.470164435) + tdSql.checkData(0,3,2.652476145) + + # # bug for crash when query sub table + tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1") + tdSql.checkData(0,0,378.215547010) + tdSql.checkData(0,1,353.808067460) + tdSql.checkData(0,2,2114.237451187) + tdSql.checkData(0,3,2.125468151) + + tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from stb1 ") + tdSql.checkData(0,0,490.358032462) + tdSql.checkData(0,1,400.460106627) + tdSql.checkData(0,2,2551.470164435) + tdSql.checkData(0,3,2.652476145) + + + # regular table with aggregate functions + + tdSql.error("select udf1(num1) , count(num1) from tb;") + tdSql.error("select udf1(num1) , avg(num1) from tb;") + tdSql.error("select udf1(num1) , twa(num1) from tb;") + tdSql.error("select udf1(num1) , irate(num1) from tb;") + tdSql.error("select udf1(num1) , sum(num1) from tb;") + tdSql.error("select udf1(num1) , stddev(num1) from tb;") + tdSql.error("select udf1(num1) , mode(num1) from tb;") + tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") + # stable + tdSql.error("select udf1(c1) , count(c1) from stb1;") + tdSql.error("select udf1(c1) , avg(c1) from stb1;") + tdSql.error("select udf1(c1) , twa(c1) from stb1;") + tdSql.error("select udf1(c1) , irate(c1) from stb1;") + tdSql.error("select udf1(c1) , sum(c1) from stb1;") + tdSql.error("select udf1(c1) , stddev(c1) from stb1;") + tdSql.error("select udf1(c1) , mode(c1) from stb1;") + tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") + + # regular table with select functions + + tdSql.query("select udf1(num1) , max(num1) from tb;") + tdSql.checkRows(1) + tdSql.query("select floor(num1) , max(num1) from tb;") + tdSql.checkRows(1) + tdSql.query("select udf1(num1) , min(num1) from tb;") + tdSql.checkRows(1) + tdSql.query("select ceil(num1) , min(num1) from tb;") + tdSql.checkRows(1) + tdSql.query("select udf1(num1) , first(num1) from tb;") + + tdSql.query("select abs(num1) , first(num1) from tb;") + + tdSql.query("select udf1(num1) , last(num1) from tb;") + + tdSql.query("select round(num1) , last(num1) from tb;") + + tdSql.query("select udf1(num1) , top(num1,1) from tb;") + tdSql.checkRows(1) + tdSql.query("select udf1(num1) , bottom(num1,1) from tb;") + tdSql.checkRows(1) + # tdSql.query("select udf1(num1) , last_row(num1) from tb;") + # tdSql.checkRows(1) + + # tdSql.query("select round(num1) , last_row(num1) from tb;") + # tdSql.checkRows(1) + + + # stable + tdSql.query("select udf1(c1) , max(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select abs(c1) , max(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select udf1(c1) , min(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select floor(c1) , min(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select udf1(c1) , first(c1) from stb1;") + + tdSql.query("select udf1(c1) , last(c1) from stb1;") + + tdSql.query("select udf1(c1) , top(c1 ,1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select abs(c1) , top(c1 ,1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select udf1(c1) , bottom(c1,1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;") + tdSql.checkRows(1) + + # tdSql.query("select udf1(c1) , last_row(c1) from stb1;") + # tdSql.checkRows(1) + # tdSql.query("select ceil(c1) , last_row(c1) from stb1;") + # tdSql.checkRows(1) + + # regular table with compute functions + + tdSql.query("select udf1(num1) , abs(num1) from tb;") + tdSql.checkRows(12) + tdSql.query("select floor(num1) , abs(num1) from tb;") + tdSql.checkRows(12) + + # # bug need fix + + #tdSql.query("select udf1(num1) , csum(num1) from tb;") + #tdSql.checkRows(9) + #tdSql.query("select ceil(num1) , csum(num1) from tb;") + #tdSql.checkRows(9) + #tdSql.query("select udf1(c1) , csum(c1) from stb1;") + #tdSql.checkRows(22) + #tdSql.query("select floor(c1) , csum(c1) from stb1;") + #tdSql.checkRows(22) + + # stable with compute functions + tdSql.query("select udf1(c1) , abs(c1) from stb1;") + tdSql.checkRows(25) + tdSql.query("select abs(c1) , ceil(c1) from stb1;") + tdSql.checkRows(25) + + # nest query + tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 order by ts;") + tdSql.checkRows(25) + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(1,0,88) + tdSql.checkData(1,1,8) + + tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") + tdSql.checkRows(13) + tdSql.checkData(0,0,88) + tdSql.checkData(0,1,8) + tdSql.checkData(1,0,88) + tdSql.checkData(1,1,7) + + # bug fix for crash + # order by udf function result + for _ in range(50): + tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)") + print(tdSql.queryResult) + + # udf functions with filter + + tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 where c1 is null order by ts;") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + + tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") + tdSql.checkRows(3) + tdSql.checkData(0,0,9) + tdSql.checkData(0,1,88) + tdSql.checkData(0,2,-99.990000000) + tdSql.checkData(0,3,88) + + tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,0) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,10) + + tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,88) + tdSql.checkData(0,1,88) + tdSql.checkData(1,0,88) + tdSql.checkData(1,1,88) + + tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,88) + tdSql.checkData(0,2,0) + tdSql.checkData(0,3,88) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,88) + tdSql.checkData(1,2,10) + tdSql.checkData(1,3,88) + + tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,16.881943016) + tdSql.checkData(0,1,168.819430161) + tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + + # udf functions with group by + tdSql.query("select udf1(c1) from ct1 group by c1") + tdSql.checkRows(10) + tdSql.query("select udf1(c1) from stb1 group by c1") + tdSql.checkRows(11) + tdSql.query("select c1,c2, udf1(c1,c2) from ct1 group by c1,c2") + tdSql.checkRows(10) + tdSql.query("select c1,c2, udf1(c1,c2) from stb1 group by c1,c2") + tdSql.checkRows(11) + + tdSql.query("select udf2(c1) from ct1 group by c1") + tdSql.checkRows(10) + tdSql.query("select udf2(c1) from stb1 group by c1") + tdSql.checkRows(11) + tdSql.query("select c1,c2, udf2(c1,c6) from ct1 group by c1,c2") + tdSql.checkRows(10) + tdSql.query("select c1,c2, udf2(c1,c6) from stb1 group by c1,c2") + tdSql.checkRows(11) + tdSql.query("select udf2(c1) from stb1 group by udf1(c1)") + tdSql.checkRows(2) + tdSql.query("select udf2(c1) from stb1 group by floor(c1)") + tdSql.checkRows(11) + + # udf mix with order by + tdSql.query("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)") + tdSql.checkRows(11) + + + def multi_cols_udf(self): + tdSql.query("select num1,num2,num3,udf1(num1,num2,num3) from tb") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,1) + tdSql.checkData(0,2,1.000000000) + tdSql.checkData(0,3,None) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) + tdSql.checkData(1,2,1.110000000) + tdSql.checkData(1,3,88) + + tdSql.query("select c1,c6,udf1(c1,c6) from stb1 order by ts") + tdSql.checkData(1,0,8) + tdSql.checkData(1,1,88.880000000) + tdSql.checkData(1,2,88) + + tdSql.query("select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;") + tdSql.checkRows(22) + + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + + def try_query_sql(self): + udf1_sqls = [ + "select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb" , + "select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1" , + "select udf1(num1) , max(num1) from tb;" , + "select udf1(num1) , min(num1) from tb;" , + #"select udf1(num1) , top(num1,1) from tb;" , + #"select udf1(num1) , bottom(num1,1) from tb;" , + "select udf1(c1) , max(c1) from stb1;" , + "select udf1(c1) , min(c1) from stb1;" , + #"select udf1(c1) , top(c1 ,1) from stb1;" , + #"select udf1(c1) , bottom(c1,1) from stb1;" , + "select udf1(num1) , abs(num1) from tb;" , + #"select udf1(num1) , csum(num1) from tb;" , + #"select udf1(c1) , csum(c1) from stb1;" , + "select udf1(c1) , abs(c1) from stb1;" , + "select abs(udf1(c1)) , abs(ceil(c1)) from stb1 order by ts;" , + "select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;" , + "select abs(udf1(c1)) , abs(ceil(c1)) from stb1 where c1 is null order by ts;" , + "select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts" , + "select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select udf1(c1) from ct1 group by c1" , + "select udf1(c1) from stb1 group by c1" , + "select c1,c2, udf1(c1,c2) from ct1 group by c1,c2" , + "select c1,c2, udf1(c1,c2) from stb1 group by c1,c2" , + "select num1,num2,num3,udf1(num1,num2,num3) from tb" , + "select c1,c6,udf1(c1,c6) from stb1 order by ts" , + "select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;" + ] + udf2_sqls = ["select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select udf2(c1) from stb1 group by 1-udf1(c1)" , + "select udf2(num1) ,udf2(num2), udf2(num3) from tb" , + "select udf2(num1)+100 ,udf2(num2)-100, udf2(num3)*100 ,udf2(num3)/100 from tb" , + "select udf2(c1) ,udf2(c6) from stb1 " , + "select udf2(c1)+100 ,udf2(c6)-100 ,udf2(c1)*100 ,udf2(c6)/100 from stb1 " , + "select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1" , + "select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from stb1 " , + "select udf2(c1) from ct1 group by c1" , + "select udf2(c1) from stb1 group by c1" , + "select c1,c2, udf2(c1,c6) from ct1 group by c1,c2" , + "select c1,c2, udf2(c1,c6) from stb1 group by c1,c2" , + "select udf2(c1) from stb1 group by udf1(c1)" , + "select udf2(c1) from stb1 group by floor(c1)" , + "select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)" , + + "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null"] + + return udf1_sqls ,udf2_sqls + + + + def unexpected_create(self): + + tdLog.info(" create function with out bufsize ") + tdSql.query("drop function udf1 ") + tdSql.query("drop function udf2 ") + + # create function without buffer + tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1) + tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2) + udf1_sqls ,udf2_sqls = self.try_query_sql() + + for scalar_sql in udf1_sqls: + tdSql.query(scalar_sql) + for aggregate_sql in udf2_sqls: + tdSql.error(aggregate_sql) + + # create function without aggregate + + tdLog.info(" create function with out aggregate ") + tdSql.query("drop function udf1 ") + tdSql.query("drop function udf2 ") + + # create function without buffer + tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) + udf1_sqls ,udf2_sqls = self.try_query_sql() + + for scalar_sql in udf1_sqls: + tdSql.error(scalar_sql) + for aggregate_sql in udf2_sqls: + tdSql.error(aggregate_sql) + + tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.error(" select db(c1) from stb1 ") + tdSql.error(" select db(c1,c6), db(c6) from stb1 ") + tdSql.error(" select db(num1,num2), db(num1) from tb ") + tdSql.error(" select test(c1) from stb1 ") + tdSql.error(" select test(c1,c6), test(c6) from stb1 ") + tdSql.error(" select test(num1,num2), test(num1) from tb ") + + + + def loop_kill_udfd(self): + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + + cfgPath = buildPath + "/../sim/dnode1/cfg" + udfdPath = buildPath +'/build/bin/udfd' + + for i in range(3): + + tdLog.info(" loop restart udfd %d_th" % i) + + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + # stop udfd cmds + get_processID = "ps -ef | grep -w udfd | grep -v grep| grep -v defunct | awk '{print $2}'" + processID = subprocess.check_output(get_processID, shell=True).decode("utf-8") + stop_udfd = " kill -9 %s" % processID + os.system(stop_udfd) + + time.sleep(2) + + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + + # # start udfd cmds + # start_udfd = "nohup " + udfdPath +'-c' +cfgPath +" > /dev/null 2>&1 &" + # tdLog.info("start udfd : %s " % start_udfd) + + def test_function_name(self): + tdLog.info(" create function name is not build_in functions ") + tdSql.execute(" drop function udf1 ") + tdSql.execute(" drop function udf2 ") + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2) + + def restart_taosd_query_udf(self): + + self.create_udf_function() + + for i in range(5): + tdLog.info(" this is %d_th restart taosd " %i) + tdSql.execute("use db ") + tdSql.query("select count(*) from stb1") + tdSql.checkRows(1) + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + tdDnodes.stop(1) + tdDnodes.start(1) + time.sleep(2) + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + + print(" env is ok for all ") + self.prepare_udf_so() + self.prepare_data() + self.create_udf_function() + self.basic_udf_query() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 3b0dd76a30..ffa7befd69 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -11,6 +11,8 @@ python3 ./test.py -f 0-others/udfTest.py python3 ./test.py -f 0-others/udf_create.py python3 ./test.py -f 0-others/udf_restart_taosd.py python3 ./test.py -f 0-others/cachelast.py +python3 ./test.py -f 0-others/udf_cfg1.py +python3 ./test.py -f 0-others/udf_cfg2.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py From 1e3fff30e73098b8725b73e47829f36e88010c69 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Jul 2022 12:16:40 +0000 Subject: [PATCH 003/124] more vnode snapshot --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/meta/metaSnapshot.c | 67 ++++++++++----- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 95 ++++++++++++++-------- 3 files changed, 109 insertions(+), 54 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7b298ba830..d6b0dd48e8 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -309,6 +309,7 @@ void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); struct SSnapDataHdr { int8_t type; + int64_t index; int64_t size; uint8_t data[]; }; diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index ac84842e85..900416f1b8 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -26,60 +26,72 @@ struct SMetaSnapReader { int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) { int32_t code = 0; int32_t c = 0; - SMetaSnapReader* pMetaSnapReader = NULL; + SMetaSnapReader* pReader = NULL; // alloc - pMetaSnapReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pMetaSnapReader)); - if (pMetaSnapReader == NULL) { + pReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); + if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pMetaSnapReader->pMeta = pMeta; - pMetaSnapReader->sver = sver; - pMetaSnapReader->ever = ever; + pReader->pMeta = pMeta; + pReader->sver = sver; + pReader->ever = ever; // impl - code = tdbTbcOpen(pMeta->pTbDb, &pMetaSnapReader->pTbc, NULL); + code = tdbTbcOpen(pMeta->pTbDb, &pReader->pTbc, NULL); if (code) { + taosMemoryFree(pReader); goto _err; } - code = tdbTbcMoveTo(pMetaSnapReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c); + code = tdbTbcMoveTo(pReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c); if (code) { + taosMemoryFree(pReader); goto _err; } - *ppReader = pMetaSnapReader; + metaInfo("vgId:%d vnode snapshot meta reader opened", TD_VID(pMeta->pVnode)); + + *ppReader = pReader; return code; _err: - metaError("vgId:%d meta snap reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); + metaError("vgId:%d vnode snapshot meta reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); *ppReader = NULL; return code; } int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) { + int32_t code = 0; + tdbTbcClose((*ppReader)->pTbc); taosMemoryFree(*ppReader); *ppReader = NULL; - return 0; + + return code; } int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { + int32_t code = 0; const void* pKey = NULL; const void* pData = NULL; int32_t nKey = 0; int32_t nData = 0; - int32_t code = 0; + STbDbKey key; + *ppData = NULL; for (;;) { - code = tdbTbcGet(pReader->pTbc, &pKey, &nKey, &pData, &nData); - if (code || ((STbDbKey*)pData)->version > pReader->ever) { - code = TSDB_CODE_VND_READ_END; + if (tdbTbcGet(pReader->pTbc, &pKey, &nKey, &pData, &nData)) { goto _exit; } - if (((STbDbKey*)pData)->version < pReader->sver) { + key = ((STbDbKey*)pKey)[0]; + if (key.version > pReader->ever) { + goto _exit; + } + + if (key.version < pReader->sver) { tdbTbcMoveToNext(pReader->pTbc); continue; } @@ -88,17 +100,28 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { break; } - // copy the data - if (tRealloc(ppData, sizeof(SSnapDataHdr) + nData) < 0) { + ASSERT(pData && nData); + + *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData); + if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - return code; + goto _err; } - ((SSnapDataHdr*)(*ppData))->type = 0; // TODO: use macro - ((SSnapDataHdr*)(*ppData))->size = nData; - memcpy(((SSnapDataHdr*)(*ppData))->data, pData, nData); + + SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); + pHdr->type = 0; // TODO: use macro + pHdr->size = nData; + memcpy(pHdr->data, pData, nData); + + metaInfo("vgId:%d vnode snapshot meta read data, version:%" PRId64 " uid:%" PRId64 " nData:%d", + TD_VID(pReader->pMeta->pVnode), key.version, key.uid, nData); _exit: return code; + +_err: + metaError("vgId:%d vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode), tstrerror(code)); + return code; } // SMetaSnapWriter ======================================== diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 27f30ec787..60801832a9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -20,13 +20,13 @@ struct SVSnapReader { SVnode *pVnode; int64_t sver; int64_t ever; + int64_t index; // meta int8_t metaDone; SMetaSnapReader *pMetaReader; // tsdb int8_t tsdbDone; STsdbSnapReader *pTsdbReader; - uint8_t *pData; }; int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) { @@ -42,12 +42,7 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapRe pReader->sver = sver; pReader->ever = ever; - code = metaSnapReaderOpen(pVnode->pMeta, sver, ever, &pReader->pMetaReader); - if (code) goto _err; - - code = tsdbSnapReaderOpen(pVnode->pTsdb, sver, ever, &pReader->pTsdbReader); - if (code) goto _err; - + vInfo("vgId:%d vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever); *ppReader = pReader; return code; @@ -60,50 +55,75 @@ _err: int32_t vnodeSnapReaderClose(SVSnapReader *pReader) { int32_t code = 0; - tFree(pReader->pData); - if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader); - if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader); - taosMemoryFree(pReader); + // tFree(pReader->pData); + // if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader); + // if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader); + // taosMemoryFree(pReader); + vInfo("vgId:%d vnode snapshot reader closed", TD_VID(pReader->pVnode)); return code; } int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) { int32_t code = 0; + // META ============== if (!pReader->metaDone) { - code = metaSnapRead(pReader->pMetaReader, &pReader->pData); + // open reader if not + if (pReader->pMetaReader == NULL) { + code = metaSnapReaderOpen(pReader->pVnode->pMeta, pReader->sver, pReader->ever, &pReader->pMetaReader); + if (code) goto _err; + } + + code = metaSnapRead(pReader->pMetaReader, ppData); if (code) { - if (code == TSDB_CODE_VND_READ_END) { + goto _err; + } else { + if (*ppData) { + goto _exit; + } else { pReader->metaDone = 1; - } else { - goto _err; + code = metaSnapReaderClose(&pReader->pMetaReader); + if (code) goto _err; + + vInfo("vgId:%d vnode snapshot meta data read end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index); } - } else { - *ppData = pReader->pData; - *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size; - goto _exit; } } + // TSDB ============== if (!pReader->tsdbDone) { - code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData); - if (code) { - if (code == TSDB_CODE_VND_READ_END) { - pReader->tsdbDone = 1; - } else { - goto _err; - } - } else { - *ppData = pReader->pData; - *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size; - goto _exit; - } + // open if not + // if (pReader->pTsdbReader == NULL) { + // code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, &pReader->pTsdbReader); + // if (code) goto _err; + // } + + // code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData); + // if (code) { + // if (code == TSDB_CODE_VND_READ_END) { + // pReader->tsdbDone = 1; + // } else { + // goto _err; + // } + // } else { + // *ppData = pReader->pData; + // *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size; + // goto _exit; + // } } - code = TSDB_CODE_VND_READ_END; + *ppData = NULL; + *nData = 0; _exit: + if (*ppData) { + pReader->index++; + ((SSnapDataHdr *)(*ppData))->index = pReader->index; + vInfo("vgId:%d vnode snapshot read data, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index); + } else { + vInfo("vgId:%d vnode snapshot read data end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index); + } return code; _err: @@ -116,6 +136,7 @@ struct SVSnapWriter { SVnode *pVnode; int64_t sver; int64_t ever; + int64_t index; // meta SMetaSnapWriter *pMetaSnapWriter; // tsdb @@ -148,16 +169,22 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr pWriter->sver = sver; pWriter->ever = ever; + vInfo("vgId:%d vnode snapshot writer opened", TD_VID(pVnode)); + + *ppWriter = pWriter; return code; _err: vError("vgId:%d vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code)); + *ppWriter = NULL; return code; } int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) { int32_t code = 0; + goto _exit; + if (rollback) { code = vnodeSnapRollback(pWriter); if (code) goto _err; @@ -166,6 +193,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) { if (code) goto _err; } +_exit: taosMemoryFree(pWriter); return code; @@ -179,6 +207,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { SSnapDataHdr *pSnapDataHdr = (SSnapDataHdr *)pData; SVnode *pVnode = pWriter->pVnode; + goto _exit; + ASSERT(pSnapDataHdr->size + sizeof(SSnapDataHdr) == nData); if (pSnapDataHdr->type == 0) { @@ -201,6 +231,7 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { if (code) goto _err; } +_exit: return code; _err: From 98cb6ae092219c66580f098b2bc317adf22fe24c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Jul 2022 12:46:27 +0000 Subject: [PATCH 004/124] more vnode snapshot --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 54 ++++++++++++---------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 60801832a9..0163ea0449 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -58,9 +58,9 @@ int32_t vnodeSnapReaderClose(SVSnapReader *pReader) { // tFree(pReader->pData); // if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader); // if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader); - // taosMemoryFree(pReader); vInfo("vgId:%d vnode snapshot reader closed", TD_VID(pReader->pVnode)); + taosMemoryFree(pReader); return code; } @@ -118,9 +118,12 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) _exit: if (*ppData) { + SSnapDataHdr *pHdr = (SSnapDataHdr *)(*ppData); + pReader->index++; - ((SSnapDataHdr *)(*ppData))->index = pReader->index; - vInfo("vgId:%d vnode snapshot read data, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index); + *nData = sizeof(SSnapDataHdr) + pHdr->size; + pHdr->index = pReader->index; + vInfo("vgId:%d vnode snapshot read data, nData:%d index:%" PRId64, TD_VID(pReader->pVnode), *nData, pReader->index); } else { vInfo("vgId:%d vnode snapshot read data end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index); } @@ -194,44 +197,45 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) { } _exit: + vInfo("vgId:%d vnode snapshot writer closed, rollback:%d", TD_VID(pWriter->pVnode), rollback); taosMemoryFree(pWriter); return code; _err: - vError("vgId:%d vnode snapshow writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code)); + vError("vgId:%d vnode snapshot writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code)); return code; } int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { int32_t code = 0; - SSnapDataHdr *pSnapDataHdr = (SSnapDataHdr *)pData; + SSnapDataHdr *pHdr = (SSnapDataHdr *)pData; SVnode *pVnode = pWriter->pVnode; - goto _exit; + // ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData); - ASSERT(pSnapDataHdr->size + sizeof(SSnapDataHdr) == nData); + // if (pHdr->type == 0) { + // // meta + // if (pWriter->pMetaSnapWriter == NULL) { + // code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter); + // if (code) goto _err; + // } - if (pSnapDataHdr->type == 0) { - // meta - if (pWriter->pMetaSnapWriter == NULL) { - code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter); - if (code) goto _err; - } + // code = metaSnapWrite(pWriter->pMetaSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); + // if (code) goto _err; + // } else { + // // tsdb + // if (pWriter->pTsdbSnapWriter == NULL) { + // code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter); + // if (code) goto _err; + // } - code = metaSnapWrite(pWriter->pMetaSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); - if (code) goto _err; - } else { - // tsdb - if (pWriter->pTsdbSnapWriter == NULL) { - code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter); - if (code) goto _err; - } - - code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); - if (code) goto _err; - } + // code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); + // if (code) goto _err; + // } _exit: + vInfo("vgId:%d vnode snapshot write data, type:%d index:%" PRId64 " nData:%d", TD_VID(pVnode), pHdr->type, + pHdr->index, nData); return code; _err: From f977a50e9b3cb9569022d3008f9c489060dc2c32 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 13 Jul 2022 01:11:55 +0000 Subject: [PATCH 005/124] more vnode snapshot --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 3 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 46 +++++++++++----------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 06d4a86116..b02e1dabd3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -232,11 +232,12 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe goto _err; } + tsdbInfo("vgId:%d vnode snapshot tsdb reader opened", TD_VID(pTsdb->pVnode)); *ppReader = pReader; return code; _err: - tsdbError("vgId:%d snapshot reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); *ppReader = NULL; return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 0163ea0449..0e14df0380 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -55,9 +55,13 @@ _err: int32_t vnodeSnapReaderClose(SVSnapReader *pReader) { int32_t code = 0; - // tFree(pReader->pData); - // if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader); - // if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader); + if (pReader->pTsdbReader) { + tsdbSnapReaderClose(&pReader->pTsdbReader); + } + + if (pReader->pMetaReader) { + metaSnapReaderClose(&pReader->pMetaReader); + } vInfo("vgId:%d vnode snapshot reader closed", TD_VID(pReader->pVnode)); taosMemoryFree(pReader); @@ -85,8 +89,6 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) pReader->metaDone = 1; code = metaSnapReaderClose(&pReader->pMetaReader); if (code) goto _err; - - vInfo("vgId:%d vnode snapshot meta data read end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index); } } } @@ -94,23 +96,23 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) // TSDB ============== if (!pReader->tsdbDone) { // open if not - // if (pReader->pTsdbReader == NULL) { - // code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, &pReader->pTsdbReader); - // if (code) goto _err; - // } + if (pReader->pTsdbReader == NULL) { + code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, &pReader->pTsdbReader); + if (code) goto _err; + } - // code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData); - // if (code) { - // if (code == TSDB_CODE_VND_READ_END) { - // pReader->tsdbDone = 1; - // } else { - // goto _err; - // } - // } else { - // *ppData = pReader->pData; - // *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size; - // goto _exit; - // } + code = tsdbSnapRead(pReader->pTsdbReader, ppData); + if (code) { + goto _err; + } else { + if (*ppData) { + goto _exit; + } else { + pReader->tsdbDone = 1; + code = tsdbSnapReaderClose(&pReader->pTsdbReader); + if (code) goto _err; + } + } } *ppData = NULL; @@ -130,7 +132,7 @@ _exit: return code; _err: - vError("vgId:% snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); + vError("vgId:% vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); return code; } From 338a7638537358d6fddafd766b5f02d87409a39c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 13 Jul 2022 02:41:37 +0000 Subject: [PATCH 006/124] more vnode snapshot --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 4 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 67 ++++++++++++---------- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 7 ++- 5 files changed, 46 insertions(+), 36 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4df9f96514..819076536d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -202,7 +202,7 @@ int32_t tsdbFSStateUpsertDelFile(STsdbFSState *pState, SDelFile *pDelFile); int32_t tsdbFSStateUpsertDFileSet(STsdbFSState *pState, SDFileSet *pSet); void tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid); SDelFile *tsdbFSStateGetDelFile(STsdbFSState *pState); -SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid); +SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid, int32_t flag); // tsdbReaderWriter.c ============================================================================================== // SDataFWriter int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8a189d98b3..577ef03a8f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -263,7 +263,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { taosArrayClear(pCommitter->aBlockIdx); tMapDataReset(&pCommitter->oBlockMap); tBlockDataReset(&pCommitter->oBlockData); - pRSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, pCommitter->commitFid); + pRSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, pCommitter->commitFid, TD_EQ); if (pRSet) { code = tsdbDataFReaderOpen(&pCommitter->pReader, pTsdb, pRSet); if (code) goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 53b6735c30..70fbd90646 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -698,6 +698,6 @@ void tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid) { SDelFile *tsdbFSStateGetDelFile(STsdbFSState *pState) { return pState->pDelFile; } -SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid) { - return (SDFileSet *)taosArraySearch(pState->aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ); +SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid, int32_t flag) { + return (SDFileSet *)taosArraySearch(pState->aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, flag); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index b02e1dabd3..7dc7e29ad7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -40,10 +40,12 @@ struct STsdbSnapReader { static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; + STsdb* pTsdb = pReader->pTsdb; while (true) { if (pReader->pDataFReader == NULL) { SDFileSet* pSet = NULL; + // taosArraySearch(pTsdb->fs->cState->aDFileSet, &(SDFileSet){.fid = pReader->fid}, tDFileSe) // search the next data file set to read (todo) if (0 /* TODO */) { @@ -106,10 +108,13 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { } _exit: + // if (*ppData) { + // tsdbInfo("vgId:%d "); + // } return code; _err: - tsdbError("vgId:%d snap read data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb read data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -209,14 +214,13 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe pReader->sver = sver; pReader->ever = ever; + pReader->fid = INT32_MIN; pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pReader->aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pReader->mBlock = tMapDataInit(); - code = tBlockDataInit(&pReader->blkData); if (code) goto _err; @@ -225,7 +229,6 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pReader->aDelData = taosArrayInit(0, sizeof(SDelData)); if (pReader->aDelData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -246,37 +249,42 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { int32_t code = 0; STsdbSnapReader* pReader = *ppReader; - taosArrayDestroy(pReader->aDelData); - taosArrayDestroy(pReader->aDelIdx); - if (pReader->pDelFReader) { - tsdbDelFReaderClose(&pReader->pDelFReader); - } - tBlockDataClear(&pReader->blkData); - tMapDataClear(&pReader->mBlock); - taosArrayDestroy(pReader->aBlockIdx); if (pReader->pDataFReader) { tsdbDataFReaderClose(&pReader->pDataFReader); } + taosArrayDestroy(pReader->aBlockIdx); + tMapDataClear(&pReader->mBlock); + tBlockDataClear(&pReader->blkData); + + if (pReader->pDelFReader) { + tsdbDelFReaderClose(&pReader->pDelFReader); + } + taosArrayDestroy(pReader->aDelIdx); + taosArrayDestroy(pReader->aDelData); + + tsdbInfo("vgId:%d vnode snapshot tsdb reader closed", TD_VID(pReader->pTsdb->pVnode)); + taosMemoryFree(pReader); *ppReader = NULL; - return code; } int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; + *ppData = NULL; + // read data file if (!pReader->dataDone) { code = tsdbSnapReadData(pReader, ppData); if (code) { - if (code == TSDB_CODE_VND_READ_END) { - pReader->dataDone = 1; - } else { - goto _err; - } + goto _err; } else { - goto _exit; + if (*ppData) { + goto _exit; + } else { + pReader->dataDone = 1; + } } } @@ -284,23 +292,24 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { if (!pReader->delDone) { code = tsdbSnapReadDel(pReader, ppData); if (code) { - if (code == TSDB_CODE_VND_READ_END) { - pReader->delDone = 1; - } else { - goto _err; - } + goto _err; } else { - goto _exit; + if (*ppData) { + goto _exit; + } else { + pReader->delDone = 1; + } } } - code = TSDB_CODE_VND_READ_END; - _exit: + if (*ppData) { + } else { + } return code; _err: - tsdbError("vgId:%d snapshot read failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb read failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } @@ -520,7 +529,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 if (code) goto _err; pWriter->fid = fid; - SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid); + SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ); // reader if (pSet) { // open diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 0e14df0380..ed09f0edc3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -125,7 +125,8 @@ _exit: pReader->index++; *nData = sizeof(SSnapDataHdr) + pHdr->size; pHdr->index = pReader->index; - vInfo("vgId:%d vnode snapshot read data, nData:%d index:%" PRId64, TD_VID(pReader->pVnode), *nData, pReader->index); + vInfo("vgId:%d vnode snapshot read data,index:%" PRId64 " type:%d nData:%d ", TD_VID(pReader->pVnode), + pReader->index, pHdr->type, *nData); } else { vInfo("vgId:%d vnode snapshot read data end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index); } @@ -236,8 +237,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { // } _exit: - vInfo("vgId:%d vnode snapshot write data, type:%d index:%" PRId64 " nData:%d", TD_VID(pVnode), pHdr->type, - pHdr->index, nData); + vInfo("vgId:%d vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->type, nData, + pHdr->index); return code; _err: From b691e76782011d2c612497e05b25379f28e9880d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 13 Jul 2022 05:51:37 +0000 Subject: [PATCH 007/124] more vnode snapshot --- source/dnode/vnode/src/inc/tsdb.h | 4 + source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 85 +++++++++++---- source/dnode/vnode/src/tsdb/tsdbUtil.c | 116 +++++++++++++++++++++ 3 files changed, 182 insertions(+), 23 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 819076536d..bfb319dce4 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -126,6 +126,8 @@ void tColDataClear(void *ph); int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal); int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); +int32_t tPutColData(uint8_t *p, SColData *pColData); +int32_t tGetColData(uint8_t *p, SColData *pColData); // SBlockData #define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) #define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) @@ -142,6 +144,8 @@ int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlo int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); +int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); +int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); // SDelIdx int32_t tPutDelIdx(uint8_t *p, void *ph); int32_t tGetDelIdx(uint8_t *p, void *ph); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 7dc7e29ad7..8bf1b29454 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -29,7 +29,8 @@ struct STsdbSnapReader { SBlockIdx* pBlockIdx; SMapData mBlock; // SMapData int32_t iBlock; - SBlockData blkData; + SBlockData oBlockData; + SBlockData nBlockData; // for del file int8_t delDone; SDelFReader* pDelFReader; @@ -44,16 +45,11 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { while (true) { if (pReader->pDataFReader == NULL) { - SDFileSet* pSet = NULL; - // taosArraySearch(pTsdb->fs->cState->aDFileSet, &(SDFileSet){.fid = pReader->fid}, tDFileSe) + SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->cState, pReader->fid, TD_GT); - // search the next data file set to read (todo) - if (0 /* TODO */) { - code = TSDB_CODE_VND_READ_END; - goto _exit; - } + if (pSet == NULL) goto _exit; - // open + pReader->fid = pSet->fid; code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); if (code) goto _err; @@ -63,6 +59,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->iBlockIdx = 0; pReader->pBlockIdx = NULL; + + tsdbInfo("vgId:%d vnode snapshot tsdb open data file to read, fid:%d", TD_VID(pTsdb->pVnode), pReader->fid); } while (true) { @@ -75,17 +73,15 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); pReader->iBlockIdx++; - // SBlock code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock, NULL); if (code) goto _err; pReader->iBlock = 0; } + SBlock block; + SBlock* pBlock = █ while (true) { - SBlock block; - SBlock* pBlock = █ - if (pReader->iBlock >= pReader->mBlock.nItem) { pReader->pBlockIdx = NULL; break; @@ -94,23 +90,63 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, pBlock, tGetBlock); pReader->iBlock++; - if ((pBlock->minVersion >= pReader->sver && pBlock->minVersion <= pReader->ever) || - (pBlock->maxVersion >= pReader->sver && pBlock->maxVersion <= pReader->ever)) { - // overlap (todo) + if (pBlock->minVersion > pReader->ever || pBlock->maxVersion < pReader->sver) continue; - code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->blkData, NULL, NULL); + code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->oBlockData, NULL, NULL); + if (code) goto _err; + + // filter + tBlockDataReset(&pReader->nBlockData); + for (int32_t iColData = 0; iColData < taosArrayGetSize(pReader->oBlockData.aIdx); iColData++) { + SColData* pColDataO = tBlockDataGetColDataByIdx(&pReader->oBlockData, iColData); + SColData* pColDataN = NULL; + + code = tBlockDataAddColData(&pReader->nBlockData, taosArrayGetSize(pReader->nBlockData.aIdx), &pColDataN); if (code) goto _err; - goto _exit; + tColDataInit(pColDataN, pColDataO->cid, pColDataO->type, pColDataO->smaOn); } + + for (int32_t iRow = 0; iRow < pReader->oBlockData.nRow; iRow++) { + TSDBROW row = tsdbRowFromBlockData(&pReader->oBlockData, iRow); + int64_t version = TSDBROW_VERSION(&row); + + if (version < pReader->sver || version > pReader->ever) continue; + + code = tBlockDataAppendRow(&pReader->nBlockData, &row, NULL); + if (code) goto _err; + } + + // org data (todo) + int32_t size = sizeof(TABLEID) + tPutBlockData(NULL, &pReader->nBlockData); + + *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); + if (*ppData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); + pHdr->type = 1; + pHdr->size = size; + + TABLEID* pId = (TABLEID*)(&pHdr[1]); + pId->suid = pReader->pBlockIdx->suid; + pId->uid = pReader->pBlockIdx->uid; + + tPutBlockData((uint8_t*)(&pId[1]), &pReader->nBlockData); + + tsdbInfo("vgId:%d vnode snapshot read data, fid:%d suid:%" PRId64 " uid:%" PRId64 + " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d", + TD_VID(pTsdb->pVnode), pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid, + pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow); + + goto _exit; } } } _exit: - // if (*ppData) { - // tsdbInfo("vgId:%d "); - // } return code; _err: @@ -221,7 +257,9 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe goto _err; } pReader->mBlock = tMapDataInit(); - code = tBlockDataInit(&pReader->blkData); + code = tBlockDataInit(&pReader->oBlockData); + if (code) goto _err; + code = tBlockDataInit(&pReader->nBlockData); if (code) goto _err; pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); @@ -254,7 +292,8 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { } taosArrayDestroy(pReader->aBlockIdx); tMapDataClear(&pReader->mBlock); - tBlockDataClear(&pReader->blkData); + tBlockDataClear(&pReader->oBlockData); + tBlockDataClear(&pReader->nBlockData); if (pReader->pDelFReader) { tsdbDelFReaderClose(&pReader->pDelFReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index dc6c823368..db408df944 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -921,6 +921,76 @@ _exit: return code; } +int32_t tPutColData(uint8_t *p, SColData *pColData) { + int32_t n = 0; + + n += tPutI16v(p ? p + n : p, pColData->cid); + n += tPutI8(p ? p + n : p, pColData->type); + n += tPutI8(p ? p + n : p, pColData->smaOn); + n += tPutI32v(p ? p + n : p, pColData->nVal); + n += tPutU8(p ? p + n : p, pColData->flag); + + if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit; + if (pColData->flag != HAS_VALUE) { + // bitmap + + int32_t size = BIT2_SIZE(pColData->nVal); + if (p) { + memcpy(p + n, pColData->pBitMap, size); + } + n += size; + } + if (IS_VAR_DATA_TYPE(pColData->type)) { + // offset + + int32_t size = sizeof(int32_t) * pColData->nVal; + if (p) { + memcpy(p + n, pColData->aOffset, size); + } + n += size; + } + n += tPutI32v(p ? p + n : p, pColData->nData); + if (p) { + memcpy(p + n, pColData->pData, pColData->nData); + } + n += pColData->nData; + +_exit: + return n; +} + +int32_t tGetColData(uint8_t *p, SColData *pColData) { + int32_t n = 0; + + n += tGetI16v(p + n, &pColData->cid); + n += tGetI8(p + n, &pColData->type); + n += tGetI8(p + n, &pColData->smaOn); + n += tGetI32v(p + n, &pColData->nVal); + n += tGetU8(p + n, &pColData->flag); + + if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit; + if (pColData->flag != HAS_VALUE) { + // bitmap + + int32_t size = BIT2_SIZE(pColData->nVal); + pColData->pBitMap = p + n; + n += size; + } + if (IS_VAR_DATA_TYPE(pColData->type)) { + // offset + + int32_t size = sizeof(int32_t) * pColData->nVal; + pColData->aOffset = (int32_t *)(p + n); + n += size; + } + n += tGetI32v(p + n, &pColData->nData); + pColData->pData = p + n; + n += pColData->nData; + +_exit: + return n; +} + static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) { SColData *pColData1 = (SColData *)p1; SColData *pColData2 = (SColData *)p2; @@ -1239,6 +1309,52 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD *ppColData = NULL; } +int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData) { + int32_t n = 0; + + n += tPutI32v(p ? p + n : p, pBlockData->nRow); + if (p) { + memcpy(p + n, pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow); + } + n = n + sizeof(int64_t) * pBlockData->nRow; + if (p) { + memcpy(p + n, pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow); + } + n = n + sizeof(TSKEY) * pBlockData->nRow; + + int32_t nCol = taosArrayGetSize(pBlockData->aIdx); + n += tPutI32v(p ? p + n : p, nCol); + for (int32_t iCol = 0; iCol < nCol; iCol++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); + n += tPutColData(p ? p + n : p, pColData); + } + + return n; +} + +int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData) { + int32_t n = 0; + + tBlockDataReset(pBlockData); + + n += tGetI32v(p + n, &pBlockData->nRow); + pBlockData->aVersion = (int64_t *)(p + n); + n = n + sizeof(int64_t) * pBlockData->nRow; + pBlockData->aTSKEY = (TSKEY *)(p + n); + n = n + sizeof(TSKEY) * pBlockData->nRow; + + int32_t nCol; + n += tGetI32v(p + n, &nCol); + for (int32_t iCol = 0; iCol < nCol; iCol++) { + SColData *pColData; + + if (tBlockDataAddColData(pBlockData, iCol, &pColData)) return -1; + n += tGetColData(p + n, pColData); + } + + return n; +} + // ALGORITHM ============================== void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { SColVal colVal; From 2fd04241d6a3a418c24130c875ef6c46060f5b8a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 13 Jul 2022 06:26:13 +0000 Subject: [PATCH 008/124] more vnode snapshot reader --- include/util/taoserror.h | 1 - source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 85 +++++++++++----------- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 4 +- source/util/src/terror.c | 2 - 4 files changed, 45 insertions(+), 47 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ce434612c3..a07da1aa45 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -331,7 +331,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_INVALID_TABLE_ACTION TAOS_DEF_ERROR_CODE(0, 0x0519) #define TSDB_CODE_VND_COL_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x051a) #define TSDB_CODE_VND_TABLE_COL_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x051b) -#define TSDB_CODE_VND_READ_END TAOS_DEF_ERROR_CODE(0, 0x051c) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 8bf1b29454..fbe14088df 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -34,8 +34,8 @@ struct STsdbSnapReader { // for del file int8_t delDone; SDelFReader* pDelFReader; + SArray* aDelIdx; // SArray int32_t iDelIdx; - SArray* aDelIdx; // SArray SArray* aDelData; // SArray }; @@ -117,7 +117,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { if (code) goto _err; } - // org data (todo) + // org data + // compress data (todo) int32_t size = sizeof(TABLEID) + tPutBlockData(NULL, &pReader->nBlockData); *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); @@ -137,9 +138,10 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { tPutBlockData((uint8_t*)(&pId[1]), &pReader->nBlockData); tsdbInfo("vgId:%d vnode snapshot read data, fid:%d suid:%" PRId64 " uid:%" PRId64 - " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d", + " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d size:%d", TD_VID(pTsdb->pVnode), pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid, - pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow); + pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow, + size); goto _exit; } @@ -161,7 +163,6 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { if (pReader->pDelFReader == NULL) { if (pDelFile == NULL) { - code = TSDB_CODE_VND_READ_END; goto _exit; } @@ -176,15 +177,20 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->iDelIdx = 0; } - while (pReader->iDelIdx < taosArrayGetSize(pReader->aDelIdx)) { + while (true) { + if (pReader->iDelIdx >= taosArrayGetSize(pReader->aDelIdx)) { + tsdbDelFReaderClose(&pReader->pDelFReader); + break; + } + SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pReader->aDelIdx, pReader->iDelIdx); - int32_t size = 0; pReader->iDelIdx++; code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData, NULL); if (code) goto _err; + int32_t size = 0; for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) { SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData); @@ -193,46 +199,44 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { } } - if (size > 0) { - int64_t n = 0; + if (size == 0) continue; - size = size + sizeof(SSnapDataHdr) + sizeof(TABLEID); - code = tRealloc(ppData, size); - if (code) goto _err; - - // SSnapDataHdr - SSnapDataHdr* pSnapDataHdr = (SSnapDataHdr*)(*ppData + n); - pSnapDataHdr->type = 1; - pSnapDataHdr->size = size; // TODO: size here may incorrect - n += sizeof(SSnapDataHdr); - - // TABLEID - TABLEID* pId = (TABLEID*)(*ppData + n); - pId->suid = pDelIdx->suid; - pId->uid = pDelIdx->uid; - n += sizeof(*pId); - - // DATA - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) { - SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData); - - if (pDelData->version >= pReader->sver && pDelData->version <= pReader->ever) { - n += tPutDelData(*ppData + n, pDelData); - } - } - - goto _exit; + // org data + size = sizeof(TABLEID) + size; + *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); + if (*ppData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } - } - code = TSDB_CODE_VND_READ_END; - tsdbDelFReaderClose(&pReader->pDelFReader); + SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); + pHdr->type = 2; + pHdr->size = size; + + TABLEID* pId = (TABLEID*)(&pHdr[1]); + pId->suid = pDelIdx->suid; + pId->uid = pDelIdx->uid; + int32_t n = sizeof(SSnapDataHdr) + sizeof(TABLEID); + for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) { + SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData); + + if (pDelData->version < pReader->sver) continue; + if (pDelData->version > pReader->ever) continue; + + n += tPutDelData((*ppData) + n, pDelData); + } + + tsdbInfo("vgId:%d vnode snapshot tsdb read del data, suid:%" PRId64 " uid:%d" PRId64 " size:%d", + TD_VID(pTsdb->pVnode), pDelIdx->suid, pDelIdx->uid, size); + + break; + } _exit: return code; _err: - tsdbError("vgId:%d snap read del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb read del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -342,9 +346,6 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { } _exit: - if (*ppData) { - } else { - } return code; _err: diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index ed09f0edc3..26dbf75426 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -237,8 +237,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { // } _exit: - vInfo("vgId:%d vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->type, nData, - pHdr->index); + vInfo("vgId:%d vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->index, + pHdr->type, nData); return code; _err: diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ef6697b3b5..941f3bf558 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -335,8 +335,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_NOT_EXIST, "Table does not exists TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TABLE_ACTION, "Invalid table action") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS, "Table column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_COL_NOT_EXISTS, "Table column not exists") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_READ_END, "Read end") - // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") From 583742089e0b54a69c6d0ca0e87d8525fc474958 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 13 Jul 2022 08:14:20 +0000 Subject: [PATCH 009/124] more vnode snapshot writer --- source/dnode/vnode/src/inc/tsdb.h | 14 ++-- source/dnode/vnode/src/tsdb/tsdbCache.c | 12 ++-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 28 ++++---- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 50 ++++++++++---- source/dnode/vnode/src/tsdb/tsdbUtil.c | 4 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 69 ++++++++++--------- 7 files changed, 105 insertions(+), 76 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index bfb319dce4..2e680511c3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -24,12 +24,12 @@ extern "C" { // tsdbDebug ================ // clang-format off -#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) -#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) -#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) -#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) -#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSD FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) +#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSD ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) +#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TSD WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TSD ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) +#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSD ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSD ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on typedef struct TSDBROW TSDBROW; @@ -137,7 +137,7 @@ int32_t tBlockDataInit(SBlockData *pBlockData); void tBlockDataReset(SBlockData *pBlockData); int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema); void tBlockDataClearData(SBlockData *pBlockData); -void tBlockDataClear(SBlockData *pBlockData); +void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear); int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema); int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 605f4bf3d5..b3e0695c10 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -476,9 +476,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { - // tBlockDataClear(&state->blockData); + // tBlockDataClear(&state->blockData, 1); if (state->pBlockData) { - tBlockDataClear(state->pBlockData); + tBlockDataClear(state->pBlockData, 1); state->pBlockData = NULL; } @@ -582,8 +582,8 @@ _err: state->aBlockIdx = NULL; } if (state->pBlockData) { - // tBlockDataClear(&state->blockData); - tBlockDataClear(state->pBlockData); + // tBlockDataClear(&state->blockData, 1); + tBlockDataClear(state->pBlockData, 1); state->pBlockData = NULL; } @@ -609,8 +609,8 @@ int32_t clearNextRowFromFS(void *iter) { state->aBlockIdx = NULL; } if (state->pBlockData) { - // tBlockDataClear(&state->blockData); - tBlockDataClear(state->pBlockData); + // tBlockDataClear(&state->blockData, 1); + tBlockDataClear(state->pBlockData, 1); state->pBlockData = NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 577ef03a8f..ce14a92734 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1001,10 +1001,10 @@ _exit: static void tsdbCommitDataEnd(SCommitter *pCommitter) { taosArrayDestroy(pCommitter->aBlockIdx); tMapDataClear(&pCommitter->oBlockMap); - tBlockDataClear(&pCommitter->oBlockData); + tBlockDataClear(&pCommitter->oBlockData, 1); taosArrayDestroy(pCommitter->aBlockIdxN); tMapDataClear(&pCommitter->nBlockMap); - tBlockDataClear(&pCommitter->nBlockData); + tBlockDataClear(&pCommitter->nBlockData, 1); tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema); } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 4e7a9d3b04..0ed2e12542 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -979,21 +979,21 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl code = tBlockDataCopy(pBlockData, pBlockData2); if (code) { - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); goto _err; } code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); if (code) { - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); goto _err; } } - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); } tFree(pBuf1); @@ -1115,29 +1115,29 @@ int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *p for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); if (code) { - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); goto _err; } code = tBlockDataCopy(pBlockData, pBlockData2); if (code) { - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); goto _err; } // merge two block data code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); if (code) { - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); goto _err; } } - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); } ASSERT(pBlock->nRow == pBlockData->nRow); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index fbe14088df..563458ad85 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -296,8 +296,8 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { } taosArrayDestroy(pReader->aBlockIdx); tMapDataClear(&pReader->mBlock); - tBlockDataClear(&pReader->oBlockData); - tBlockDataClear(&pReader->nBlockData); + tBlockDataClear(&pReader->oBlockData, 1); + tBlockDataClear(&pReader->nBlockData, 1); if (pReader->pDelFReader) { tsdbDelFReaderClose(&pReader->pDelFReader); @@ -555,11 +555,21 @@ _err: } static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - STsdb* pTsdb = pWriter->pTsdb; - int64_t skey; // todo - int64_t ekey; // todo + int32_t code = 0; + STsdb* pTsdb = pWriter->pTsdb; + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; + TABLEID id = *(TABLEID*)(&pHdr[1]); + int64_t n; + SBlockData bData = {0}; + SBlockData* pBlockData = &bData; + // decode + code = tBlockDataInit(pBlockData); + if (code) goto _err; + n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData); + ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData); + +#if 0 int32_t fid = tsdbKeyFid(skey, pWriter->minutes, pWriter->precision); ASSERT(fid == tsdbKeyFid(ekey, pWriter->minutes, pWriter->precision)); @@ -600,7 +610,10 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 code = tsdbSnapWriteTableData(pWriter, pData, nData); if (code) goto _err; +#endif + tsdbInfo("vgId:%d vnode snapshot tsdb write data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d", TD_VID(pTsdb->pVnode), + id.suid, id.suid, pBlockData->nRow); return code; _err: @@ -755,6 +768,12 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr pWriter->sver = sver; pWriter->ever = ever; + pWriter->minutes = pTsdb->keepCfg.days; + pWriter->precision = pTsdb->keepCfg.precision; + pWriter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; + pWriter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; + pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; + *ppWriter = pWriter; return code; @@ -793,24 +812,29 @@ _err: } int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - int8_t type = pData[0]; + int32_t code = 0; + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; // ts data - if (type == 0) { - code = tsdbSnapWriteData(pWriter, pData + 1, nData - 1); + if (pHdr->type == 1) { + code = tsdbSnapWriteData(pWriter, pData, nData); if (code) goto _err; + + goto _exit; } else { - code = tsdbSnapWriteDataEnd(pWriter); - if (code) goto _err; + if (pWriter->pDataFWriter) { + code = tsdbSnapWriteDataEnd(pWriter); + if (code) goto _err; + } } // del data - if (type == 1) { + if (pHdr->type == 2) { code = tsdbSnapWriteDel(pWriter, pData + 1, nData - 1); if (code) goto _err; } +_exit: return code; _err: diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index db408df944..7667127fc9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1032,11 +1032,11 @@ void tBlockDataReset(SBlockData *pBlockData) { taosArrayClear(pBlockData->aIdx); } -void tBlockDataClear(SBlockData *pBlockData) { +void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear) { tFree((uint8_t *)pBlockData->aVersion); tFree((uint8_t *)pBlockData->aTSKEY); taosArrayDestroy(pBlockData->aIdx); - taosArrayDestroyEx(pBlockData->aColData, tColDataClear); + taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL); } int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema) { diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 26dbf75426..674beb51ec 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -176,7 +176,6 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr pWriter->ever = ever; vInfo("vgId:%d vnode snapshot writer opened", TD_VID(pVnode)); - *ppWriter = pWriter; return code; @@ -189,15 +188,15 @@ _err: int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) { int32_t code = 0; - goto _exit; + // TODO - if (rollback) { - code = vnodeSnapRollback(pWriter); - if (code) goto _err; - } else { - code = vnodeSnapCommit(pWriter); - if (code) goto _err; - } + // if (rollback) { + // code = vnodeSnapRollback(pWriter); + // if (code) goto _err; + // } else { + // code = vnodeSnapCommit(pWriter); + // if (code) goto _err; + // } _exit: vInfo("vgId:%d vnode snapshot writer closed, rollback:%d", TD_VID(pWriter->pVnode), rollback); @@ -214,34 +213,40 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { SSnapDataHdr *pHdr = (SSnapDataHdr *)pData; SVnode *pVnode = pWriter->pVnode; - // ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData); + ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData); + ASSERT(pHdr->index == pWriter->index + 1); + pWriter->index = pHdr->index; - // if (pHdr->type == 0) { - // // meta - // if (pWriter->pMetaSnapWriter == NULL) { - // code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter); - // if (code) goto _err; - // } - - // code = metaSnapWrite(pWriter->pMetaSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); - // if (code) goto _err; - // } else { - // // tsdb - // if (pWriter->pTsdbSnapWriter == NULL) { - // code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter); - // if (code) goto _err; - // } - - // code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); - // if (code) goto _err; - // } - -_exit: vInfo("vgId:%d vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->index, pHdr->type, nData); + + if (pHdr->type == 0) { + // meta + + // if (pWriter->pMetaSnapWriter == NULL) { + // code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter); + // if (code) goto _err; + // } + + // code = metaSnapWrite(pWriter->pMetaSnapWriter, pData , nData); + // if (code) goto _err; + } else { + // tsdb + + if (pWriter->pTsdbSnapWriter == NULL) { + code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter); + if (code) goto _err; + } + + code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData, nData); + if (code) goto _err; + } + +_exit: return code; _err: - vError("vgId:%d vnode snapshot write failed since %s", TD_VID(pVnode), tstrerror(code)); + vError("vgId:%d vnode snapshot write failed since %s, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), + tstrerror(code), pHdr->index, pHdr->type, nData); return code; } \ No newline at end of file From a32c147b3b2bda16abc16b3425fb4d44435e2733 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 13 Jul 2022 09:55:55 +0000 Subject: [PATCH 010/124] more vnode snapshot writer --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 11 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 169 ++++++++++++++------- 2 files changed, 115 insertions(+), 65 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index ce14a92734..470bff1eae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -284,16 +284,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { .fLast = {.commitID = pCommitter->commitID, .size = 0}, .fSma = pRSet->fSma}; } else { - STfs *pTfs = pTsdb->pVnode->pTfs; - SDiskID did = {.level = 0, .id = 0}; - - // TODO: alloc a new disk - // tfsAllocDisk(pTfs, 0, &did); - - // create the directory - tfsMkdirRecurAt(pTfs, pTsdb->path, did); - - wSet = (SDFileSet){.diskId = did, + wSet = (SDFileSet){.diskId = (SDiskID){.level = 0, .id = 0}, .fid = pCommitter->commitFid, .fHead = {.commitID = pCommitter->commitID, .offset = 0, .size = 0}, .fData = {.commitID = pCommitter->commitID, .size = 0}, diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 563458ad85..193f4997b0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -365,36 +365,39 @@ struct STsdbSnapWriter { int32_t minRow; int32_t maxRow; int8_t cmprAlg; + int64_t commitID; // for data file + SBlockData bData; + int32_t fid; SDataFReader* pDataFReader; - SArray* aBlockIdx; + SArray* aBlockIdx; // SArray int32_t iBlockIdx; SBlockIdx* pBlockIdx; - SMapData mBlock; + SMapData mBlock; // SMapData int32_t iBlock; SBlock* pBlock; SBlock block; - SBlockData blockData; + SBlockData bDataR; int32_t iRow; SDataFWriter* pDataFWriter; - SArray* aBlockIdxN; - SBlockIdx* pBlockIdxN; + SArray* aBlockIdxW; // SArray + SBlockIdx* pBlockIdxW; SBlockIdx blockIdx; - SMapData mBlockN; - SBlock* pBlockN; - SBlock blockN; - SBlockData nBlockData; + SMapData mBlockW; // SMapData + SBlock* pBlockW; + SBlock blockW; + SBlockData bDataW; // for del file SDelFReader* pDelFReader; SDelFWriter* pDelFWriter; int32_t iDelIdx; - SArray* aDelIdx; + SArray* aDelIdxR; SArray* aDelData; - SArray* aDelIdxN; + SArray* aDelIdxW; }; static int32_t tsdbSnapRollback(STsdbSnapWriter* pWriter) { @@ -440,7 +443,7 @@ static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, SBlockData* pBlockData = NULL; // todo while (iRow < nRow) { - code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL); + code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL); if (code) goto _err; } @@ -478,14 +481,14 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData, if (pWriter->pBlockIdx == NULL || tTABLEIDCmprFn(&id, pWriter->pBlockIdx) < 0) { int32_t c; - if (pWriter->pBlockIdxN && ((c = tTABLEIDCmprFn(&id, pWriter->pBlockIdxN)) != 0)) { + if (pWriter->pBlockIdxW && ((c = tTABLEIDCmprFn(&id, pWriter->pBlockIdxW)) != 0)) { ASSERT(c > 0); code = tsdbSnapWriteTableDataEnd(pWriter); if (code) goto _err; } - if (pWriter->pBlockIdxN == NULL) { + if (pWriter->pBlockIdxW == NULL) { pWriter->pBlockIdx = &pWriter->blockIdx; pWriter->pBlockIdx->suid = id.suid; pWriter->pBlockIdx->uid = id.uid; @@ -496,12 +499,12 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData, int32_t nRow = 0; // todo SBlockData* pBlockData = NULL; // todo for (int32_t iRow = 0; iRow < nRow; iRow++) { - code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL); + code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL); if (code) goto _err; - if (pWriter->nBlockData.nRow > pWriter->maxRow * 4 / 5) { - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->nBlockData, NULL, NULL, pWriter->pBlockIdxN, - pWriter->pBlockN, pWriter->cmprAlg); + if (pWriter->bDataW.nRow > pWriter->maxRow * 4 / 5) { + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + pWriter->pBlockW, pWriter->cmprAlg); if (code) goto _err; } } @@ -512,7 +515,7 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData, if (pWriter->pBlock->last) break; if (tBlockCmprFn(&(SBlock){.minKey = {0}, .maxKey = {0}}, pWriter->pBlock) >= 0) break; - code = tMapDataPutItem(&pWriter->mBlockN, pWriter->pBlock, tPutBlock); + code = tMapDataPutItem(&pWriter->mBlockW, pWriter->pBlock, tPutBlock); if (code) goto _err; } @@ -533,15 +536,15 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData, SBlockData* pBlockData = NULL; for (int32_t iRow = 0; iRow < nRow; iRow++) { - code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL); + code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL); if (code) goto _err; - if (pWriter->nBlockData.nRow >= pWriter->maxRow * 4 / 5) { - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->nBlockData, NULL, NULL, pWriter->pBlockIdxN, - pWriter->pBlockN, pWriter->cmprAlg); + if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) { + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + pWriter->pBlockW, pWriter->cmprAlg); if (code) goto _err; - tBlockDataClearData(&pWriter->nBlockData); + tBlockDataClearData(&pWriter->bDataW); } } } @@ -560,60 +563,77 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; TABLEID id = *(TABLEID*)(&pHdr[1]); int64_t n; - SBlockData bData = {0}; - SBlockData* pBlockData = &bData; // decode - code = tBlockDataInit(pBlockData); - if (code) goto _err; + SBlockData* pBlockData = &pWriter->bData; n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData); ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData); -#if 0 - int32_t fid = tsdbKeyFid(skey, pWriter->minutes, pWriter->precision); - ASSERT(fid == tsdbKeyFid(ekey, pWriter->minutes, pWriter->precision)); + // open file + TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); + TSDBKEY keyLast = tBlockDataLastKey(pBlockData); - // begin + int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision); + ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision)); if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) { - code = tsdbSnapWriteDataEnd(pWriter); + code = tsdbSnapWriteDataEnd(pWriter); // todo if (code) goto _err; pWriter->fid = fid; - SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ); - // reader + + // read + SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ); // todo: check nState is valid if (pSet) { - // open code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet); if (code) goto _err; - // SBlockIdx code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx, NULL); if (code) goto _err; } else { + ASSERT(pWriter->pDataFReader == NULL); taosArrayClear(pWriter->aBlockIdx); } pWriter->iBlockIdx = 0; + pWriter->pBlockIdx = NULL; + tMapDataReset(&pWriter->mBlock); + pWriter->iBlock = 0; + pWriter->pBlock = NULL; + tBlockDataReset(&pWriter->bDataR); + pWriter->iRow = 0; - // writer - SDFileSet wSet = {0}; - if (pSet == NULL) { - wSet = (SDFileSet){0}; // todo + // write + SDFileSet wSet; + + if (pSet) { + wSet = (SDFileSet){.diskId = pSet->diskId, + .fid = fid, + .fHead = {.commitID = pWriter->commitID, .offset = 0, .size = 0}, + .fData = pSet->fData, + .fLast = {.commitID = pWriter->commitID, .size = 0}, + .fSma = pSet->fSma}; } else { - wSet = (SDFileSet){0}; // todo + wSet = (SDFileSet){.diskId = (SDiskID){.level = 0, .id = 0}, + .fid = fid, + .fHead = {.commitID = pWriter->commitID, .offset = 0, .size = 0}, + .fData = {.commitID = pWriter->commitID, .size = 0}, + .fLast = {.commitID = pWriter->commitID, .size = 0}, + .fSma = {.commitID = pWriter->commitID, .size = 0}}; } code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet); if (code) goto _err; - taosArrayClear(pWriter->aBlockIdxN); + taosArrayClear(pWriter->aBlockIdxW); + pWriter->pBlockIdxW = NULL; + tMapDataReset(&pWriter->mBlockW); + pWriter->pBlockW = NULL; + tBlockDataReset(&pWriter->bDataW); } - code = tsdbSnapWriteTableData(pWriter, pData, nData); - if (code) goto _err; -#endif + // write data block (todo) - tsdbInfo("vgId:%d vnode snapshot tsdb write data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d", TD_VID(pTsdb->pVnode), - id.suid, id.suid, pBlockData->nRow); + tsdbInfo("vgId:%d vnode snapshot tsdb write data, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d", + TD_VID(pTsdb->pVnode), fid, id.suid, id.suid, pBlockData->nRow); return code; _err: @@ -633,7 +653,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL); if (code) goto _err; - code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdx, NULL); + code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR, NULL); if (code) goto _err; } @@ -653,8 +673,8 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 SDelIdx delIdx; int8_t toBreak = 0; - if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx)) { - pDelIdx = taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx); + if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR)) { + pDelIdx = taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); } if (pDelIdx) { @@ -695,7 +715,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx); if (code) goto _err; - if (taosArrayPush(pWriter->aDelIdxN, &delIdx) == NULL) { + if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -716,8 +736,8 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { STsdb* pTsdb = pWriter->pTsdb; if (pWriter->pDelFWriter == NULL) goto _exit; - for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx); pWriter->iDelIdx++) { - SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx); + for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) { + SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL); if (code) goto _err; @@ -726,7 +746,7 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx); if (code) goto _err; - if (taosArrayPush(pWriter->aDelIdx, &delIdx) == NULL) { + if (taosArrayPush(pWriter->aDelIdxR, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -768,11 +788,50 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr pWriter->sver = sver; pWriter->ever = ever; + // config pWriter->minutes = pTsdb->keepCfg.days; pWriter->precision = pTsdb->keepCfg.precision; pWriter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; pWriter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; + pWriter->commitID = pTsdb->pVnode->state.commitID; + + // for data file + code = tBlockDataInit(&pWriter->bData); + + if (code) goto _err; + pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pWriter->aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + code = tBlockDataInit(&pWriter->bDataR); + if (code) goto _err; + + pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx)); + if (pWriter->aBlockIdxW == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + code = tBlockDataInit(&pWriter->bDataW); + if (code) goto _err; + + // for del file + pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx)); + if (pWriter->aDelIdxR == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pWriter->aDelData = taosArrayInit(0, sizeof(SDelData)); + if (pWriter->aDelData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pWriter->aDelIdxW = taosArrayInit(0, sizeof(SDelIdx)); + if (pWriter->aDelIdxW == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } *ppWriter = pWriter; return code; From e6e9926d3887688ada9ca076f8b5fa32eadbe2c8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 13 Jul 2022 12:02:59 +0000 Subject: [PATCH 011/124] more vnode snapshot writer --- source/dnode/vnode/src/inc/tsdb.h | 5 --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 -- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 41 ++++++++++++++++++++-- source/dnode/vnode/src/tsdb/tsdbUtil.c | 17 --------- 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2e680511c3..a2ac995c13 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -115,7 +115,6 @@ int32_t tGetBlock(uint8_t *p, void *ph); int32_t tBlockCmprFn(const void *p1, const void *p2); bool tBlockHasSma(SBlock *pBlock); // SBlockIdx -void tBlockIdxReset(SBlockIdx *pBlockIdx); int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tCmprBlockIdx(void const *lhs, void const *rhs); @@ -361,10 +360,6 @@ struct TSDBROW { struct SBlockIdx { int64_t suid; int64_t uid; - TSKEY minKey; - TSKEY maxKey; - int64_t minVersion; - int64_t maxVersion; int64_t offset; int64_t size; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index b3e0695c10..723caca5d7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -500,9 +500,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (code) goto _err; /* if (state->pBlockIdx) { */ - /* tBlockIdxReset(state->blockIdx); */ /* } */ - /* tBlockIdxReset(state->blockIdx); */ /* code = tMapDataSearch(&state->blockIdxMap, state->pBlockIdxExp, tGetBlockIdx, tCmprBlockIdx, * &state->blockIdx); */ diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 193f4997b0..f3b321dfae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -383,14 +383,15 @@ struct STsdbSnapWriter { int32_t iRow; SDataFWriter* pDataFWriter; - SArray* aBlockIdxW; // SArray SBlockIdx* pBlockIdxW; SBlockIdx blockIdx; - SMapData mBlockW; // SMapData SBlock* pBlockW; SBlock blockW; SBlockData bDataW; + SMapData mBlockW; // SMapData + SArray* aBlockIdxW; // SArray + // for del file SDelFReader* pDelFReader; SDelFWriter* pDelFWriter; @@ -460,6 +461,7 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWrite) { return code; } +#if 0 static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; TABLEID id = {0}; // TODO @@ -556,6 +558,38 @@ _err: tsdbError("vgId:%d tsdb snapshot write table data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } +#endif + +static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) { + int32_t code = 0; + SBlockData* pBlockData = &pWriter->bData; + + if (pWriter->pDataFReader == NULL) { + // no old data + + // end last table data commit if id not same + if (pWriter->pBlockIdxW) { + int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id); + if (c < 0) { + // commit last table data and reset (todo) + pWriter->pBlockIdxW = NULL; + } else if (c > 0) { + ASSERT(0); + } + } + + // start a new table data if need + if (pWriter->pBlockIdxW == NULL) { + pWriter->pBlockIdxW = &pWriter->blockIdx; + pWriter->pBlockIdxW->suid = id.suid; + pWriter->pBlockIdxW->uid = id.uid; + } + + } else { + } + + return code; +} static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; @@ -630,7 +664,8 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 tBlockDataReset(&pWriter->bDataW); } - // write data block (todo) + code = tsdbSnapWriteDataImpl(pWriter, id); + if (code) goto _err; tsdbInfo("vgId:%d vnode snapshot tsdb write data, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d", TD_VID(pTsdb->pVnode), fid, id.suid, id.suid, pBlockData->nRow); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 7667127fc9..be1b848d96 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -189,25 +189,12 @@ static FORCE_INLINE int32_t tGetTSDBKEY(uint8_t *p, TSDBKEY *pKey) { } // SBlockIdx ====================================================== -void tBlockIdxReset(SBlockIdx *pBlockIdx) { - pBlockIdx->minKey = TSKEY_MAX; - pBlockIdx->maxKey = TSKEY_MIN; - pBlockIdx->minVersion = VERSION_MAX; - pBlockIdx->maxVersion = VERSION_MIN; - pBlockIdx->offset = -1; - pBlockIdx->size = -1; -} - int32_t tPutBlockIdx(uint8_t *p, void *ph) { int32_t n = 0; SBlockIdx *pBlockIdx = (SBlockIdx *)ph; n += tPutI64(p ? p + n : p, pBlockIdx->suid); n += tPutI64(p ? p + n : p, pBlockIdx->uid); - n += tPutI64(p ? p + n : p, pBlockIdx->minKey); - n += tPutI64(p ? p + n : p, pBlockIdx->maxKey); - n += tPutI64v(p ? p + n : p, pBlockIdx->minVersion); - n += tPutI64v(p ? p + n : p, pBlockIdx->maxVersion); n += tPutI64v(p ? p + n : p, pBlockIdx->offset); n += tPutI64v(p ? p + n : p, pBlockIdx->size); @@ -220,10 +207,6 @@ int32_t tGetBlockIdx(uint8_t *p, void *ph) { n += tGetI64(p + n, &pBlockIdx->suid); n += tGetI64(p + n, &pBlockIdx->uid); - n += tGetI64(p + n, &pBlockIdx->minKey); - n += tGetI64(p + n, &pBlockIdx->maxKey); - n += tGetI64v(p + n, &pBlockIdx->minVersion); - n += tGetI64v(p + n, &pBlockIdx->maxVersion); n += tGetI64v(p + n, &pBlockIdx->offset); n += tGetI64v(p + n, &pBlockIdx->size); From 20fb3563889666d13546e3ac8960ea85ae8efcdb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 13 Jul 2022 12:31:18 +0000 Subject: [PATCH 012/124] more vnode snap writer --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 37 ++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index f3b321dfae..8927155173 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -583,12 +583,49 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) { pWriter->pBlockIdxW = &pWriter->blockIdx; pWriter->pBlockIdxW->suid = id.suid; pWriter->pBlockIdxW->uid = id.uid; + + pWriter->pBlockW = &pWriter->blockW; + tBlockReset(pWriter->pBlockW); + + tBlockDataReset(&pWriter->bDataW); + + tMapDataReset(&pWriter->mBlockW); } + // set block schema (todo) + + // add rows + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + TSDBROW* pRow = &tsdbRowFromBlockData(pBlockData, iRow); + + code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + if (code) goto _err; + + if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) { + // write the block to file + pWriter->pBlockW->last = 0; + + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + pWriter->pBlockW, pWriter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pWriter->mBlockW, pWriter->pBlockW, tPutBlock); + if (code) goto _err; + + // reset + tBlockReset(pWriter->pBlockW); + tBlockDataReset(&pWriter->bDataW); + } + } } else { } return code; + +_err: + tsdbError("vgId:%d vnode snapshot tsdb write data impl failed since %s", TD_VID(pWriter->pTsdb->pVnode), + tstrerror(code)); + return code; } static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { From 25a2d04b85c669c7d4c68f9add1bc26cb767f6a0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 14 Jul 2022 07:37:53 +0000 Subject: [PATCH 013/124] more vnode snapshot writer --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 153 ++++++--------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 40 ++++++ 4 files changed, 85 insertions(+), 111 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f0a6b6505d..6b5f795eb0 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -138,7 +138,7 @@ void *tsdbGetIdx(SMeta *pMeta); void *tsdbGetIvtIdx(SMeta *pMeta); int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, void **pReader); -int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray* pTableUids); +int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids); int32_t tsdbLastrowReaderClose(void *pReader); int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a2ac995c13..3f667a7465 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -135,6 +135,7 @@ int32_t tGetColData(uint8_t *p, SColData *pColData); int32_t tBlockDataInit(SBlockData *pBlockData); void tBlockDataReset(SBlockData *pBlockData); int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema); +int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom); void tBlockDataClearData(SBlockData *pBlockData); void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear); int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 8927155173..5c5e81cd49 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -383,11 +383,10 @@ struct STsdbSnapWriter { int32_t iRow; SDataFWriter* pDataFWriter; - SBlockIdx* pBlockIdxW; - SBlockIdx blockIdx; - SBlock* pBlockW; + SBlockIdx* pBlockIdxW; // NULL when no committing table SBlock blockW; SBlockData bDataW; + SBlockIdx blockIdxW; SMapData mBlockW; // SMapData SArray* aBlockIdxW; // SArray @@ -455,110 +454,40 @@ _err: return code; } -static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWrite) { +static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; - // TODO - return code; -} -#if 0 -static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - TABLEID id = {0}; // TODO + ASSERT(pWriter->pBlockIdxW != NULL); - // skip - while (pWriter->pBlockIdx && tTABLEIDCmprFn(&id, pWriter->pBlockIdx) < 0) { - code = tsdbSnapWriteTableDataEnd(pWriter); + // write remain data if has + if (pWriter->bDataW.nRow > 0) { + if (pWriter->bDataW.nRow >= pWriter->minRow) { + pWriter->blockW.last = 0; + } else { + pWriter->blockW.last = 1; + } + + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + &pWriter->blockW, pWriter->cmprAlg); if (code) goto _err; - pWriter->iBlockIdx++; - if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { - pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); - } else { - pWriter->pBlockIdx = NULL; - } + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + if (code) goto _err; } - // new or merge - if (pWriter->pBlockIdx == NULL || tTABLEIDCmprFn(&id, pWriter->pBlockIdx) < 0) { - int32_t c; + code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); + if (code) goto _err; - if (pWriter->pBlockIdxW && ((c = tTABLEIDCmprFn(&id, pWriter->pBlockIdxW)) != 0)) { - ASSERT(c > 0); - - code = tsdbSnapWriteTableDataEnd(pWriter); - if (code) goto _err; - } - - if (pWriter->pBlockIdxW == NULL) { - pWriter->pBlockIdx = &pWriter->blockIdx; - pWriter->pBlockIdx->suid = id.suid; - pWriter->pBlockIdx->uid = id.uid; - } - - // loop to write the data - TSDBROW* pRow = NULL; // todo - int32_t nRow = 0; // todo - SBlockData* pBlockData = NULL; // todo - for (int32_t iRow = 0; iRow < nRow; iRow++) { - code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL); - if (code) goto _err; - - if (pWriter->bDataW.nRow > pWriter->maxRow * 4 / 5) { - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - pWriter->pBlockW, pWriter->cmprAlg); - if (code) goto _err; - } - } - } else { - // skip - while (true) { - if (pWriter->pBlock == NULL) break; - if (pWriter->pBlock->last) break; - if (tBlockCmprFn(&(SBlock){.minKey = {0}, .maxKey = {0}}, pWriter->pBlock) >= 0) break; - - code = tMapDataPutItem(&pWriter->mBlockW, pWriter->pBlock, tPutBlock); - if (code) goto _err; - } - - if (pWriter->pBlock) { - if (pWriter->pBlock->last) { - // load the last block and merge with the data (todo) - } else { - int32_t c = tBlockCmprFn(&(SBlock){0 /*TODO*/}, pWriter->pBlock); - - if (c > 0) { - // commit until pWriter->pBlock (todo) - } else { - // load the block and merge with the data (todo) - } - } - } else { - int32_t nRow = 0; - SBlockData* pBlockData = NULL; - - for (int32_t iRow = 0; iRow < nRow; iRow++) { - code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL); - if (code) goto _err; - - if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) { - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - pWriter->pBlockW, pWriter->cmprAlg); - if (code) goto _err; - - tBlockDataClearData(&pWriter->bDataW); - } - } - } + if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } return code; _err: - tsdbError("vgId:%d tsdb snapshot write table data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -#endif static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) { int32_t code = 0; @@ -567,11 +496,16 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) { if (pWriter->pDataFReader == NULL) { // no old data - // end last table data commit if id not same + // end last table write if need if (pWriter->pBlockIdxW) { - int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id); + int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id); + if (c < 0) { - // commit last table data and reset (todo) + // end last table data write + code = tsdbSnapWriteTableDataEnd(pWriter); + if (code) goto _err; + + // reset pWriter->pBlockIdxW = NULL; } else if (c > 0) { ASSERT(0); @@ -580,41 +514,40 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) { // start a new table data if need if (pWriter->pBlockIdxW == NULL) { - pWriter->pBlockIdxW = &pWriter->blockIdx; + pWriter->pBlockIdxW = &pWriter->blockIdxW; pWriter->pBlockIdxW->suid = id.suid; pWriter->pBlockIdxW->uid = id.uid; - pWriter->pBlockW = &pWriter->blockW; - tBlockReset(pWriter->pBlockW); + tBlockReset(&pWriter->blockW); tBlockDataReset(&pWriter->bDataW); tMapDataReset(&pWriter->mBlockW); } - // set block schema (todo) + // set block schema + code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData); + if (code) goto _err; // add rows for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - TSDBROW* pRow = &tsdbRowFromBlockData(pBlockData, iRow); + TSDBROW row = tsdbRowFromBlockData(pBlockData, iRow); - code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + code = tBlockDataAppendRow(&pWriter->bDataW, &row, NULL); if (code) goto _err; if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) { - // write the block to file - pWriter->pBlockW->last = 0; - + pWriter->blockW.last = 0; code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - pWriter->pBlockW, pWriter->cmprAlg); + &pWriter->blockW, pWriter->cmprAlg); if (code) goto _err; - code = tMapDataPutItem(&pWriter->mBlockW, pWriter->pBlockW, tPutBlock); + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); if (code) goto _err; // reset - tBlockReset(pWriter->pBlockW); - tBlockDataReset(&pWriter->bDataW); + tBlockReset(&pWriter->blockW); + tBlockDataClearData(&pWriter->bDataW); } } } else { @@ -647,6 +580,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision); ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision)); if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) { + // end last file data write if need code = tsdbSnapWriteDataEnd(pWriter); // todo if (code) goto _err; @@ -697,7 +631,6 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 taosArrayClear(pWriter->aBlockIdxW); pWriter->pBlockIdxW = NULL; tMapDataReset(&pWriter->mBlockW); - pWriter->pBlockW = NULL; tBlockDataReset(&pWriter->bDataW); } @@ -709,7 +642,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 return code; _err: - tsdbError("vgId:%d tsdb snapshot write data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb write data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index be1b848d96..6320a48e77 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1132,6 +1132,46 @@ _err: return code; } +int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom) { + int32_t code = 0; + + int32_t iColData = 0; + for (int32_t iColDataFrom = 0; iColDataFrom < taosArrayGetSize(pBlockDataFrom->aIdx); iColDataFrom++) { + SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColDataFrom); + + while (true) { + SColData *pColData; + if (iColData < taosArrayGetSize(pBlockData->aIdx)) { + pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + } else { + pColData = NULL; + } + + if (pColData == NULL || pColData->cid > pColDataFrom->cid) { + code = tBlockDataAddColData(pBlockData, iColData, &pColData); + if (code) goto _exit; + + tColDataInit(pColData, pColDataFrom->cid, pColData->type, pColData->smaOn); + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); + if (code) goto _exit; + } + + iColData++; + break; + } else if (pColData->cid == pColDataFrom->cid) { + iColData++; + break; + } else { + iColData++; + } + } + } + +_exit: + return code; +} + int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData) { int32_t code = 0; From 4ad539f24291f7f92a4be14c3387ce0dc958e79d Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 14 Jul 2022 16:27:26 +0800 Subject: [PATCH 014/124] test:temp commit --- tests/system-test/2-query/queryQnode.py | 25 +++++++++++++++++++++++++ tests/system-test/2-query/tsbsQuery.py | 5 +++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/queryQnode.py b/tests/system-test/2-query/queryQnode.py index 3fdc09478d..0460ccf135 100644 --- a/tests/system-test/2-query/queryQnode.py +++ b/tests/system-test/2-query/queryQnode.py @@ -392,6 +392,31 @@ class TDTestCase: tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") assert unionallQnode==tdSql.queryResult + queryPolicy=1 + + tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy) + tdSql.query("show local variables;") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "queryPolicy" : + if int(tdSql.queryResult[i][1]) == int(queryPolicy): + tdLog.success('alter queryPolicy to %d successfully'%queryPolicy) + else : + tdLog.debug(tdSql.queryResult) + tdLog.exit("alter queryPolicy to %d failed"%queryPolicy) + tdSql.execute("reset query cache") + + tdSql.execute("use db1;") + tdSql.query("show dnodes;") + dnodeId=tdSql.getData(0,0) + tdSql.query("select max(c1) from stb10;") + assert maxQnode==tdSql.getData(0,0) + tdSql.query("select min(c1) from stb11;") + assert minQnode==tdSql.getData(0,0) + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + assert unionQnode==tdSql.queryResult + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + assert unionallQnode==tdSql.queryResult + # test case : queryPolicy = 2 def test_case2(self): self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 10, 2, 1*10) diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 8180f511e2..d04993e558 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -63,7 +63,8 @@ class TDTestCase: tdSql.execute("use db_tsbs") # test interval and partition - tdSql.query(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet; ") + tdSql.queryt(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet; ") + print(tdSql.queryResult) parRows=tdSql.queryRows tdSql.query(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet interval(10m); ") # tdSql.checkRows(parRows) @@ -85,7 +86,7 @@ class TDTestCase: # tdSql.checkRows(10) # test partition interval Pseudo time-column - tdSql.query("SELECT count(ms1)/144 FROM (SELECT _wstartts as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;") + tdSql.queryt("SELECT count(ms1)/144 FROM (SELECT _wstartts as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;") # test From b554559889adec4047d7242089aada54bccb5ae1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 14 Jul 2022 09:36:23 +0000 Subject: [PATCH 015/124] more vnode snapshot --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 118 +++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 5c5e81cd49..51514f1257 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -489,9 +489,66 @@ _err: return code; } +static int32_t tsdbSnapWriteTableDataAhead(STsdbSnapWriter* pWriter, TABLEID id) { + int32_t code = 0; + + if (pWriter->pDataFReader == NULL) goto _exit; + + while (true) { + if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break; + + SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); + int32_t c = tTABLEIDCmprFn(pBlockIdx, &id); + + if (c >= 0) break; + + pWriter->iBlockIdx++; + + code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL); + if (code) goto _err; + + SBlock block; + tMapDataReset(&pWriter->mBlockW); + for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) { + tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock); + + if (block.last) { + code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); + if (code) goto _err; + + tBlockReset(&block); + block.last = 1; + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, + pWriter->cmprAlg); + if (code) goto _err; + } + + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + if (code) goto _err; + } + + SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; + code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, &blockIdx); + if (code) goto _err; + + if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + +_exit: + return code; + +_err: + return code; +} + static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) { int32_t code = 0; SBlockData* pBlockData = &pWriter->bData; + TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); + TSDBKEY keyLast = tBlockDataLastKey(pBlockData); if (pWriter->pDataFReader == NULL) { // no old data @@ -551,6 +608,67 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) { } } } else { + // has old data + + // TABLE ================================================== + // end last table data if id not same (todo) + if (pWriter->pBlockIdxW) { + int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id); + if (c < 0) { + } else if (c > 0) { + ASSERT(0); + } + } + + // start new table data if need (todo) + if (pWriter->pBlockIdxW == NULL) { + // commit table data ahead + code = tsdbSnapWriteTableDataAhead(pWriter, id); + if (code) goto _err; + + // reader + if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { + pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); + int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id); + if (c) { + pWriter->pBlockIdx = NULL; + } + } else { + pWriter->pBlockIdx = NULL; + } + + if (pWriter->pBlockIdx) { + code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlockW, NULL); + if (code) goto _err; + + pWriter->iBlock = 0; + } + + // writer + pWriter->pBlockIdxW = &pWriter->blockIdxW; + pWriter->pBlockIdxW->suid = id.suid; + pWriter->pBlockIdxW->uid = id.uid; + + tBlockReset(&pWriter->blockW); + tBlockDataReset(&pWriter->bDataW); + tMapDataReset(&pWriter->mBlockW); + } + + // BLOCK ================================================== + // write block ahead + while (true) { + if (pWriter->iBlock >= pWriter->mBlock.nItem) break; + + SBlock block; + tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); + + if (tsdbKeyCmprFn(&block.maxKey, &keyFirst) >= 0) break; + + pWriter->iBlock++; + + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + if (code) goto _err; + } } return code; From edfd4b5175d00d5a0db23f88d648afd6845abd1c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 14 Jul 2022 18:51:34 +0800 Subject: [PATCH 016/124] enh: rsma support async commit --- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/sma/smaCommit.c | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7b298ba830..394c314819 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -167,6 +167,7 @@ int32_t smaBegin(SSma* pSma); int32_t smaPreCommit(SSma* pSma); int32_t smaCommit(SSma* pSma); int32_t smaPostCommit(SSma* pSma); +int32_t smaAsyncCommit(SSma* pSma); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index e241c14fc7..889c4adf24 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -18,6 +18,7 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma); static int32_t tdProcessRSmaCommitImpl(SSma *pSma); static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma); /** * @brief Only applicable to Rollup SMA @@ -43,6 +44,14 @@ int32_t smaCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); } */ int32_t smaPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); } +/** + * @brief Only applicable to Rollup SMA + * + * @param pSma + * @return int32_t + */ +int32_t smaAsyncCommit(SSma *pSma) { return tdProcessRSmaAsyncCommitImpl(pSma); } + /** * @brief set rsma trigger stat active * @@ -224,3 +233,15 @@ static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) { regfree(®ex); return TSDB_CODE_SUCCESS; } + +/** + * @brief Rsma async commit implementation + * 1) Wait all running fetch task finish to fetch and put submitMsg into level 2/3 wQueue(blocking level 1 write) + * + * @param pSma + * @return int32_t + */ +static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { + // TODO + return TSDB_CODE_SUCCESS; +} From e01297d09dae23ed8a254461e16beb24e6f2f4e2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 14 Jul 2022 19:29:23 +0800 Subject: [PATCH 017/124] refactor(query): optimize table scan by reusing reader --- source/libs/executor/src/scanoperator.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0f44ac48a4..f5b6b0c7a6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -516,10 +516,16 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); - tsdbReaderClose(pInfo->dataReader); - int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, - GET_TASKID(pTaskInfo)); + int32_t code; + if (pInfo->dataReader != NULL) { + code = tsdbReaderReset(pInfo->dataReader, &pInfo->cond); + ASSERT(code == 0); + } else { + code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, + GET_TASKID(pTaskInfo)); + ASSERT(code == 0); + } } SSDataBlock* result = doTableScanGroup(pOperator); @@ -871,6 +877,7 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ } resetTableScanInfo(pInfo->pTableScanOp->info, &win); + pInfo->pTableScanOp->status = OP_OPENED; return true; } @@ -1193,8 +1200,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } } - ASSERT(pInfo->pRes->pDataBlock != NULL); - // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, From e8a0cf5b75b11f029be943a7b49303f8f05dd53a Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 14 Jul 2022 21:11:39 +0800 Subject: [PATCH 018/124] test:add test case of tsbs query --- tests/pytest/util/sql.py | 47 ++++++++++++++++++-------- tests/system-test/2-query/tsbsQuery.py | 15 ++++---- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 585594e035..bc6727a8fc 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -79,22 +79,39 @@ class TDSql: self.queryResult = None tdLog.info("sql:%s, expect error occured" % (sql)) - def query(self, sql, row_tag=None): + def query(self, sql, row_tag=None,queyTimes=10): self.sql = sql - try: - self.cursor.execute(sql) - self.queryResult = self.cursor.fetchall() - self.queryRows = len(self.queryResult) - self.queryCols = len(self.cursor.description) - except Exception as e: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - traceback.print_exc() - raise Exception(repr(e)) - if row_tag: - return self.queryResult - return self.queryRows + i=1 + while i <= queyTimes: + try: + self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(self.cursor.description) + if row_tag: + return self.queryResult + return self.queryRows + except Exception as e: + i+=1 + tdLog.notice("Try to query again, query times: %d "%i) + pass + else: + try: + tdLog.notice("Try the last query ") + self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(self.cursor.description) + if row_tag: + return self.queryResult + return self.queryRows + except Exception as e: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + traceback.print_exc() + raise Exception(repr(e)) + def is_err_sql(self, sql): err_flag = True diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index d04993e558..5fe025ea07 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -63,11 +63,11 @@ class TDTestCase: tdSql.execute("use db_tsbs") # test interval and partition - tdSql.queryt(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet; ") + tdSql.query(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet; ") print(tdSql.queryResult) parRows=tdSql.queryRows tdSql.query(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet interval(10m); ") - # tdSql.checkRows(parRows) + tdSql.checkRows(parRows) # test insert into @@ -78,16 +78,19 @@ class TDTestCase: # test paitition interval fill - # tdSql.query("SELECT name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0) ;") + tdSql.query("SELECT name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0) ;") - # # test partition interval limit - # tdSql.query("SELECT ts,model,floor(2*(sum(nzs)/count(nzs)))/floor(2*(sum(nzs)/count(nzs))) AS broken_down FROM (SELECT ts,model, status/status AS nzs FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' ) WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition BY model,ts interval(10m) limit 10;") + # test partition interval limit (PRcore-TD-17410) + # tdSql.query("select name,driver from (SELECT name,driver,fleet ,avg(velocity) as mean_velocity FROM readings partition BY name,driver,fleet interval (10m) limit 1);") # tdSql.checkRows(10) # test partition interval Pseudo time-column - tdSql.queryt("SELECT count(ms1)/144 FROM (SELECT _wstartts as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;") + tdSql.query("SELECT count(ms1)/144 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;") + #PR TD-17413 + tdSql.query("SELECT ms1 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-01T00:01:01Z';") + tdSql.checkRows(0) # test def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring From d554bf6588c6cf35f6a315e05b9333efb076e1af Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 14 Jul 2022 21:22:43 +0800 Subject: [PATCH 019/124] test:modify test frame that add multiple retries for sql execution and query --- tests/pytest/util/sql.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index bc6727a8fc..4961355f06 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -283,16 +283,27 @@ class TDSql: time.sleep(1) continue - def execute(self, sql): + def execute(self, sql,queyTimes=10): self.sql = sql - try: - self.affectedRows = self.cursor.execute(sql) - except Exception as e: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - raise Exception(repr(e)) - return self.affectedRows + i=1 + while i <= queyTimes: + try: + self.affectedRows = self.cursor.execute(sql) + return self.affectedRows + except Exception as e: + i+=1 + tdLog.notice("Try to execute sql again, query times: %d "%i) + pass + else: + try: + tdLog.notice("Try the last execute sql ") + self.affectedRows = self.cursor.execute(sql) + return self.affectedRows + except Exception as e: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) def checkAffectedRows(self, expectAffectedRows): if self.affectedRows != expectAffectedRows: From f1e37cde780b182fce99a22be55c617fae7eb403 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 15 Jul 2022 10:21:47 +0800 Subject: [PATCH 020/124] fix: fix msg disorder issue --- source/libs/qworker/inc/qwInt.h | 2 +- source/libs/qworker/src/qwDbg.c | 2 +- source/libs/qworker/src/qwMsg.c | 3 +-- source/libs/qworker/src/qworker.c | 38 +++++++++++-------------------- 4 files changed, 16 insertions(+), 29 deletions(-) diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 539643c390..6c9871425b 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -135,7 +135,6 @@ typedef struct SQWTaskCtx { int32_t execId; bool queryRsped; - bool queryFetched; bool queryEnd; bool queryContinue; bool queryInQueue; @@ -228,6 +227,7 @@ typedef struct SQWorkerMgmt { #define QW_SET_EVENT_PROCESSED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_PROCESSED) #define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase) +#define QW_SET_PHASE(ctx, _value) do { if ((_value) != QW_PHASE_PRE_FETCH && (_value) != QW_PHASE_POST_FETCH) { atomic_store_8(&(ctx)->phase, _value); } } while (0) #define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code) #define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 869eedf8f6..66a89cf57a 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -166,7 +166,7 @@ int32_t qwDbgResponseREdirect(SQWMsg *qwMsg, SQWTaskCtx *ctx) { } if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType) { - ctx->phase = QW_PHASE_POST_QUERY; + QW_SET_PHASE(ctx, QW_PHASE_POST_QUERY); qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); gQWDebug.tmp = false; return TSDB_CODE_SUCCESS; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 73110472f7..e1f16b9719 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -374,8 +374,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int qwMsg.msgInfo.needFetch = msg->needFetch; char * sql = strndup(msg->msg, msg->sqlLen); - QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, sql:%s", node, TMSG_INFO(pMsg->msgType), pMsg->info.handle, sql); - + QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, SQL:%s", node, TMSG_INFO(pMsg->msgType), pMsg->info.handle, sql); QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql)); QW_SCH_TASK_DLOG("processQuery end, node:%p", node); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 3e8ced318c..fb8ce09615 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -293,11 +293,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_LOCK(QW_WRITE, &ctx->lock); - if (QW_PHASE_PRE_FETCH == phase) { - atomic_store_8((int8_t *)&ctx->queryFetched, true); - } else { - atomic_store_8(&ctx->phase, phase); - } + QW_SET_PHASE(ctx, phase); if (atomic_load_8((int8_t *)&ctx->queryEnd)) { QW_TASK_ELOG_E("query already end"); @@ -370,6 +366,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } _return: + if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); @@ -390,7 +387,6 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp int32_t code = 0; SQWTaskCtx *ctx = NULL; SRpcHandleInfo connInfo = {0}; - SRpcHandleInfo *rspConnection = NULL; QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase)); @@ -403,13 +399,6 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } - if (QW_PHASE_POST_QUERY == phase) { - connInfo = ctx->ctrlConnInfo; - rspConnection = &connInfo; - - ctx->queryRsped = true; - } - if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { if (QW_PHASE_POST_FETCH == phase) { QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase)); @@ -437,17 +426,16 @@ _return: qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC); } - if (rspConnection) { - qwBuildAndSendQueryRsp(input->msgType + 1, rspConnection, code, ctx); - QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", rspConnection->handle, code, tstrerror(code)); + if (QW_PHASE_POST_QUERY == phase) { + ctx->queryRsped = true; + qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); } if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); - if (QW_PHASE_POST_FETCH != phase) { - atomic_store_8(&ctx->phase, phase); - } + QW_SET_PHASE(ctx, phase); QW_UNLOCK(QW_WRITE, &ctx->lock); qwReleaseTaskCtx(mgmt, ctx); @@ -634,8 +622,8 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_LOCK(QW_WRITE, &ctx->lock); if (queryEnd || code || 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) { - // Note: if necessary, fetch need to put cquery to queue again - atomic_store_8(&ctx->phase, 0); + // Note: query is not running anymore + QW_SET_PHASE(ctx, 0); QW_UNLOCK(QW_WRITE, &ctx->lock); break; } @@ -722,7 +710,7 @@ _return: int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; - bool rsped = false; + bool dropped = false; SQWTaskCtx *ctx = NULL; bool locked = false; @@ -740,14 +728,14 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { if (QW_QUERY_RUNNING(ctx)) { QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP); - } else if (ctx->phase > 0) { + } else if (QW_GET_PHASE(ctx) > 0) { QW_ERR_JRET(qwDropTask(QW_FPARAMS())); - rsped = true; + dropped = true; } else { // task not started } - if (!rsped) { + if (!dropped) { ctx->ctrlConnInfo = qwMsg->connInfo; QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); From 7b2dc59860a1a083bb928fba9dd267839a80ab29 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 15 Jul 2022 10:42:47 +0800 Subject: [PATCH 021/124] test: add test case for precison --- tests/system-test/2-query/db.py | 63 +++++++++++++++++++++++++++++++++ tests/system-test/fulltest.sh | 1 + 2 files changed, 64 insertions(+) create mode 100644 tests/system-test/2-query/db.py diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py new file mode 100644 index 0000000000..a4d603bada --- /dev/null +++ b/tests/system-test/2-query/db.py @@ -0,0 +1,63 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +import random + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, + "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, + "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "udfDebugFlag": 143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def case1(self): + tdSql.execute("create database if not exists dbms precision 'ms'") + tdSql.execute("create database if not exists dbus precision 'us'") + tdSql.execute("create database if not exists dbns precision 'ns'") + + tdSql.execute("create table dbms.ntb (ts timestamp, c1 int, c2 bigint)") + tdSql.execute("create table dbus.ntb (ts timestamp, c1 int, c2 bigint)") + tdSql.execute("create table dbns.ntb (ts timestamp, c1 int, c2 bigint)") + + tdSql.execute("insert into dbms.ntb values ('2022-01-01 08:00:00.001', 1, 2)") + tdSql.execute("insert into dbms.ntb values ('2022-01-01 08:00:00.002', 3, 4)") + + tdSql.execute("insert into dbus.ntb values ('2022-01-01 08:00:00.000001', 1, 2)") + tdSql.execute("insert into dbus.ntb values ('2022-01-01 08:00:00.000002', 3, 4)") + + tdSql.execute("insert into dbns.ntb values ('2022-01-01 08:00:00.000000001', 1, 2)") + tdSql.execute("insert into dbns.ntb values ('2022-01-01 08:00:00.000000002', 3, 4)") + + tdSql.query("select count(c1) from dbms.ntb interval(1a)") + tdSql.checkRows(2) + + tdSql.query("select count(c1) from dbus.ntb interval(1u)") + tdSql.checkRows(2) + + tdSql.query("select count(c1) from dbns.ntb interval(1b)") + tdSql.checkRows(2) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========start case1 run ...............") + + self.case1() + + tdLog.printNoPrefix("==========end case1 run ...............") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 8bafe3c966..7a72dc75b2 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -29,6 +29,7 @@ python3 ./test.py -f 1-insert/block_wise.py python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py +python3 ./test.py -f 2-query/db.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py From c67fa0c1ce604ab4d6b738a5282e2c93f897ac45 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 15 Jul 2022 14:17:00 +0800 Subject: [PATCH 022/124] enh: add taosd query exit processing --- include/libs/transport/trpc.h | 2 +- source/client/src/clientHb.c | 4 +++- source/libs/qworker/inc/qwInt.h | 4 ++-- source/libs/qworker/src/qwUtil.c | 26 +++++++++++++++++--------- source/libs/qworker/src/qworker.c | 4 ++-- source/libs/scheduler/src/schRemote.c | 1 + source/libs/transport/src/trans.c | 2 +- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 7441b38321..2ae1f7b854 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -135,7 +135,7 @@ void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg int32_t rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); void rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); -int64_t rpcAllocHandle(); +void* rpcAllocHandle(); #ifdef __cplusplus } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index d7c2c26d23..f16937c836 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -171,6 +171,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; + tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, pTscObj->pAppInfo->totalDnodes); if (pRsp->query->killRid) { tscDebug("request rid %" PRIx64 " need to be killed now", pRsp->query->killRid); @@ -286,7 +287,8 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { taosMemoryFreeClear(param); if (code != 0) { - (*pInst)->onlineDnodes = 0; + (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1); + tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes, (*pInst)->totalDnodes); } if (rspNum) { diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 6c9871425b..2cc2993601 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -362,7 +362,7 @@ int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx); int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx); int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx); void qwReleaseTaskCtx(SQWorker *mgmt, void *ctx); -int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx); +int32_t qwKillTaskHandle(SQWTaskCtx *ctx); int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status); int32_t qwDropTask(QW_FPARAMS_DEF); void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx); @@ -372,7 +372,7 @@ int32_t qwUpdateTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type); int64_t qwGetTimeInQueue(SQWorker *mgmt, EQueueType type); void qwClearExpiredSch(SQWorker *mgmt, SArray* pExpiredSch); int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch); -void qwFreeTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx *ctx); +void qwFreeTaskCtx(SQWTaskCtx *ctx); void qwDbgDumpMgmtInfo(SQWorker *mgmt); int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore); diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 9c49cbcb1f..1d5d9a989a 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -270,7 +270,7 @@ int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { return qwAddTask void qwReleaseTaskCtx(SQWorker *mgmt, void *ctx) { taosHashRelease(mgmt->ctxHash, ctx); } -void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { +void qwFreeTaskHandle(qTaskInfo_t *taskHandle) { // Note: free/kill may in RC qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { @@ -278,7 +278,7 @@ void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { } } -int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { +int32_t qwKillTaskHandle(SQWTaskCtx *ctx) { int32_t code = 0; // Note: free/kill may in RC qTaskInfo_t taskHandle = atomic_load_ptr(&ctx->taskHandle); @@ -290,7 +290,7 @@ int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { QW_RET(code); } -void qwFreeTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { +void qwFreeTaskCtx(SQWTaskCtx *ctx) { if (ctx->ctrlConnInfo.handle) { tmsgReleaseHandle(&ctx->ctrlConnInfo, TAOS_CONN_SERVER); } @@ -300,7 +300,7 @@ void qwFreeTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { // NO need to release dataConnInfo - qwFreeTaskHandle(QW_FPARAMS(), &ctx->taskHandle); + qwFreeTaskHandle(&ctx->taskHandle); if (ctx->sinkHandle) { dsDestroyDataSinker(ctx->sinkHandle); @@ -336,7 +336,7 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } - qwFreeTaskCtx(QW_FPARAMS(), &octx); + qwFreeTaskCtx(&octx); QW_TASK_DLOG_E("task ctx dropped"); @@ -463,13 +463,21 @@ void qwDestroyImpl(void *pMgmt) { mgmt->hbTimer = NULL; taosTmrCleanUp(mgmt->timer); - // TODO STOP ALL QUERY - - // TODO FREE ALL + uint64_t qId, tId; + int32_t eId; + void *pIter = taosHashIterate(mgmt->ctxHash, NULL); + while (pIter) { + SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; + void *key = taosHashGetKey(pIter, NULL); + QW_GET_QTID(key, qId, tId, eId); + qwFreeTaskCtx(ctx); + QW_TASK_DLOG("task ctx freed"); + pIter = taosHashIterate(mgmt->ctxHash, pIter); + } taosHashCleanup(mgmt->ctxHash); - void *pIter = taosHashIterate(mgmt->schHash, NULL); + pIter = taosHashIterate(mgmt->schHash, NULL); while (pIter) { SQWSchStatus *sch = (SQWSchStatus *)pIter; qwDestroySchStatus(sch); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index fb8ce09615..b15a3bd293 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -726,7 +726,7 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } if (QW_QUERY_RUNNING(ctx)) { - QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); + QW_ERR_JRET(qwKillTaskHandle(ctx)); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP); } else if (QW_GET_PHASE(ctx) > 0) { QW_ERR_JRET(qwDropTask(QW_FPARAMS())); @@ -940,7 +940,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { _return: - qwFreeTaskCtx(QW_FPARAMS(), &ctx); + qwFreeTaskCtx(&ctx); QW_RET(TSDB_CODE_SUCCESS); } diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index d195c22c37..41d9f46a87 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1010,6 +1010,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); persistHandle = true; + SCH_SET_TASK_HANDLE(pTask, rpcAllocHandle()); break; } case TDMT_SCH_FETCH: diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 8a16b20a6f..725f3b32cf 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -170,7 +170,7 @@ void rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { transSetDefaultAddr(thandle, ip, fqdn); } -int64_t rpcAllocHandle() { return transAllocHandle(); } +void* rpcAllocHandle() { return (void*)transAllocHandle(); } int32_t rpcInit() { transInit(); From 5b6c7bc38df52ccda414e977f31361576c50e82b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 14:26:18 +0800 Subject: [PATCH 023/124] refactor(sync): add leader transfer callback --- source/dnode/mnode/impl/inc/mndInt.h | 25 +++++------ source/dnode/mnode/impl/src/mndMain.c | 9 ++++ source/dnode/mnode/impl/src/mndSync.c | 7 ++++ source/dnode/vnode/src/vnd/vnodeSync.c | 5 +++ .../script/tsim/sync/mnodeLeaderTransfer.sim | 41 +++++++++++++++++++ 5 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 tests/script/tsim/sync/mnodeLeaderTransfer.sim diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 9f168e2c83..8ad1ac56e4 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -50,8 +50,8 @@ extern "C" { // clang-format on #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) -#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) -#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) +#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) typedef int32_t (*MndMsgFp)(SRpcMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); @@ -61,7 +61,7 @@ typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef struct SQWorker SQHandle; typedef struct { - const char * name; + const char *name; MndInitFp initFp; MndCleanupFp cleanupFp; } SMnodeStep; @@ -70,7 +70,7 @@ typedef struct { int64_t showId; ShowRetrieveFp retrieveFps[TSDB_MGMT_TABLE_MAX]; ShowFreeIterFp freeIterFps[TSDB_MGMT_TABLE_MAX]; - SCacheObj * cache; + SCacheObj *cache; } SShowMgmt; typedef struct { @@ -84,12 +84,13 @@ typedef struct { } STelemMgmt; typedef struct { - tsem_t syncSem; + tsem_t syncSem; int64_t sync; bool standby; SReplica replica; int32_t errCode; int32_t transId; + int8_t leaderTransferFinish; } SSyncMgmt; typedef struct { @@ -107,14 +108,14 @@ typedef struct SMnode { bool stopped; bool restored; bool deploy; - char * path; + char *path; int64_t checkTime; - SSdb * pSdb; - SArray * pSteps; - SQHandle * pQuery; - SHashObj * infosMeta; - SHashObj * perfsMeta; - SWal * pWal; + SSdb *pSdb; + SArray *pSteps; + SQHandle *pQuery; + SHashObj *infosMeta; + SHashObj *perfsMeta; + SWal *pWal; SShowMgmt showMgmt; SProfileMgmt profileMgmt; STelemMgmt telemMgmt; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 29e68ce4e8..66ec4b28f7 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -367,8 +367,17 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { } void mndPreClose(SMnode *pMnode) { + mDebug("vgId:1, mnode pre-close"); if (pMnode != NULL) { + atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 0); syncLeaderTransfer(pMnode->syncMgmt.sync); + + mDebug("vgId:1, wait for mnode leader transfer"); + // wait for leader transfer finish + while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) { + taosMsleep(10); + } + mDebug("vgId:1, mnode leader transfer finish"); } } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 3c3864b620..1414bea880 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -153,6 +153,12 @@ int32_t mndSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int return sdbDoWrite(pMnode->pSdb, pWriter, pBuf, len); } +void mndLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + SMnode *pMnode = pFsm->data; + atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 1); + mDebug("vgId:1, mnd leader transfer finish"); +} + SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pMnode; @@ -160,6 +166,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { pFsm->FpPreCommitCb = NULL; pFsm->FpRollBackCb = NULL; pFsm->FpRestoreFinishCb = mndRestoreFinish; + pFsm->FpLeaderTransferCb = mndLeaderTransfer; pFsm->FpReConfigCb = mndReConfig; pFsm->FpGetSnapshot = mndSyncGetSnapshot; pFsm->FpGetSnapshotInfo = mndSyncGetSnapshotInfo; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 87148a8450..38b9df52b8 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -536,6 +536,10 @@ static int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void * #endif } +static void vnodeLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { + SVnode *pVnode = pFsm->data; +} + static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pVnode; @@ -544,6 +548,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpRollBackCb = vnodeSyncRollBackMsg; pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshot; pFsm->FpRestoreFinishCb = NULL; + pFsm->FpLeaderTransferCb = vnodeLeaderTransfer; pFsm->FpReConfigCb = vnodeSyncReconfig; pFsm->FpSnapshotStartRead = vnodeSnapshotStartRead; pFsm->FpSnapshotStopRead = vnodeSnapshotStopRead; diff --git a/tests/script/tsim/sync/mnodeLeaderTransfer.sim b/tests/script/tsim/sync/mnodeLeaderTransfer.sim new file mode 100644 index 0000000000..6c67f639f6 --- /dev/null +++ b/tests/script/tsim/sync/mnodeLeaderTransfer.sim @@ -0,0 +1,41 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sql connect + +print =============== show dnodes +sql show mnodes; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data02 != leader then + return -1 +endi + +print =============== create dnodes +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sleep 3000 + + +print =============== create mnode 2 +sql create mnode on dnode 2 +sql create mnode on dnode 3 +sleep 3000 + +print =============== create user +sql create user user1 PASS 'user1' +sql show users +if $rows != 2 then + return -1 +endi + From 95023fd56ea2132537db921038aebcd7179365e9 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 14:34:58 +0800 Subject: [PATCH 024/124] refactor(sync): add leader transfer callback --- source/dnode/mnode/impl/src/mndMain.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 66ec4b28f7..40ac4514c5 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -367,17 +367,17 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { } void mndPreClose(SMnode *pMnode) { - mDebug("vgId:1, mnode pre-close"); if (pMnode != NULL) { atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 0); syncLeaderTransfer(pMnode->syncMgmt.sync); - mDebug("vgId:1, wait for mnode leader transfer"); + mDebug("vgId:1, mnode start leader transfer"); // wait for leader transfer finish while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) { taosMsleep(10); + mDebug("vgId:1, mnode waiting for leader transfer"); } - mDebug("vgId:1, mnode leader transfer finish"); + mDebug("vgId:1, mnode finish leader transfer"); } } From c5675bc8212bf2c4942fae29f69eb67f2bc5d18c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 15 Jul 2022 14:40:42 +0800 Subject: [PATCH 025/124] fix: fix msg disorder issue --- source/libs/qworker/src/qwUtil.c | 2 +- source/libs/transport/src/transCli.c | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 1d5d9a989a..b56cb29628 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -472,7 +472,7 @@ void qwDestroyImpl(void *pMgmt) { QW_GET_QTID(key, qId, tId, eId); qwFreeTaskCtx(ctx); - QW_TASK_DLOG("task ctx freed"); + QW_TASK_DLOG_E("task ctx freed"); pIter = taosHashIterate(mgmt->ctxHash, pIter); } taosHashCleanup(mgmt->ctxHash); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 3805787ce2..851824e439 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1206,7 +1206,13 @@ SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle, bool* validHandle) { if (idx < 0) return NULL; return ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; } - return transGetWorkThrdFromHandle(handle, validHandle); + SCliThrd* pThrd = transGetWorkThrdFromHandle(handle, validHandle); + if (*validHandle == true && pThrd == NULL) { + int idx = cliRBChoseIdx(trans); + if (idx < 0) return NULL; + pThrd = ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; + } + return pThrd; } void transReleaseCliHandle(void* handle) { int idx = -1; From 35e8ad28117a9b1630381586540a3a201c54c0fb Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Jul 2022 14:51:20 +0800 Subject: [PATCH 026/124] feat: add time unsynced check --- include/common/tmsg.h | 2 + include/util/taoserror.h | 1 + source/client/inc/clientInt.h | 2 +- source/client/src/clientEnv.c | 4 +- source/client/src/clientHb.c | 82 ++++++++++++------------ source/client/src/clientImpl.c | 2 +- source/client/src/clientMsgHandler.c | 12 ++++ source/common/src/tmsg.c | 4 ++ source/dnode/mnode/impl/src/mndProfile.c | 4 +- source/libs/function/src/udfd.c | 9 +++ source/libs/index/src/indexCache.c | 5 +- source/libs/transport/src/transCli.c | 12 ++-- source/libs/transport/src/transSvr.c | 4 +- source/util/src/terror.c | 1 + 14 files changed, 89 insertions(+), 55 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0ab65e7978..cb8e1171e4 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -525,6 +525,7 @@ typedef struct { int8_t superUser; int8_t connType; SEpSet epSet; + int32_t svrTimestamp; char sVer[TSDB_VERSION_LEN]; char sDetailVer[128]; } SConnectRsp; @@ -2224,6 +2225,7 @@ typedef struct { typedef struct { int64_t reqId; int64_t rspId; + int32_t svrTimestamp; SArray* rsps; // SArray } SClientHbBatchRsp; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ce434612c3..24837aa35f 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -73,6 +73,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MSG_DECODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0031) #define TSDB_CODE_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0032) #define TSDB_CODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0033) +#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0034) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0040) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0041) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 700a4d9daf..3999eaccb2 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -285,7 +285,7 @@ static FORCE_INLINE SReqResultInfo* tscGetCurResInfo(TAOS_RES* res) { extern SAppInfo appInfo; extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; -extern void* tscQhandle; +extern int32_t timestampDeltaLimit; __async_send_cb_fn_t getMsgRspHandle(int32_t msgType); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 207ac01a2c..67e6b69233 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -35,6 +35,8 @@ SAppInfo appInfo; int32_t clientReqRefPool = -1; int32_t clientConnRefPool = -1; +int32_t timestampDeltaLimit = 900; // s + static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; volatile int32_t tscInitRes = 0; @@ -181,7 +183,7 @@ void destroyTscObj(void *pObj) { destroyAllRequests(pTscObj->pRequests); taosHashCleanup(pTscObj->pRequests); - + schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); tscDebug("connObj 0x%" PRIx64 " p:%p destroyed, remain inst totalConn:%" PRId64, pTscObj->id, pTscObj, pTscObj->pAppInfo->numOfConns); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 2a9d113108..b12806ee0f 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -70,7 +70,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (NULL == vgInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - + vgInfo->vgVersion = rsp->vgVersion; vgInfo->hashMethod = rsp->hashMethod; vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); @@ -156,18 +156,18 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { STscObj *pTscObj = (STscObj *)acquireTscObj(pRsp->connKey.tscRid); if (NULL == pTscObj) { tscDebug("tscObj rid %" PRIx64 " not exist", pRsp->connKey.tscRid); - } else { + } else { if (pRsp->query->totalDnodes > 1 && !isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pRsp->query->epSet)) { - SEpSet* pOrig = &pTscObj->pAppInfo->mgmtEp.epSet; - SEp* pOrigEp = &pOrig->eps[pOrig->inUse]; - SEp* pNewEp = &pRsp->query->epSet.eps[pRsp->query->epSet.inUse]; - tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in hb", - pOrig->inUse, pOrig->numOfEps, pOrigEp->fqdn, pOrigEp->port, - pRsp->query->epSet.inUse, pRsp->query->epSet.numOfEps, pNewEp->fqdn, pNewEp->port); - + SEpSet *pOrig = &pTscObj->pAppInfo->mgmtEp.epSet; + SEp *pOrigEp = &pOrig->eps[pOrig->inUse]; + SEp *pNewEp = &pRsp->query->epSet.eps[pRsp->query->epSet.inUse]; + tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in hb", pOrig->inUse, pOrig->numOfEps, + pOrigEp->fqdn, pOrigEp->port, pRsp->query->epSet.inUse, pRsp->query->epSet.numOfEps, pNewEp->fqdn, + pNewEp->port); + updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pRsp->query->epSet); } - + pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; @@ -263,13 +263,20 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { } static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { - static int32_t emptyRspNum = 0; + static int32_t emptyRspNum = 0; char *key = (char *)param; SClientHbBatchRsp pRsp = {0}; if (TSDB_CODE_SUCCESS == code) { tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); } - + + int32_t now = taosGetTimestampSec(); + int32_t delta = abs(now - pRsp.svrTimestamp); + if (delta > timestampDeltaLimit) { + code = TSDB_CODE_TIME_UNSYNCED; + tscError("time diff: %ds is too big", delta); + } + int32_t rspNum = taosArrayGetSize(pRsp.rsps); taosThreadMutexLock(&appInfo.mutex); @@ -373,7 +380,7 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { releaseTscObj(connKey->tscRid); return TSDB_CODE_QRY_OUT_OF_MEMORY; } - + hbBasic->connId = pTscObj->connId; int32_t numOfQueries = pTscObj->pRequests ? taosHashGetSize(pTscObj->pRequests) : 0; @@ -392,7 +399,6 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - int32_t code = hbBuildQueryDesc(hbBasic, pTscObj); if (code) { releaseTscObj(connKey->tscRid); @@ -436,13 +442,12 @@ int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S if (NULL == req->info) { req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); } - + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); return TSDB_CODE_SUCCESS; } - int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SDbVgVersion *dbs = NULL; uint32_t dbNum = 0; @@ -483,8 +488,8 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SSTableVersion *stbs = NULL; - uint32_t stbNum = 0; - int32_t code = 0; + uint32_t stbNum = 0; + int32_t code = 0; code = catalogGetExpiredSTables(pCatalog, &stbs, &stbNum); if (TSDB_CODE_SUCCESS != code) { @@ -521,20 +526,19 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC } int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { - SAppHbReq* pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); + SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL != pApp) { memcpy(&req->app, pApp, sizeof(*pApp)); } else { memset(&req->app.summary, 0, sizeof(req->app.summary)); req->app.pid = taosGetPId(); req->app.appId = clientHbMgr.appId; - taosGetAppName(req->app.name, NULL); + taosGetAppName(req->app.name, NULL); } return TSDB_CODE_SUCCESS; } - int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { int64_t *clusterId = (int64_t *)param; struct SCatalog *pCatalog = NULL; @@ -602,7 +606,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { continue; } - //hbClearClientHbReq(pOneReq); + // hbClearClientHbReq(pOneReq); pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); } @@ -615,11 +619,9 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { return pBatchReq; } -void hbThreadFuncUnexpectedStopped(void) { - atomic_store_8(&clientHbMgr.threadStop, 2); -} +void hbThreadFuncUnexpectedStopped(void) { atomic_store_8(&clientHbMgr.threadStop, 2); } -void hbMergeSummary(SAppClusterSummary* dst, SAppClusterSummary* src) { +void hbMergeSummary(SAppClusterSummary *dst, SAppClusterSummary *src) { dst->numOfInsertsReq += src->numOfInsertsReq; dst->numOfInsertRows += src->numOfInsertRows; dst->insertElapsedTime += src->insertElapsedTime; @@ -633,7 +635,7 @@ void hbMergeSummary(SAppClusterSummary* dst, SAppClusterSummary* src) { int32_t hbGatherAppInfo(void) { SAppHbReq req = {0}; - int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); + int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); if (sz > 0) { req.pid = taosGetPId(); req.appId = clientHbMgr.appId; @@ -641,11 +643,11 @@ int32_t hbGatherAppInfo(void) { } taosHashClear(clientHbMgr.appSummary); - + for (int32_t i = 0; i < sz; ++i) { SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); - uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId; - SAppHbReq* pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); + uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId; + SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL == pApp) { memcpy(&req.summary, &pAppHbMgr->pAppInstInfo->summary, sizeof(req.summary)); req.startTime = pAppHbMgr->startTime; @@ -654,7 +656,7 @@ int32_t hbGatherAppInfo(void) { if (pAppHbMgr->startTime < pApp->startTime) { pApp->startTime = pAppHbMgr->startTime; } - + hbMergeSummary(&pApp->summary, &pAppHbMgr->pAppInstInfo->summary); } } @@ -662,7 +664,6 @@ int32_t hbGatherAppInfo(void) { return TSDB_CODE_SUCCESS; } - static void *hbThreadFunc(void *param) { setThreadName("hb"); #ifdef WINDOWS @@ -681,7 +682,7 @@ static void *hbThreadFunc(void *param) { if (sz > 0) { hbGatherAppInfo(); } - + for (int i = 0; i < sz; i++) { SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); @@ -698,7 +699,7 @@ static void *hbThreadFunc(void *param) { if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); - //hbClearReqInfo(pAppHbMgr); + // hbClearReqInfo(pAppHbMgr); break; } @@ -708,7 +709,7 @@ static void *hbThreadFunc(void *param) { if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); - //hbClearReqInfo(pAppHbMgr); + // hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); break; } @@ -725,7 +726,7 @@ static void *hbThreadFunc(void *param) { SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); tFreeClientHbBatchReq(pReq); - //hbClearReqInfo(pAppHbMgr); + // hbClearReqInfo(pAppHbMgr); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } @@ -759,7 +760,7 @@ static void hbStopThread() { return; } - taosThreadJoin(clientHbMgr.thread, NULL); + taosThreadJoin(clientHbMgr.thread, NULL); tscDebug("hb thread stopped"); } @@ -808,7 +809,7 @@ void hbFreeAppHbMgr(SAppHbMgr *pTarget) { } taosHashCleanup(pTarget->activeInfo); pTarget->activeInfo = NULL; - + taosMemoryFree(pTarget->key); taosMemoryFree(pTarget); } @@ -843,7 +844,7 @@ int hbMgrInit() { clientHbMgr.appId = tGenIdPI64(); tscDebug("app %" PRIx64 " initialized", clientHbMgr.appId); - + clientHbMgr.appSummary = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *)); taosThreadMutexInit(&clientHbMgr.lock, NULL); @@ -881,7 +882,7 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, int64_t clust SClientHbReq hbReq = {0}; hbReq.connKey = connKey; hbReq.clusterId = clusterId; - //hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + // hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq)); @@ -920,4 +921,3 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } - diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 32c11983c2..940e6e8ebc 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1478,7 +1478,7 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU tsem_wait(&pParam->sem); } - if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) { + if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) { return NULL; } else { if (setupOneRowPtr) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 94bd5dd787..331ca86d78 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -52,6 +52,18 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { SConnectRsp connectRsp = {0}; tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp); + + int32_t now = taosGetTimestampSec(); + int32_t delta = abs(now - connectRsp.svrTimestamp); + if (delta > timestampDeltaLimit) { + code = TSDB_CODE_TIME_UNSYNCED; + tscError("time diff: %" PRId64 "ms is too big", delta); + taosMemoryFree(pMsg->pData); + setErrno(pRequest, code); + tsem_post(&pRequest->body.rspSem); + return code; + } + /*assert(connectRsp.epSet.numOfEps > 0);*/ if (connectRsp.epSet.numOfEps == 0) { taosMemoryFree(pMsg->pData); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index eca8d49b6a..058ef2e81d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -453,6 +453,7 @@ int32_t tSerializeSClientHbBatchRsp(void *buf, int32_t bufLen, const SClientHbBa if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI64(&encoder, pBatchRsp->reqId) < 0) return -1; if (tEncodeI64(&encoder, pBatchRsp->rspId) < 0) return -1; + if (tEncodeI32(&encoder, pBatchRsp->svrTimestamp) < 0) return -1; int32_t rspNum = taosArrayGetSize(pBatchRsp->rsps); if (tEncodeI32(&encoder, rspNum) < 0) return -1; @@ -474,6 +475,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI64(&decoder, &pBatchRsp->reqId) < 0) return -1; if (tDecodeI64(&decoder, &pBatchRsp->rspId) < 0) return -1; + if (tDecodeI32(&decoder, &pBatchRsp->svrTimestamp) < 0) return -1; int32_t rspNum = 0; if (tDecodeI32(&decoder, &rspNum) < 0) return -1; @@ -3613,6 +3615,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tEncodeI8(&encoder, pRsp->superUser) < 0) return -1; if (tEncodeI8(&encoder, pRsp->connType) < 0) return -1; if (tEncodeSEpSet(&encoder, &pRsp->epSet) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->svrTimestamp) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sVer) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sDetailVer) < 0) return -1; tEndEncode(&encoder); @@ -3634,6 +3637,7 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tDecodeI8(&decoder, &pRsp->superUser) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->connType) < 0) return -1; if (tDecodeSEpSet(&decoder, &pRsp->epSet) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->svrTimestamp) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->sVer) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->sDetailVer) < 0) return -1; tEndDecode(&decoder); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 38368b4ece..81bd887ad9 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -15,10 +15,10 @@ #define _DEFAULT_SOURCE #include "mndProfile.h" -#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndPrivilege.h" #include "mndQnode.h" #include "mndShow.h" #include "mndStb.h" @@ -274,6 +274,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.connId = pConn->id; connectRsp.connType = connReq.connType; connectRsp.dnodeNum = mndGetDnodeSize(pMnode); + connectRsp.svrTimestamp = taosGetTimestampSec(); strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, @@ -623,6 +624,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { } SClientHbBatchRsp batchRsp = {0}; + batchRsp.svrTimestamp = taosGetTimestampMs(); batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); int32_t sz = taosArrayGetSize(batchReq.reqs); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index a412b589a9..e33c785917 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -382,6 +382,15 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { if (msgInfo->rpcType == UDFD_RPC_MNODE_CONNECT) { SConnectRsp connectRsp = {0}; tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp); + + int32_t now = taosGetTimestampSec(); + int32_t delta = abs(now - connectRsp.svrTimestamp); + if (delta > 900) { + msgInfo->code = TSDB_CODE_TIME_UNSYNCED; + goto _return; + } + + if (connectRsp.epSet.numOfEps == 0) { msgInfo->code = TSDB_CODE_MND_APP_ERROR; goto _return; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 05ce418037..bfc3047719 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -516,13 +516,14 @@ static void idxCacheMakeRoomForWrite(IndexCache* cache) { idxCacheRef(cache); cache->imm = cache->mem; cache->mem = idxInternalCacheCreate(cache->type); + cache->mem->pCache = cache; cache->occupiedMem = 0; if (quit == false) { atomic_store_32(&cache->merging, 1); } - // sched to merge - // unref cache in bgwork + // 1. sched to merge + // 2. unref cache in bgwork idxCacheSchedToMerge(cache, quit); } } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 40227f02cc..ff984e464b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1041,7 +1041,7 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { STraceId* trace = &pMsg->msg.info.traceId; char tbuf[256] = {0}; EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - tGTrace("%s retry on next node, use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), tbuf, + tGDebug("%s retry on next node, use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), tbuf, pCtx->retryCnt + 1, pCtx->retryLimit); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); @@ -1133,11 +1133,11 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (hasEpSet) { char tbuf[256] = {0}; EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - tGTrace("%s conn %p extract epset from msg", CONN_GET_INST_LABEL(pConn), pConn); + tGDebug("%s conn %p extract epset from msg", CONN_GET_INST_LABEL(pConn), pConn); } if (pCtx->pSem != NULL) { - tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); + tGDebug("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (pCtx->pRsp == NULL) { tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn); } else { @@ -1146,7 +1146,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tsem_post(pCtx->pSem); pCtx->pRsp = NULL; } else { - tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); + tGDebug("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (retry == false && hasEpSet == true) { pTransInst->cfp(pTransInst->parent, pResp, &pCtx->epSet); } else { @@ -1256,7 +1256,7 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra cliMsg->refId = (int64_t)shandle; STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); ASSERT(transAsyncSend(pThrd->asyncPool, &(cliMsg->q)) == 0); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); @@ -1296,7 +1296,7 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM cliMsg->refId = (int64_t)shandle; STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); transAsyncSend(pThrd->asyncPool, &(cliMsg->q)); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 68e12a1963..9a511adf9b 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1020,7 +1020,7 @@ void transRefSrvHandle(void* handle) { return; } int ref = T_REF_INC((SSvrConn*)handle); - tDebug("conn %p ref count:%d", handle, ref); + tTrace("conn %p ref count:%d", handle, ref); } void transUnrefSrvHandle(void* handle) { @@ -1028,7 +1028,7 @@ void transUnrefSrvHandle(void* handle) { return; } int ref = T_REF_DEC((SSvrConn*)handle); - tDebug("conn %p ref count:%d", handle, ref); + tTrace("conn %p ref count:%d", handle, ref); if (ref == 0) { destroyConn((SSvrConn*)handle, true); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ef6697b3b5..8600c61587 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -78,6 +78,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIMESTAMP, "Invalid timestamp for TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR, "Msg decode error") TAOS_DEFINE_ERROR(TSDB_CODE_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_NOT_FOUND, "Not found") +TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Unsynced time") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") From ec2a069185934e9db7441092fd0c0a7b87a6333f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 15 Jul 2022 06:53:56 +0000 Subject: [PATCH 027/124] more vnode snapshot writer --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 280 ++++++++++++--------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 6 +- 2 files changed, 164 insertions(+), 122 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 51514f1257..d9a67ce487 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -377,10 +377,9 @@ struct STsdbSnapWriter { SBlockIdx* pBlockIdx; SMapData mBlock; // SMapData int32_t iBlock; - SBlock* pBlock; - SBlock block; - SBlockData bDataR; + SBlockData* pBlockData; int32_t iRow; + SBlockData bDataR; SDataFWriter* pDataFWriter; SBlockIdx* pBlockIdxW; // NULL when no committing table @@ -550,127 +549,171 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) { TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); TSDBKEY keyLast = tBlockDataLastKey(pBlockData); - if (pWriter->pDataFReader == NULL) { - // no old data + // TABLE ==================================== - // end last table write if need - if (pWriter->pBlockIdxW) { - int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id); - - if (c < 0) { - // end last table data write - code = tsdbSnapWriteTableDataEnd(pWriter); - if (code) goto _err; - - // reset - pWriter->pBlockIdxW = NULL; - } else if (c > 0) { - ASSERT(0); - } - } - - // start a new table data if need - if (pWriter->pBlockIdxW == NULL) { - pWriter->pBlockIdxW = &pWriter->blockIdxW; - pWriter->pBlockIdxW->suid = id.suid; - pWriter->pBlockIdxW->uid = id.uid; - - tBlockReset(&pWriter->blockW); - - tBlockDataReset(&pWriter->bDataW); - - tMapDataReset(&pWriter->mBlockW); - } - - // set block schema - code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData); - if (code) goto _err; - - // add rows - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - TSDBROW row = tsdbRowFromBlockData(pBlockData, iRow); - - code = tBlockDataAppendRow(&pWriter->bDataW, &row, NULL); + // end last table write if should + if (pWriter->pBlockIdxW) { + int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id); + if (c < 0) { + // end + code = tsdbSnapWriteTableDataEnd(pWriter); if (code) goto _err; - if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) { - pWriter->blockW.last = 0; - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - &pWriter->blockW, pWriter->cmprAlg); - if (code) goto _err; - - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); - if (code) goto _err; - - // reset - tBlockReset(&pWriter->blockW); - tBlockDataClearData(&pWriter->bDataW); - } - } - } else { - // has old data - - // TABLE ================================================== - // end last table data if id not same (todo) - if (pWriter->pBlockIdxW) { - int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id); - if (c < 0) { - } else if (c > 0) { - ASSERT(0); - } - } - - // start new table data if need (todo) - if (pWriter->pBlockIdxW == NULL) { - // commit table data ahead - code = tsdbSnapWriteTableDataAhead(pWriter, id); - if (code) goto _err; - - // reader - if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { - pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); - int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id); - if (c) { - pWriter->pBlockIdx = NULL; - } - } else { - pWriter->pBlockIdx = NULL; - } - - if (pWriter->pBlockIdx) { - code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlockW, NULL); - if (code) goto _err; - - pWriter->iBlock = 0; - } - - // writer - pWriter->pBlockIdxW = &pWriter->blockIdxW; - pWriter->pBlockIdxW->suid = id.suid; - pWriter->pBlockIdxW->uid = id.uid; - - tBlockReset(&pWriter->blockW); - tBlockDataReset(&pWriter->bDataW); - tMapDataReset(&pWriter->mBlockW); - } - - // BLOCK ================================================== - // write block ahead - while (true) { - if (pWriter->iBlock >= pWriter->mBlock.nItem) break; - - SBlock block; - tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); - - if (tsdbKeyCmprFn(&block.maxKey, &keyFirst) >= 0) break; - - pWriter->iBlock++; - - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); - if (code) goto _err; + // reset + pWriter->pBlockIdxW = NULL; + } else if (c > 0) { + ASSERT(0); } } + // start new table data write if need + if (pWriter->pBlockIdxW == NULL) { + // write table data ahead + code = tsdbSnapWriteTableDataAhead(pWriter, id); + if (code) goto _err; + + // reader + if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { + ASSERT(pWriter->pDataFReader); + + pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlock); + int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id); + if (c) { + ASSERT(c > 0); + pWriter->pBlockIdx = NULL; + } else { + pWriter->iBlockIdx++; + } + } else { + pWriter->pBlockIdx = NULL; + } + + if (pWriter->pBlockIdx) { + code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock, NULL); + if (code) goto _err; + } else { + tMapDataReset(&pWriter->mBlock); + } + pWriter->iBlock = 0; + pWriter->pBlockData = NULL; + pWriter->iRow = 0; + + // writer + pWriter->pBlockIdxW = &pWriter->blockIdxW; + pWriter->pBlockIdxW->suid = id.suid; + pWriter->pBlockIdxW->uid = id.uid; + + tBlockReset(&pWriter->blockW); + tBlockDataReset(&pWriter->bDataW); + tMapDataReset(&pWriter->mBlockW); + } + + ASSERT(pWriter->pBlockIdxW && pWriter->pBlockIdxW->suid == id.suid && pWriter->pBlockIdxW->uid == id.uid); + ASSERT(pWriter->pBlockIdx == NULL || (pWriter->pBlockIdx->suid == id.suid && pWriter->pBlockIdx->uid == id.uid)); + + // BLOCK ==================================== + int32_t iRow = 0; + TSDBROW* pRow = &tsdbRowFromBlockData(pBlockData, iRow); + while (true) { + if (pRow == NULL) break; + + if (pWriter->pBlockData) { + ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); + + int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow)); + + ASSERT(c); + + if (c < 0) { + code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + if (code) goto _err; + + iRow++; + if (iRow < pWriter->pBlockData->nRow) { + pRow = &tsdbRowFromBlockData(pBlockData, iRow); + } else { + pRow = NULL; + } + } else if (c > 0) { + code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL); + if (code) goto _err; + + pWriter->iRow++; + if (pWriter->iRow >= pWriter->pBlockData->nRow) { + pWriter->pBlockData = NULL; + } + } + } else { + SBlock block; + + if (pWriter->iBlock < pWriter->mBlock.nItem) { + tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); + int32_t c; + + c = tsdbKeyCmprFn(&block.maxKey, &TSDBROW_KEY(pRow)); + + ASSERT(c); + + if (c < 0) { + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + if (code) goto _err; + + pWriter->iBlock++; + continue; + } + + c = tsdbKeyCmprFn(&block.minKey, &TSDBROW_KEY(pRow)); + + ASSERT(c); + + if (c > 0) { + code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + if (code) goto _err; + + iRow++; + if (iRow < pWriter->pBlockData->nRow) { + pRow = &tsdbRowFromBlockData(pBlockData, iRow); + } else { + pRow = NULL; + } + + goto _check_write; + } + + pWriter->pBlockData = &pWriter->bDataR; + code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); + if (code) goto _err; + + pWriter->iRow = 0; + } + + code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + if (code) goto _err; + + iRow++; + if (iRow < pWriter->pBlockData->nRow) { + pRow = &tsdbRowFromBlockData(pBlockData, iRow); + } else { + pRow = NULL; + } + } + + _check_write: + if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue; + + _write_block: + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdx, &pWriter->blockW, + pWriter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + if (code) goto _err; + + tBlockReset(&pWriter->blockW); + tBlockDataClearData(&pWriter->bDataW); + } + +_exit: return code; _err: @@ -720,7 +763,6 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 pWriter->pBlockIdx = NULL; tMapDataReset(&pWriter->mBlock); pWriter->iBlock = 0; - pWriter->pBlock = NULL; tBlockDataReset(&pWriter->bDataR); pWriter->iRow = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 6320a48e77..04368ddfb7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -36,15 +36,15 @@ int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(u // alloc code = tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem); - if (code) goto _err; + if (code) goto _exit; code = tRealloc(&pMapData->pData, pMapData->nData); - if (code) goto _err; + if (code) goto _exit; // put pMapData->aOffset[nItem] = offset; tPutItemFn(pMapData->pData + offset, pItem); -_err: +_exit: return code; } From 681993884f12727f777d59879ce8abb078e40c79 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 14:55:33 +0800 Subject: [PATCH 028/124] refactor(sync): add leader transfer callback --- source/dnode/mnode/impl/src/mndMain.c | 16 +++++++++------- source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/src/syncAppendEntries.c | 7 +++++++ source/libs/sync/src/syncMain.c | 5 ++++- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 40ac4514c5..041fc2a2d1 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -371,13 +371,15 @@ void mndPreClose(SMnode *pMnode) { atomic_store_8(&(pMnode->syncMgmt.leaderTransferFinish), 0); syncLeaderTransfer(pMnode->syncMgmt.sync); - mDebug("vgId:1, mnode start leader transfer"); - // wait for leader transfer finish - while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) { - taosMsleep(10); - mDebug("vgId:1, mnode waiting for leader transfer"); - } - mDebug("vgId:1, mnode finish leader transfer"); + /* + mDebug("vgId:1, mnode start leader transfer"); + // wait for leader transfer finish + while (!atomic_load_8(&(pMnode->syncMgmt.leaderTransferFinish))) { + taosMsleep(10); + mDebug("vgId:1, mnode waiting for leader transfer"); + } + mDebug("vgId:1, mnode finish leader transfer"); + */ } } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index e361c8021c..1f26033f63 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -253,6 +253,7 @@ bool syncNodeCheckNewConfig(SSyncNode* pSyncNode, const SSyncCfg* pNewCfg); int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode); int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader); +int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry); // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 8409e1c711..dca80f6826 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -477,6 +477,13 @@ static int32_t syncNodeDoMakeLogSame(SSyncNode* ths, SyncIndex FromIndex) { static int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); + + // leader transfer + if (pEntry->originalRpcType == TDMT_SYNC_LEADER_TRANSFER) { + int32_t code = syncDoLeaderTransfer(ths, &rpcMsg, pEntry); + ASSERT(code == 0); + } + if (ths->pFsm != NULL) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta = {0}; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 80b2032ab7..387400c25e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2598,7 +2598,7 @@ const char* syncStr(ESyncState state) { } } -static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { +int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); syncNodeEventLog(ths, "do leader transfer"); @@ -2811,11 +2811,14 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ASSERT(code == 0); } +#if 0 + // execute in pre-commit // leader transfer if (pEntry->originalRpcType == TDMT_SYNC_LEADER_TRANSFER) { code = syncDoLeaderTransfer(ths, &rpcMsg, pEntry); ASSERT(code == 0); } +#endif // restore finish // if only snapshot, a noop entry will be append, so syncLogLastIndex is always ok From 8006ee1d8e0d03f73f6ca22112d838b97018add4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Jul 2022 14:56:23 +0800 Subject: [PATCH 029/124] feat: add time unsynced check --- source/client/src/clientMsgHandler.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 331ca86d78..520a566e2b 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -57,7 +57,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { int32_t delta = abs(now - connectRsp.svrTimestamp); if (delta > timestampDeltaLimit) { code = TSDB_CODE_TIME_UNSYNCED; - tscError("time diff: %" PRId64 "ms is too big", delta); + tscError("time diff:%ds is too big", delta); taosMemoryFree(pMsg->pData); setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 81bd887ad9..a1da68606b 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -624,7 +624,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { } SClientHbBatchRsp batchRsp = {0}; - batchRsp.svrTimestamp = taosGetTimestampMs(); + batchRsp.svrTimestamp = taosGetTimestampSec(); batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); int32_t sz = taosArrayGetSize(batchReq.reqs); From f60f821c13a1508cf8a8182e11f31b071895e601 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 15:04:22 +0800 Subject: [PATCH 030/124] fix(query): fix partition by bug for last_row query. --- source/libs/executor/src/cachescanoperator.c | 27 +++++++++++++++----- source/libs/planner/src/planOptimizer.c | 3 ++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 78a7e58ee1..395b42aa86 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -139,18 +139,29 @@ SSDataBlock* doScanLastrow(SOperatorInfo* pOperator) { colDataAppend(pDst, 0, p, isNull); } + pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); + pInfo->pRes->info.rows = 1; + if (pInfo->pseudoExprSup.numOfExprs > 0) { SExprSupp* pSup = &pInfo->pseudoExprSup; - addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return NULL; + } } - pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); - int64_t* groupId = taosHashGet(pTableList->map, &pInfo->pRes->info.uid, sizeof(int64_t)); - pInfo->pRes->info.groupId = *groupId; + if (pTableList->map != NULL) { + int64_t* groupId = taosHashGet(pTableList->map, &pInfo->pRes->info.uid, sizeof(int64_t)); + pInfo->pRes->info.groupId = *groupId; + } else { + ASSERT(taosArrayGetSize(pTableList->pTableList) == 1); + STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, 0); + pInfo->pRes->info.groupId = pKeyInfo->groupId; + } pInfo->indexOfBufferedRes += 1; - pInfo->pRes->info.rows = 1; return pInfo->pRes; } else { doSetOperatorCompleted(pOperator); @@ -182,8 +193,12 @@ SSDataBlock* doScanLastrow(SOperatorInfo* pOperator) { STableKeyInfo* pKeyInfo = taosArrayGet(pGroupTableList, 0); pInfo->pRes->info.groupId = pKeyInfo->groupId; - addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return NULL; + } } tsdbLastrowReaderClose(pInfo->pLastrowReader); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 97858f83fb..2d05f6a16a 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1993,7 +1993,8 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { SNode* pFunc = NULL; FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { if (FUNCTION_TYPE_LAST_ROW != ((SFunctionNode*)pFunc)->funcType && - FUNCTION_TYPE_SELECT_VALUE != ((SFunctionNode*)pFunc)->funcType) { + FUNCTION_TYPE_SELECT_VALUE != ((SFunctionNode*)pFunc)->funcType && + FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pFunc)->funcType) { return false; } } From c8758058745550dc26f7907e748b266abea7cde6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Jul 2022 15:49:30 +0800 Subject: [PATCH 031/124] refactor code --- source/libs/transport/src/transCli.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 851824e439..3956add423 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1,4 +1,5 @@ /** Copyright (c) 2019 TAOS Data, Inc. + * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 @@ -809,7 +810,7 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) { conn = exh->handle; if (conn == NULL) { conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); - *ignore = (conn && 0 == specifyConnRef(conn, true, refId)) ? false : true; + if (conn != NULL) specifyConnRef(conn, true, refId); } transReleaseExHandle(transGetRefMgt(), refId); } @@ -849,14 +850,20 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { bool ignore = false; SCliConn* conn = cliGetConn(pMsg, pThrd, &ignore); if (ignore == true) { + tError("ignore msg"); return; } + if (conn != NULL) { transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); cliSend(conn); } else { conn = cliCreateConn(pThrd); + + int64_t refId = (int64_t)pMsg->msg.info.handle; + if (refId != 0) specifyConnRef(conn, true, refId); + transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); From 9e49a0012d603aebd9c3123a1fb947ad5eebeb7e Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 15:52:32 +0800 Subject: [PATCH 032/124] feat(query): add percentile/apercentile scalar version TD-17344 --- include/libs/scalar/scalar.h | 2 ++ source/libs/function/src/builtins.c | 2 ++ source/libs/scalar/src/sclfunc.c | 31 +++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index d5e9a2e625..df3e27225f 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -104,6 +104,8 @@ int32_t maxScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t avgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stddevScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6fb808a2e8..e274146d25 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2034,6 +2034,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getPercentileFuncEnv, .initFunc = percentileFunctionSetup, .processFunc = percentileFunction, + .sprocessFunc = percentileScalarFunction, .finalizeFunc = percentileFinalize, .invertFunc = NULL, .combineFunc = NULL, @@ -2046,6 +2047,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getApercentileFuncEnv, .initFunc = apercentileFunctionSetup, .processFunc = apercentileFunction, + .sprocessFunc = apercentileScalarFunction, .finalizeFunc = apercentileFinalize, .invertFunc = NULL, .combineFunc = apercentileCombine, diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index ba35356a9c..8808d17454 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2307,3 +2307,34 @@ int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPa pOutput->numOfRows = 1; return TSDB_CODE_SUCCESS; } + +int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int32_t type = GET_PARAM_TYPE(pInput); + + double val; + bool hasNull = false; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + hasNull = true; + break; + } + char *in = pInputData->pData; + GET_TYPED_DATA(val, double, type, in); + } + + if (hasNull) { + colDataAppendNULL(pOutputData, 0); + } else { + colDataAppend(pOutputData, 0, (char *)&val, false); + } + + pOutput->numOfRows = 1; + return TSDB_CODE_SUCCESS; +} + +int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return percentileScalarFunction(pInput, inputNum, pOutput); +} From 02af3581f33095b62ff2946dcb8be1c037434a64 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 15 Jul 2022 16:01:42 +0800 Subject: [PATCH 033/124] enh: add debug info --- source/libs/qworker/inc/qwInt.h | 2 ++ source/libs/qworker/inc/qwMsg.h | 1 + source/libs/qworker/src/qwDbg.c | 42 +++++++++++++++++++++++---- source/libs/qworker/src/qwMsg.c | 14 +++++++++ source/libs/scheduler/src/schRemote.c | 2 +- 5 files changed, 55 insertions(+), 6 deletions(-) diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index d3347f4d1f..3c6c02ede6 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -75,6 +75,8 @@ typedef struct SQWDebug { bool lockEnable; bool statusEnable; bool dumpEnable; + bool sleepSimulate; + bool deadSimulate; bool tmp; } SQWDebug; diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index acb7004a51..16fe6b21c2 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -40,6 +40,7 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComple int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SQWTaskCtx *ctx); int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SExplainExecInfo *execInfo, int32_t num); +int32_t qwBuildAndSendErrorRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code); void qwFreeFetchRsp(void *msg); int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *rsp, int32_t code); diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 893f31acd1..cd54c5e5f9 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -9,7 +9,7 @@ #include "tmsg.h" #include "tname.h" -SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = false, .tmp = false}; +SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true, .tmp = false}; int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { if (!gQWDebug.statusEnable) { @@ -175,29 +175,61 @@ int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx) { return TSDB_CODE_SUCCESS; } +void qwDbgSimulateSleep() { + if (!gQWDebug.sleepSimulate) { + return; + } + + taosSsleep(taosRand() % 10); +} + +void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t msgType) { + if (!gQWDebug.deadSimulate) { + return; + } + + //FETCH OR QUERY USE DIFFERENT CONNINFO + qwBuildAndSendErrorRsp(msgType + 1, ctx->dataConnInfo, TSDB_CODE_RPC_BROKEN_LINK); + + qwDropTask(QW_FPARAMS()); +} + + int32_t qwDbgEnableDebug(char *option) { if (0 == strcasecmp(option, "lock")) { gQWDebug.lockEnable = true; - qDebug("qw lock debug enabled"); + qError("qw lock debug enabled"); return TSDB_CODE_SUCCESS; } if (0 == strcasecmp(option, "status")) { gQWDebug.statusEnable = true; - qDebug("qw status debug enabled"); + qError("qw status debug enabled"); return TSDB_CODE_SUCCESS; } if (0 == strcasecmp(option, "dump")) { gQWDebug.dumpEnable = true; - qDebug("qw dump debug enabled"); + qError("qw dump debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "sleep")) { + gQWDebug.sleepSimulate = true; + qError("qw sleep debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "dead")) { + gQWDebug.sleepSimulate = true; + qError("qw dead debug enabled"); return TSDB_CODE_SUCCESS; } if (0 == strcasecmp(option, "tmp")) { gQWDebug.tmp = true; - qDebug("qw tmp debug enabled"); + qError("qw tmp debug enabled"); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 147ce76fc2..8bbd03f735 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -43,6 +43,20 @@ void qwFreeFetchRsp(void *msg) { } } +int32_t qwBuildAndSendErrorRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code) { + SRpcMsg rpcRsp = { + .msgType = rspType, + .pCont = NULL, + .contLen = 0, + .code = code, + .info = *pConn, + }; + + tmsgSendRsp(&rpcRsp); + + return TSDB_CODE_SUCCESS; +} + int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SQWTaskCtx *ctx) { STbVerInfo* tbInfo = ctx ? &ctx->tbInfo : NULL; int64_t affectedRows = ctx ? ctx->affectedRows : 0; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 41d9f46a87..ec07ee85fd 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1010,7 +1010,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); persistHandle = true; - SCH_SET_TASK_HANDLE(pTask, rpcAllocHandle()); + //SCH_SET_TASK_HANDLE(pTask, rpcAllocHandle()); break; } case TDMT_SCH_FETCH: From a12f6a3e3d6d7c575b665b4af38e9bf22772e578 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 16:03:16 +0800 Subject: [PATCH 034/124] refactor: let transfer leader msg to be sent even when the dnode exiting --- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 84491a82b0..49207225a5 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -148,9 +148,9 @@ static int32_t mmStart(SMnodeMgmt *pMgmt) { static void mmStop(SMnodeMgmt *pMgmt) { dDebug("mnode-mgmt start to stop"); + mndPreClose(pMgmt->pMnode); taosThreadRwlockWrlock(&pMgmt->lock); pMgmt->stopped = 1; - mndPreClose(pMgmt->pMnode); taosThreadRwlockUnlock(&pMgmt->lock); mndStop(pMgmt->pMnode); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 9052fb20ca..7efa46c514 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -221,11 +221,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) { static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { SDnode *pDnode = dmInstance(); - if (pDnode->status != DND_STAT_RUNNING) { + if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG) { rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; terrno = TSDB_CODE_NODE_OFFLINE; - dError("failed to send rpc msg since %s, handle:%p", terrstr(), pMsg->info.handle); + dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); return -1; } else { rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL); From ed34f490ca883533e76f5f02d488a8f4dcdd5122 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 15 Jul 2022 16:06:41 +0800 Subject: [PATCH 035/124] enh: async commit api for rsma --- source/dnode/vnode/src/inc/sma.h | 35 +++++-- source/dnode/vnode/src/inc/vnodeInt.h | 9 +- source/dnode/vnode/src/sma/smaCommit.c | 126 ++++++++++++++++++++--- source/dnode/vnode/src/sma/smaEnv.c | 4 +- source/dnode/vnode/src/sma/smaRollup.c | 52 ++++------ source/dnode/vnode/src/vnd/vnodeCommit.c | 4 +- 6 files changed, 168 insertions(+), 62 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index e767d94ebd..cfeb4ebeb2 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -67,7 +67,9 @@ struct SRSmaStat { int64_t submitVer; int64_t refId; // shared by fetch tasks int8_t triggerStat; // shared by fetch tasks + int8_t commitStat; // 0 not in committing, 1 in committing SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; + SHashObj *iRsmaInfoHash; // key: stbUid, value: SRSmaInfo; immutable rsmaInfoHash }; struct SSmaStat { @@ -78,12 +80,29 @@ struct SSmaStat { T_REF_DECLARE() }; -#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) -#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) -#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) -#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) -#define RSMA_REF_ID(r) ((r)->refId) -#define RSMA_SUBMIT_VER(r) ((r)->submitVer) +#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) +#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) +#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) +#define RSMA_IMU_INFO_HASH(r) ((r)->iRsmaInfoHash) +#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) +#define RSMA_COMMIT_STAT(r) (&(r)->commitStat) +#define RSMA_REF_ID(r) ((r)->refId) +#define RSMA_SUBMIT_VER(r) ((r)->submitVer) + +struct SRSmaInfoItem { + void *taskInfo; // qTaskInfo_t + int64_t refId; + tmr_h tmrId; + int32_t maxDelay; + int8_t level; + int8_t triggerStat; +}; + +struct SRSmaInfo { + STSchema *pTSchema; + int64_t suid; + SRSmaInfoItem items[TSDB_RETENTION_L2]; +}; enum { TASK_TRIGGER_STAT_INIT = 0, @@ -94,6 +113,8 @@ enum { TASK_TRIGGER_STAT_DROPPED = 5, }; +#define RSMA_TASK_INFO_HASH_SLOT 8 + void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -183,7 +204,7 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { atomic_or_fetch_8(&pTStat->state, TSDB_SMA_STAT_DROPPED); } } - +void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 394c314819..3b32ca9c60 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -164,10 +164,11 @@ void smaCleanUp(); int32_t smaOpen(SVnode* pVnode); int32_t smaClose(SSma* pSma); int32_t smaBegin(SSma* pSma); -int32_t smaPreCommit(SSma* pSma); -int32_t smaCommit(SSma* pSma); -int32_t smaPostCommit(SSma* pSma); -int32_t smaAsyncCommit(SSma* pSma); +int32_t smaSyncPreCommit(SSma* pSma); +int32_t smaSyncCommit(SSma* pSma); +int32_t smaSyncPostCommit(SSma* pSma); +int32_t smaAsyncPreCommit(SSma* pSma); +int32_t smaAsyncPostCommit(SSma* pSma); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 889c4adf24..4729187e8c 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -18,7 +18,8 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma); static int32_t tdProcessRSmaCommitImpl(SSma *pSma); static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma); -static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); /** * @brief Only applicable to Rollup SMA @@ -26,7 +27,7 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma); * @param pSma * @return int32_t */ -int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); } +int32_t smaSyncPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); } /** * @brief Only applicable to Rollup SMA @@ -34,7 +35,7 @@ int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); } * @param pSma * @return int32_t */ -int32_t smaCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); } +int32_t smaSyncCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); } /** * @brief Only applicable to Rollup SMA @@ -42,7 +43,7 @@ int32_t smaCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); } * @param pSma * @return int32_t */ -int32_t smaPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); } +int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); } /** * @brief Only applicable to Rollup SMA @@ -50,7 +51,15 @@ int32_t smaPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); } * @param pSma * @return int32_t */ -int32_t smaAsyncCommit(SSma *pSma) { return tdProcessRSmaAsyncCommitImpl(pSma); } +int32_t smaAsyncPreCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); } + +/** + * @brief Only applicable to Rollup SMA + * + * @param pSma + * @return int32_t + */ +int32_t smaAsyncPostCommit(SSma *pSma) { return tdProcessRSmaAsyncPostCommitImpl(pSma); } /** * @brief set rsma trigger stat active @@ -71,18 +80,17 @@ int32_t smaBegin(SSma *pSma) { atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED, TASK_TRIGGER_STAT_ACTIVE); switch (rsmaTriggerStat) { case TASK_TRIGGER_STAT_PAUSED: { - smaDebug("vgId:%d rsma trigger stat from paused to active", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma trigger stat from paused to active", SMA_VID(pSma)); break; } case TASK_TRIGGER_STAT_INIT: { atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE); - smaDebug("vgId:%d rsma trigger stat from init to active", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma trigger stat from init to active", SMA_VID(pSma)); break; } default: { atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE); - smaWarn("vgId:%d rsma trigger stat %" PRIi8 " is unexpected", SMA_VID(pSma), rsmaTriggerStat); - ASSERT(0); + smaError("vgId:%d, rsma trigger stat %" PRIi8 " is unexpected", SMA_VID(pSma), rsmaTriggerStat); break; } } @@ -90,7 +98,7 @@ int32_t smaBegin(SSma *pSma) { } /** - * @brief pre-commit for rollup sma. + * @brief pre-commit for rollup sma(sync commit). * 1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED. * 2) wait all triggered fetch tasks finished * 3) perform persist task for qTaskInfo @@ -107,8 +115,7 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) { SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); - - // step 1: set persistence task paused + // step 1: set rsma stat paused atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); // step 2: wait all triggered fetch tasks finished @@ -236,12 +243,101 @@ static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) { /** * @brief Rsma async commit implementation - * 1) Wait all running fetch task finish to fetch and put submitMsg into level 2/3 wQueue(blocking level 1 write) + * 1) set rsma stat TASK_TRIGGER_STAT_PAUSED + * 2) Wait all running fetch task finish to fetch and put submitMsg into level 2/3 wQueue(blocking level 1 write) + * 3) * * @param pSma * @return int32_t */ -static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { - // TODO +static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + if (!pSmaEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + + // step 1: set rsma stat paused + atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); + + // step 2: wait all triggered fetch tasks finished + int32_t nLoops = 0; + while (1) { + if (T_REF_VAL_GET(pStat) == 0) { + smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma)); + break; + } else { + smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma)); + } + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + + // step 3: swap rsmaInfoHash and iRsmaInfoHash + ASSERT(!RSMA_IMU_INFO_HASH(pRSmaStat)); + ASSERT(RSMA_INFO_HASH(pRSmaStat)); + + RSMA_IMU_INFO_HASH(pRSmaStat) = RSMA_INFO_HASH(pRSmaStat); + RSMA_INFO_HASH(pRSmaStat) = + taosHashInit(RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + if (!RSMA_INFO_HASH(pRSmaStat)) { + smaError("vgId:%d, rsma async commit failed since %s", SMA_VID(pSma), terrstr()); + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Migrate rsmaInfo from iRsmaInfo to rsmaInfo if rsmaInfoHash not empty. + * + * @param pSma + * @return int32_t + */ +static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + if (!pSmaEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + + // step 1: merge rsmaInfoHash and iRsmaInfoHash + + if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) { + // TODO: + } + + void *pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), NULL); + while (pIter) { + tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); + + if (!taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t))) { + taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); + smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), *pSuid); + } else { + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + SRSmaInfoItem *pItem = &pRSmaInfo->items[i]; + if (pItem->taskInfo) { + tdFreeQTaskInfo(pItem->taskInfo, SMA_VID(pSma), i + 1); + } + } + smaDebug("vgId:%d, rsma async post commit, free rsma info since already COW for table:%" PRIi64, SMA_VID(pSma), + *pSuid); + } + + pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter); + } + + taosHashCleanup(RSMA_IMU_INFO_HASH(pRSmaStat)); + RSMA_IMU_INFO_HASH(pRSmaStat) = NULL; + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 4a7d4db874..77377cf089 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -17,7 +17,6 @@ typedef struct SSmaStat SSmaStat; -#define RSMA_TASK_INFO_HASH_SLOT 8 #define SMA_MGMT_REF_NUM 10240 extern SSmaMgmt smaMgmt; @@ -311,7 +310,6 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (taosRemoveRef(smaMgmt.rsetId, RSMA_REF_ID(pRSmaStat)) < 0) { smaError("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " failed since %s", SMA_VID(pRSmaStat->pSma), RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId, terrstr()); - ASSERT(0); } else { smaDebug("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " succeed", SMA_VID(pRSmaStat->pSma), RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId); @@ -361,7 +359,7 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { } break; default: - smaError("vgId:%d undefined smaType:%", SMA_VID(pSma), smaType); + smaError("vgId:%d, undefined smaType:%", SMA_VID(pSma), smaType); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index a6d56acbfb..300a0ec4be 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -48,20 +48,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables); static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed); static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed); -struct SRSmaInfoItem { - void *taskInfo; // qTaskInfo_t - int64_t refId; - tmr_h tmrId; - int32_t maxDelay; - int8_t level; - int8_t triggerStat; -}; -struct SRSmaInfo { - STSchema *pTSchema; - int64_t suid; - SRSmaInfoItem items[TSDB_RETENTION_L2]; -}; static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) { // adapt accordingly if definition of SRSmaInfo update @@ -102,7 +89,7 @@ static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskInfoIter *pIter) { taosMemoryFreeClear(pIter->pBuf); } -static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { +void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { @@ -123,7 +110,7 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { i + 1); taosTmrStopA(&pItem->tmrId); } - tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pSma), i + 1); + tdFreeQTaskInfo(&pItem->taskInfo, SMA_VID(pSma), i + 1); } else { smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma), pInfo->suid, i + 1); @@ -257,22 +244,21 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx) { - SRetention *pRetention = SMA_RETENTION(pSma); - STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); + if ((param->qmsgLen > 0) && param->qmsg[idx]) { + SRetention *pRetention = SMA_RETENTION(pSma); + STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); + SReadHandle handle = { + .meta = pSma->pVnode->pMeta, + .vnode = pSma->pVnode, + .initTqReader = 1, + }; - SReadHandle handle = { - .meta = pSma->pVnode->pMeta, - .vnode = pSma->pVnode, - .initTqReader = 1, - }; - - if (param->qmsg[idx]) { SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); pItem->refId = RSMA_REF_ID(pStat); pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); if (!pItem->taskInfo) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; - goto _err; + return TSDB_CODE_FAILED; } pItem->triggerStat = TASK_TRIGGER_STAT_INACTIVE; if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { @@ -286,13 +272,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; } pItem->level = idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2; - smaInfo("vgId:%d table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 + smaInfo("vgId:%d, table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%" PRIi32, SMA_VID(pSma), pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); } return TSDB_CODE_SUCCESS; -_err: - return TSDB_CODE_FAILED; } /** @@ -562,7 +546,9 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche SSDataBlock *output = NULL; uint64_t ts; if (qExecTask(pItem->taskInfo, &output, &ts) < 0) { - ASSERT(false); + smaError("vgId:%d, qExecTask for rsma table %" PRIi64 "l evel %" PRIi8 " failed since %s", SMA_VID(pSma), suid, + pItem->level, terrstr()); + goto _err; } if (!output) { break; @@ -572,7 +558,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche pResult = taosArrayInit(1, sizeof(SSDataBlock)); if (!pResult) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + goto _err; } } @@ -891,7 +877,7 @@ int32_t tdProcessRSmaRestoreImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; _err: - smaError("vgId:%d failed to restore rsma task since %s", SMA_VID(pSma), terrstr()); + smaError("vgId:%d, failed to restore rsma task since %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; } @@ -1217,6 +1203,10 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64 " refId:%d", SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId); + if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) { + taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay > 5000 ? 5000 : pItem->maxDelay, pItem, smaMgmt.tmrHandle, + &pItem->tmrId); + } return; } default: diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index ebbb691e28..4e55c05a47 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -233,7 +233,7 @@ int vnodeCommit(SVnode *pVnode) { walBeginSnapshot(pVnode->pWal, pVnode->state.applied); // preCommit - smaPreCommit(pVnode->pSma); + smaSyncPreCommit(pVnode->pSma); // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { @@ -276,7 +276,7 @@ int vnodeCommit(SVnode *pVnode) { pVnode->state.committed = info.state.committed; // postCommit - smaPostCommit(pVnode->pSma); + smaSyncPostCommit(pVnode->pSma); // apply the commit (TODO) walEndSnapshot(pVnode->pWal); From 59c7348c948a3abf931ad1bc59dc58d5a3203444 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 16:07:45 +0800 Subject: [PATCH 036/124] refactor(sync): add leader transfer callback --- source/libs/sync/src/syncMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 387400c25e..b6c01f2923 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2601,6 +2601,10 @@ const char* syncStr(ESyncState state) { int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); + if (ths->state != TAOS_SYNC_STATE_FOLLOWER) { + syncNodeEventLog(ths, "I am not follower, can not do leader transfer"); + return 0; + } syncNodeEventLog(ths, "do leader transfer"); bool sameId = syncUtilSameId(&(pSyncLeaderTransfer->newLeaderId), &(ths->myRaftId)); From f32d2bde1f6a11ff7be2dab98f6e255fa0621f32 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Jul 2022 16:07:54 +0800 Subject: [PATCH 037/124] feat: add time unsynced check --- source/client/src/clientHb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 0e21273305..6969e03e7c 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -293,7 +293,7 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { taosMemoryFreeClear(param); if (code != 0) { - (*pInst)->onlineDnodes = 0; + (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1); } if (rspNum) { From ca7c1151490e7e73979edb04ba63c2de5ee02522 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 16:13:08 +0800 Subject: [PATCH 038/124] refactor: get retry epset from sync module only when mnode is not initialized --- source/dnode/mnode/impl/src/mndMnode.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index d7eaa72202..13655ac21f 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -218,7 +218,6 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { } void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { -#if 0 SSdb *pSdb = pMnode->pSdb; int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); void *pIter = NULL; @@ -238,9 +237,10 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port); sdbRelease(pSdb, pObj); } -#else - syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); -#endif + + if (pEpSet->numOfEps == 0) { + syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); + } } static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { From aec4c2313a8bcb4a08f42b02f89525d0a253fdfb Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 15 Jul 2022 16:17:00 +0800 Subject: [PATCH 039/124] fix: multi-rows function validity check --- include/libs/function/functionMgt.h | 3 +- source/libs/function/inc/functionMgtInt.h | 7 ++-- source/libs/function/src/builtins.c | 22 +++++----- source/libs/function/src/functionMgt.c | 6 +-- source/libs/parser/src/parTranslater.c | 49 ++++++++++------------- source/libs/parser/test/parSelectTest.cpp | 12 +++--- 6 files changed, 45 insertions(+), 54 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 299d808017..d55adcb63c 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -190,14 +190,13 @@ bool fmIsUserDefinedFunc(int32_t funcId); bool fmIsDistExecFunc(int32_t funcId); bool fmIsForbidFillFunc(int32_t funcId); bool fmIsForbidStreamFunc(int32_t funcId); -bool fmIsForbidWindowFunc(int32_t funcId); -bool fmIsForbidGroupByFunc(int32_t funcId); bool fmIsIntervalInterpoFunc(int32_t funcId); bool fmIsInterpFunc(int32_t funcId); bool fmIsLastRowFunc(int32_t funcId); bool fmIsSystemInfoFunc(int32_t funcId); bool fmIsImplicitTsFunc(int32_t funcId); bool fmIsClientPseudoColumnFunc(int32_t funcId); +bool fmIsMultiRowsFunc(int32_t funcId); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc); diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index da5dd0433d..0b4fdefec4 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -44,10 +44,9 @@ extern "C" { #define FUNC_MGT_FORBID_FILL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(15) #define FUNC_MGT_INTERVAL_INTERPO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(16) #define FUNC_MGT_FORBID_STREAM_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(17) -#define FUNC_MGT_FORBID_WINDOW_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(18) -#define FUNC_MGT_FORBID_GROUP_BY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(19) -#define FUNC_MGT_SYSTEM_INFO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(20) -#define FUNC_MGT_CLIENT_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(21) +#define FUNC_MGT_SYSTEM_INFO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(18) +#define FUNC_MGT_CLIENT_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(19) +#define FUNC_MGT_MULTI_ROWS_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(20) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 747b1c8d9f..e690a85d1c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2078,7 +2078,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "top", .type = FUNCTION_TYPE_TOP, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateTopBot, .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, @@ -2092,7 +2092,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "bottom", .type = FUNCTION_TYPE_BOTTOM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateTopBot, .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, @@ -2319,7 +2319,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "histogram", .type = FUNCTION_TYPE_HISTOGRAM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_FILL_FUNC, .translateFunc = translateHistogram, .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, @@ -2395,7 +2395,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "diff", .type = FUNCTION_TYPE_DIFF, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateDiff, .getEnvFunc = getDiffFuncEnv, .initFunc = diffFunctionSetup, @@ -2405,7 +2405,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "statecount", .type = FUNCTION_TYPE_STATE_COUNT, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateStateCount, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -2415,7 +2415,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "stateduration", .type = FUNCTION_TYPE_STATE_DURATION, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateStateDuration, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -2425,7 +2425,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "csum", .type = FUNCTION_TYPE_CSUM, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateCsum, .getEnvFunc = getCsumFuncEnv, .initFunc = functionSetup, @@ -2435,7 +2435,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "mavg", .type = FUNCTION_TYPE_MAVG, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateMavg, .getEnvFunc = getMavgFuncEnv, .initFunc = mavgFunctionSetup, @@ -2445,7 +2445,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "sample", .type = FUNCTION_TYPE_SAMPLE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateSample, .getEnvFunc = getSampleFuncEnv, .initFunc = sampleFunctionSetup, @@ -2456,7 +2456,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "tail", .type = FUNCTION_TYPE_TAIL, .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | - FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateTail, .getEnvFunc = getTailFuncEnv, .initFunc = tailFunctionSetup, @@ -2467,7 +2467,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "unique", .type = FUNCTION_TYPE_UNIQUE, .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateUnique, .getEnvFunc = getUniqueFuncEnv, .initFunc = uniqueFunctionSetup, diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index f73a61fa60..ff74c880e3 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -175,16 +175,14 @@ bool fmIsIntervalInterpoFunc(int32_t funcId) { return isSpecificClassifyFunc(fun bool fmIsForbidStreamFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_STREAM_FUNC); } -bool fmIsForbidWindowFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_WINDOW_FUNC); } - -bool fmIsForbidGroupByFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_GROUP_BY_FUNC); } - bool fmIsSystemInfoFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SYSTEM_INFO_FUNC); } bool fmIsImplicitTsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_IMPLICIT_TS_FUNC); } bool fmIsClientPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_CLIENT_PC_FUNC); } +bool fmIsMultiRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_MULTI_ROWS_FUNC); } + bool fmIsInterpFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c88e38ddff..0d2959f21d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1096,6 +1096,24 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } + if (NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow || NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "%s function is not supported in window query or group query", pFunc->functionName); + } + if (hasInvalidFuncNesting(pFunc->pParameterList)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateMultiRowsFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsMultiRowsFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (!isSelectStmt(pCxt->pCurrStmt) || SQL_CLAUSE_SELECT != pCxt->currClause || + ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); + } if (hasInvalidFuncNesting(pFunc->pParameterList)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } @@ -1131,16 +1149,6 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctio return TSDB_CODE_SUCCESS; } -static int32_t translateForbidWindowFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (!fmIsForbidWindowFunc(pFunc->funcId)) { - return TSDB_CODE_SUCCESS; - } - if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC, pFunc->functionName); - } - return TSDB_CODE_SUCCESS; -} - static int32_t translateForbidStreamFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsForbidStreamFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1151,16 +1159,6 @@ static int32_t translateForbidStreamFunc(STranslateContext* pCxt, SFunctionNode* return TSDB_CODE_SUCCESS; } -static int32_t translateForbidGroupByFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (!fmIsForbidGroupByFunc(pFunc->funcId)) { - return TSDB_CODE_SUCCESS; - } - if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC, pFunc->functionName); - } - return TSDB_CODE_SUCCESS; -} - static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsRepeatScanFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1191,7 +1189,7 @@ static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFu if (!fmIsMultiResFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } - if (SQL_CLAUSE_SELECT != pCxt->currClause ) { + if (SQL_CLAUSE_SELECT != pCxt->currClause) { SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); if (isStar(pPara) || isTableStar(pPara)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, @@ -1322,21 +1320,18 @@ static int32_t translateNoramlFunction(STranslateContext* pCxt, SFunctionNode* p if (TSDB_CODE_SUCCESS == code) { code = translateWindowPseudoColumnFunc(pCxt, pFunc); } - if (TSDB_CODE_SUCCESS == code) { - code = translateForbidWindowFunc(pCxt, pFunc); - } if (TSDB_CODE_SUCCESS == code) { code = translateForbidStreamFunc(pCxt, pFunc); } - if (TSDB_CODE_SUCCESS == code) { - code = translateForbidGroupByFunc(pCxt, pFunc); - } if (TSDB_CODE_SUCCESS == code) { code = translateRepeatScanFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == code) { code = translateMultiResFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateMultiRowsFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == code) { setFuncClassification(pCxt->pCurrStmt, pFunc); } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index e682059793..b9a760d342 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -152,9 +152,9 @@ TEST_F(ParserSelectTest, IndefiniteRowsFuncSemanticCheck) { run("SELECT DIFF(c1), CSUM(c1) FROM t1", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT CSUM(c3) FROM t1 STATE_WINDOW(c1)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT CSUM(c3) FROM t1 STATE_WINDOW(c1)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT DIFF(c1) FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT DIFF(c1) FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } TEST_F(ParserSelectTest, useDefinedFunc) { @@ -178,9 +178,9 @@ TEST_F(ParserSelectTest, uniqueFunc) { TEST_F(ParserSelectTest, uniqueFuncSemanticCheck) { useDb("root", "test"); - run("SELECT UNIQUE(c1) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT UNIQUE(c1) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT UNIQUE(c1) FROM t1 GROUP BY c2", TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC); + run("SELECT UNIQUE(c1) FROM t1 GROUP BY c2", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } TEST_F(ParserSelectTest, tailFunc) { @@ -194,9 +194,9 @@ TEST_F(ParserSelectTest, tailFunc) { TEST_F(ParserSelectTest, tailFuncSemanticCheck) { useDb("root", "test"); - run("SELECT TAIL(c1, 10) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + run("SELECT TAIL(c1, 10) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); - run("SELECT TAIL(c1, 10) FROM t1 GROUP BY c2", TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC); + run("SELECT TAIL(c1, 10) FROM t1 GROUP BY c2", TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } TEST_F(ParserSelectTest, partitionBy) { From 4d53c09cb69ee4c0aefc4268ee03436dfbf04c9b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 16:24:46 +0800 Subject: [PATCH 040/124] test: valgrind case --- tests/script/tsim/valgrind/checkError3.sim | 38 ++++++++++++---------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 10a8f01fb3..784b83d96b 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c debugflag -v 131 -system sh/exec.sh -n dnode1 -s start -v +system sh/exec.sh -n dnode1 -s start sql connect print =============== step1: create drop show dnodes @@ -56,28 +56,32 @@ print =============== step6: alter insert sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) +print =============== restart +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start -v + print =============== stepa: query data sql select * from c1 -sql select * from stb -sql select * from stb_1 -sql select ts, c1, c2, c3 from c1 -sql select ts, c1, c2, c3 from stb -sql select ts, c1 from stb_2 -sql select ts, c1, t1 from c1 -sql select ts, c1, t1 from stb -sql select ts, c1, t1 from stb_2 +#sql select * from stb +#sql select * from stb_1 +#sql select ts, c1, c2, c3 from c1 +#sql select ts, c1, c2, c3 from stb +#sql select ts, c1 from stb_2 +#sql select ts, c1, t1 from c1 +#sql select ts, c1, t1 from stb +#sql select ts, c1, t1 from stb_2 print =============== stepb: count -sql select count(*) from c1; -sql select count(*) from stb; -sql select count(ts), count(c1), count(c2), count(c3) from c1 -sql select count(ts), count(c1), count(c2), count(c3) from stb +#sql select count(*) from c1; +#sql select count(*) from stb; +#sql select count(ts), count(c1), count(c2), count(c3) from c1 +#sql select count(ts), count(c1), count(c2), count(c3) from stb print =============== stepc: func -sql select first(ts), first(c1), first(c2), first(c3) from c1 -sql select min(c2), min(c3), min(c4) from c1 -sql select max(c2), max(c3), max(c4) from c1 -sql select sum(c2), sum(c3), sum(c4) from c1 +#sql select first(ts), first(c1), first(c2), first(c3) from c1 +#sql select min(c2), min(c3), min(c4) from c1 +#sql select max(c2), max(c3), max(c4) from c1 +#sql select sum(c2), sum(c3), sum(c4) from c1 _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT From 040f20be43458febcf81b20f12a5fcc161942bec Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 16:28:31 +0800 Subject: [PATCH 041/124] test: valgrind case --- tests/script/tsim/valgrind/basic1.sim | 32 +++++++++++----------- tests/script/tsim/valgrind/checkError3.sim | 32 +++++++++++----------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 784b83d96b..86926509bb 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -62,26 +62,26 @@ system sh/exec.sh -n dnode1 -s start -v print =============== stepa: query data sql select * from c1 -#sql select * from stb -#sql select * from stb_1 -#sql select ts, c1, c2, c3 from c1 -#sql select ts, c1, c2, c3 from stb -#sql select ts, c1 from stb_2 -#sql select ts, c1, t1 from c1 -#sql select ts, c1, t1 from stb -#sql select ts, c1, t1 from stb_2 +sql select * from stb +sql select * from stb_1 +sql select ts, c1, c2, c3 from c1 +sql select ts, c1, c2, c3 from stb +sql select ts, c1 from stb_2 +sql select ts, c1, t1 from c1 +sql select ts, c1, t1 from stb +sql select ts, c1, t1 from stb_2 print =============== stepb: count -#sql select count(*) from c1; -#sql select count(*) from stb; -#sql select count(ts), count(c1), count(c2), count(c3) from c1 -#sql select count(ts), count(c1), count(c2), count(c3) from stb +sql select count(*) from c1; +sql select count(*) from stb; +sql select count(ts), count(c1), count(c2), count(c3) from c1 +sql select count(ts), count(c1), count(c2), count(c3) from stb print =============== stepc: func -#sql select first(ts), first(c1), first(c2), first(c3) from c1 -#sql select min(c2), min(c3), min(c4) from c1 -#sql select max(c2), max(c3), max(c4) from c1 -#sql select sum(c2), sum(c3), sum(c4) from c1 +sql select first(ts), first(c1), first(c2), first(c3) from c1 +sql select min(c2), min(c3), min(c4) from c1 +sql select max(c2), max(c3), max(c4) from c1 +sql select sum(c2), sum(c3), sum(c4) from c1 _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 784b83d96b..86926509bb 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -62,26 +62,26 @@ system sh/exec.sh -n dnode1 -s start -v print =============== stepa: query data sql select * from c1 -#sql select * from stb -#sql select * from stb_1 -#sql select ts, c1, c2, c3 from c1 -#sql select ts, c1, c2, c3 from stb -#sql select ts, c1 from stb_2 -#sql select ts, c1, t1 from c1 -#sql select ts, c1, t1 from stb -#sql select ts, c1, t1 from stb_2 +sql select * from stb +sql select * from stb_1 +sql select ts, c1, c2, c3 from c1 +sql select ts, c1, c2, c3 from stb +sql select ts, c1 from stb_2 +sql select ts, c1, t1 from c1 +sql select ts, c1, t1 from stb +sql select ts, c1, t1 from stb_2 print =============== stepb: count -#sql select count(*) from c1; -#sql select count(*) from stb; -#sql select count(ts), count(c1), count(c2), count(c3) from c1 -#sql select count(ts), count(c1), count(c2), count(c3) from stb +sql select count(*) from c1; +sql select count(*) from stb; +sql select count(ts), count(c1), count(c2), count(c3) from c1 +sql select count(ts), count(c1), count(c2), count(c3) from stb print =============== stepc: func -#sql select first(ts), first(c1), first(c2), first(c3) from c1 -#sql select min(c2), min(c3), min(c4) from c1 -#sql select max(c2), max(c3), max(c4) from c1 -#sql select sum(c2), sum(c3), sum(c4) from c1 +sql select first(ts), first(c1), first(c2), first(c3) from c1 +sql select min(c2), min(c3), min(c4) from c1 +sql select max(c2), max(c3), max(c4) from c1 +sql select sum(c2), sum(c3), sum(c4) from c1 _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT From 30e09c4b19dfa2941b3b7db93e21c96627b60872 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 16:41:19 +0800 Subject: [PATCH 042/124] fix(query): fix when select percentile() has no from table clause report error issue --- source/libs/parser/src/parTranslater.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c88e38ddff..8cb0e3b32d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1165,7 +1165,11 @@ static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* p if (!fmIsRepeatScanFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } - if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { + if (isSelectStmt(pCxt->pCurrStmt)) { + //select percentile() is also valid + if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { + return TSDB_CODE_SUCCESS; + } SNode* pTable = ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable; if (QUERY_NODE_REAL_TABLE == nodeType(pTable) && (TSDB_CHILD_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType || From 0377f865db2fd4a58280462091449c897bdb04af Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 16:45:30 +0800 Subject: [PATCH 043/124] fix conflict --- source/libs/scalar/src/scalar.c | 6 +++--- tools/taos-tools | 2 +- tools/taosadapter | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index bbb7e07bad..b3bd989eb4 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -700,9 +700,9 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; SNode* tnode = NULL; - if (!fmIsScalarFunc(node->funcId)) { - return DEAL_RES_CONTINUE; - } + //if (!fmIsScalarFunc(node->funcId)) { + // return DEAL_RES_CONTINUE; + //} FOREACH(tnode, node->pParameterList) { if (!SCL_IS_CONST_NODE(tnode)) { diff --git a/tools/taos-tools b/tools/taos-tools index bd496f76b6..1163c0f60a 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit bd496f76b64931c66da2f8b0f24143a98a881cde +Subproject commit 1163c0f60aa65d6cc58283247c8bf8c56ba43b92 diff --git a/tools/taosadapter b/tools/taosadapter index df8678f070..c3815951fc 160000 --- a/tools/taosadapter +++ b/tools/taosadapter @@ -1 +1 @@ -Subproject commit df8678f070e3f707faf59baebec90065f6e1268b +Subproject commit c3815951fc80617ecd171f3743b8b4a4d0bc712e From 7d5c496e103d611a45cb5b464b5b7d0a66255065 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 16:46:13 +0800 Subject: [PATCH 044/124] fix comment --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8cb0e3b32d..b0a012e20b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1166,7 +1166,7 @@ static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* p return TSDB_CODE_SUCCESS; } if (isSelectStmt(pCxt->pCurrStmt)) { - //select percentile() is also valid + //select percentile() without from clause is also valid if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { return TSDB_CODE_SUCCESS; } From 14ae310fea9f3d57c9d740aa55a14d7d2340d9c5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 16:47:52 +0800 Subject: [PATCH 045/124] fix issue --- source/libs/scalar/src/scalar.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index b3bd989eb4..bbb7e07bad 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -700,9 +700,9 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; SNode* tnode = NULL; - //if (!fmIsScalarFunc(node->funcId)) { - // return DEAL_RES_CONTINUE; - //} + if (!fmIsScalarFunc(node->funcId)) { + return DEAL_RES_CONTINUE; + } FOREACH(tnode, node->pParameterList) { if (!SCL_IS_CONST_NODE(tnode)) { From a5a3b96e5860d7f379f7293269e92cfd44e514b2 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 16:50:51 +0800 Subject: [PATCH 046/124] enable select agg function without from clause --- source/libs/scalar/src/scalar.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index bbb7e07bad..484d95cb5a 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -700,7 +700,7 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; SNode* tnode = NULL; - if (!fmIsScalarFunc(node->funcId)) { + if (!fmIsScalarFunc(node->funcId) && (!ctx->dual)) { return DEAL_RES_CONTINUE; } From 68ecae1df5ae0462bdd5e458004f6a64e6f53651 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 16:57:45 +0800 Subject: [PATCH 047/124] refactor(sync): add case --- .../script/tsim/sync/mnodeLeaderTransfer.sim | 2 +- tests/script/tsim/sync/vnode-insert.sim | 181 ++++++++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 tests/script/tsim/sync/vnode-insert.sim diff --git a/tests/script/tsim/sync/mnodeLeaderTransfer.sim b/tests/script/tsim/sync/mnodeLeaderTransfer.sim index 6c67f639f6..00329ced76 100644 --- a/tests/script/tsim/sync/mnodeLeaderTransfer.sim +++ b/tests/script/tsim/sync/mnodeLeaderTransfer.sim @@ -27,7 +27,7 @@ sql create dnode $hostname port 7300 sleep 3000 -print =============== create mnode 2 +print =============== create mnode 2, 3 sql create mnode on dnode 2 sql create mnode on dnode 3 sleep 3000 diff --git a/tests/script/tsim/sync/vnode-insert.sim b/tests/script/tsim/sync/vnode-insert.sim new file mode 100644 index 0000000000..221e3dd2b2 --- /dev/null +++ b/tests/script/tsim/sync/vnode-insert.sim @@ -0,0 +1,181 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +$replica = 3 +$vgroups = 1 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 100 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][6] $data[2][11] $data[2][12] $data[2][13] $data[2][14] $data[2][15] $data[2][16] $data[2][17] $data[2][18] $data[2][19] +if $rows != 3 then + return -1 +endi +if $data[2][19] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] + +if $rows != $vgroups then + return -1 +endi + +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + endi + endi +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + endi + endi +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + endi + endi +else + goto check_vg_ready +endi + + +vg_ready: +print ====> create stable/child table +sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +sql create table ct1 using stb tags(1000) + + +print ====> step1 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591200000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step2 sleep 20s, checking data +sleep 20000 + + +print ====> step3 sleep 30s, kill leader +sleep 30000 + +print ====> step4 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591201000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step5 sleep 20s, checking data +sleep 20000 + +print ====> step6 stop all +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +print ====> step7 start all +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +print ====> step8 sleep 20s, checking data +sleep 20000 + +print ====> step9 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591202000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step10 sleep 20s, checking data +sleep 20000 + + From 33e4b353800647ec640a0e0f8d8b7de556d24f3b Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 17:15:04 +0800 Subject: [PATCH 048/124] refactor(sync): add case --- .../script/tsim/sync/vnodeLeaderTransfer.sim | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 tests/script/tsim/sync/vnodeLeaderTransfer.sim diff --git a/tests/script/tsim/sync/vnodeLeaderTransfer.sim b/tests/script/tsim/sync/vnodeLeaderTransfer.sim new file mode 100644 index 0000000000..cbeb7063ff --- /dev/null +++ b/tests/script/tsim/sync/vnodeLeaderTransfer.sim @@ -0,0 +1,141 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +$replica = 3 +$vgroups = 6 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 100 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][6] $data[2][11] $data[2][12] $data[2][13] $data[2][14] $data[2][15] $data[2][16] $data[2][17] $data[2][18] $data[2][19] +if $rows != 3 then + return -1 +endi +if $data[2][19] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] + +if $rows != $vgroups then + return -1 +endi + +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + endi + endi +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + endi + endi +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + endi + endi +else + goto check_vg_ready +endi + + +vg_ready: +print ====> create stable/child table +sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +sql create table ct1 using stb tags(1000) + + +print ====> step1 insert 1000 records +$N = 1000 +$count = 0 +while $count < $N + $ms = 1591200000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + +print ====> step2 sleep 20s, checking data +sleep 20000 + + +print ====> step3 sleep 30s, kill leader +sleep 30000 + From 3f11c849cdf0fbf1c24a64ad7d97995951d8303a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 17:34:54 +0800 Subject: [PATCH 049/124] refactor: do some internal refactor. --- source/libs/executor/src/executil.c | 2 +- source/libs/function/inc/builtinsimpl.h | 2 +- source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 358 ++++++++---------------- 4 files changed, 127 insertions(+), 237 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 76d853ef3e..a8ff3188c8 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -65,7 +65,7 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } rowSize += - (numOfOutput * sizeof(bool)); // expand rowSize to mark if col is null for top/bottom result(saveTupleData) + (numOfOutput * sizeof(bool)); // expand rowSize to mark if col is null for top/bottom result(doSaveTupleData) return rowSize; } diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 30fdbb245d..35669b3e42 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -106,7 +106,7 @@ bool irateFuncSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); int32_t irateFunction(SqlFunctionCtx *pCtx); int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); -int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx); +int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6795e4418c..ddff94e9e1 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2232,7 +2232,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, - .processFunc = cacheLastRowFunction, + .processFunc = cachedLastRowFunction, .finalizeFunc = firstLastFinalize }, { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index e2288d9f70..a921e4a70b 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -80,13 +80,14 @@ typedef struct STopBotRes { } STopBotRes; typedef struct SFirstLastRes { - bool hasResult; + bool hasResult; // used for last_row function only, isNullRes in SResultRowEntry can not be passed to downstream.So, // this attribute is required - bool isNull; - int32_t bytes; - int64_t ts; - char buf[]; + bool isNull; + int32_t bytes; + int64_t ts; + STuplePos pos; + char buf[]; } SFirstLastRes; typedef struct SStddevRes { @@ -1141,8 +1142,8 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); -static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); +static void doSaveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); +static void doCopyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, const char* tval) { // the data is loaded, not only the block SMA value @@ -1195,7 +1196,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = *(int64_t*)tval; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } else { if (IS_SIGNED_NUMERIC_TYPE(type)) { @@ -1207,7 +1208,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } @@ -1220,7 +1221,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else if (type == TSDB_DATA_TYPE_DOUBLE) { @@ -1232,7 +1233,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } } else if (type == TSDB_DATA_TYPE_FLOAT) { @@ -1246,7 +1247,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1271,7 +1272,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1283,7 +1284,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1302,7 +1303,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1314,7 +1315,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1333,7 +1334,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1345,7 +1346,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1364,7 +1365,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1376,7 +1377,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1397,7 +1398,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1409,7 +1410,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1428,7 +1429,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1440,7 +1441,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1459,7 +1460,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1471,7 +1472,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1490,7 +1491,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1502,7 +1503,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1522,7 +1523,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1534,7 +1535,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -1553,7 +1554,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doSaveTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } pBuf->assign = true; } else { @@ -1565,7 +1566,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if ((*val < pData[i]) ^ isMinFunc) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); + doCopyTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos); } } } @@ -2640,7 +2641,7 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) } int32_t getFirstLastInfoSize(int32_t resBytes) { - return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t) + sizeof(STuplePos); + return sizeof(SFirstLastRes) + resBytes; } bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -2669,6 +2670,33 @@ static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowInde return *(TSKEY*)colDataGetData(pTsColInfo, rowIndex); } +static void saveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, SFirstLastRes* pInfo) { + if (pCtx->subsidiaries.num <= 0) { + return; + } + + if (!pInfo->hasResult) { + doSaveTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); + } else { + doCopyTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); + } +} + +static void doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t currentTs, int32_t type, char* pData) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + if (IS_VAR_DATA_TYPE(type)) { + pInfo->bytes = varDataTLen(pData); + } + + memcpy(pInfo->buf, pData, pInfo->bytes); + pInfo->ts = currentTs; + saveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + + pInfo->hasResult = true; +} + // This ordinary first function does not care if current scan is ascending order or descending order scan // the OPTIMIZED version of first function will only handle the ascending order scan int32_t firstFunction(SqlFunctionCtx* pCtx) { @@ -2680,9 +2708,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - int32_t type = pInputCol->info.type; - int32_t bytes = pInputCol->info.bytes; - pInfo->bytes = bytes; + pInfo->bytes = pInputCol->info.bytes; // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { @@ -2700,8 +2726,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (blockDataOrder == TSDB_ORDER_ASC) { // filter according to current result firstly if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); - if (ts < startKey) { + if (pInfo->ts < startKey) { return TSDB_CODE_SUCCESS; } } @@ -2715,26 +2740,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { + doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); break; } } @@ -2742,8 +2749,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { // in case of descending order time stamp serial, which usually happens as the results of the nest query, // all data needs to be check. if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); - if (ts < endKey) { + if (pInfo->ts < endKey) { return TSDB_CODE_SUCCESS; } } @@ -2758,24 +2764,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { + doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); break; } } @@ -2821,26 +2811,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); } + break; } } else { // descending order @@ -2853,24 +2827,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf, data, bytes); - *(TSKEY*)(pInfo->buf + bytes) = cts; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - pResInfo->numOfRes = 1; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); } break; } @@ -2885,8 +2843,9 @@ static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, S int32_t start = pColInfo->startRowIndex; pOutput->bytes = pInput->bytes; - TSKEY* tsIn = (TSKEY*)(pInput->buf + pInput->bytes); - TSKEY* tsOut = (TSKEY*)(pOutput->buf + pInput->bytes); + TSKEY* tsIn = &pInput->ts; + TSKEY* tsOut = &pOutput->ts; + if (pOutput->hasResult) { if (isFirst) { if (*tsIn > *tsOut) { @@ -2898,20 +2857,12 @@ static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, S } } } + *tsOut = *tsIn; memcpy(pOutput->buf, pInput->buf, pOutput->bytes); - // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pOutput->buf + pOutput->bytes + sizeof(TSKEY)); - if (pCtx->subsidiaries.num > 0) { - if (!pOutput->hasResult) { - saveTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, start, pCtx->pSrcBlock, pTuplePos); - } - } - pOutput->hasResult = true; + saveTupleData(pCtx->pSrcBlock, start, pCtx, pOutput); - return; + pOutput->hasResult = true; } static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuery) { @@ -2953,34 +2904,34 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { colDataAppend(pCol, pBlock->info.rows, pRes->buf, pRes->isNull || pResInfo->isNullRes); // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY)); - setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows); + setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); return pResInfo->numOfRes; } int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); - SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); int32_t resultBytes = getFirstLastInfoSize(pRes->bytes); - char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + // todo check for failure + char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); memcpy(varDataVal(res), pRes, resultBytes); + varDataSetLen(res, resultBytes); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); colDataAppend(pCol, pBlock->info.rows, res, false); - // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY)); - setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows); + setSelectivityValue(pCtx, pBlock, &pRes->pos, pBlock->info.rows); taosMemoryFree(res); return 1; } +//todo rewrite: int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); char* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); @@ -2998,6 +2949,28 @@ int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { return TSDB_CODE_SUCCESS; } +static void doSaveLastrow(SqlFunctionCtx *pCtx, char* pData, int32_t rowIndex, int64_t cts, SFirstLastRes* pInfo) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + if (colDataIsNull_s(pInputCol, rowIndex)) { + pInfo->isNull = true; + } else { + pInfo->isNull = false; + + if (IS_VAR_DATA_TYPE(pInputCol->info.type)) { + pInfo->bytes = varDataTLen(pData); + } + + memcpy(pInfo->buf, pData, pInfo->bytes); + } + + pInfo->ts = cts; + saveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pInfo); + + pInfo->hasResult = true; +} + int32_t lastRowFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; @@ -3007,12 +2980,9 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; pInfo->bytes = bytes; - SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet) ? pInput->pColumnDataAgg[0] : NULL; - TSKEY startKey = getRowPTs(pInput->pPTS, 0); TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); @@ -3022,31 +2992,10 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - pInfo->isNull = true; - } else { - pInfo->isNull = false; - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf + sizeof(TSKEY), data, bytes); - } - *(TSKEY*)(pInfo->buf) = cts; - numOfElems++; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); - pResInfo->numOfRes = 1; + numOfElems++; + + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveLastrow(pCtx, data, i, cts, pInfo); } break; } @@ -3054,31 +3003,10 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { - pInfo->isNull = true; - } else { - pInfo->isNull = false; - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - memcpy(pInfo->buf + sizeof(TSKEY), data, bytes); - } - *(TSKEY*)(pInfo->buf) = cts; - numOfElems++; - // handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - pInfo->hasResult = true; - pResInfo->numOfRes = 1; - // DO_UPDATE_TAG_COLUMNS(pCtx, ts); + numOfElems++; + + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveLastrow(pCtx, data, i, cts, pInfo); } break; } @@ -3088,21 +3016,6 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t lastRowFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - int32_t slotId = pCtx->pExpr->base.resSchema.slotId; - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - - SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); - colDataAppend(pCol, pBlock->info.rows, pRes->buf + sizeof(TSKEY), pRes->isNull); - // handle selectivity - STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY)); - setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows); - - return pResInfo->numOfRes; -} - bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SDiffInfo); return true; @@ -3425,7 +3338,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // save the data of this tuple if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + doSaveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); } #ifdef BUF_PAGE_DEBUG qDebug("page_saveTuple i:%d, item:%p,pageId:%d, offset:%d\n", pEntryInfo->numOfRes, pItem, pItem->tuplePos.pageId, @@ -3449,7 +3362,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // save the data of this tuple by over writing the old data if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); + doCopyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos); } #ifdef BUF_PAGE_DEBUG qDebug("page_copyTuple pageId:%d, offset:%d", pItem->tuplePos.pageId, pItem->tuplePos.offset); @@ -3466,7 +3379,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData * |(n columns, one bit for each column)| src column #1| src column #2| * +------------------------------------+--------------+--------------+ */ -void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { +void doSaveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = NULL; // todo refactor: move away @@ -3527,7 +3440,7 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS #endif } -void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { +void doCopyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId); int32_t numOfCols = pCtx->subsidiaries.num; @@ -4843,7 +4756,7 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da if (pInfo->numSampled < pInfo->samples) { sampleAssignResult(pInfo, data, pInfo->numSampled); if (pCtx->subsidiaries.num > 0) { - saveTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[pInfo->numSampled]); + doSaveTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[pInfo->numSampled]); } pInfo->numSampled++; } else { @@ -4851,7 +4764,7 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da if (j < pInfo->samples) { sampleAssignResult(pInfo, data, j); if (pCtx->subsidiaries.num > 0) { - copyTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[j]); + doCopyTupleData(pCtx, index, pCtx->pSrcBlock, &pInfo->tuplePos[j]); } } } @@ -5993,7 +5906,7 @@ int32_t interpFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx) { +int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -6002,9 +5915,7 @@ int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; - pInfo->bytes = bytes; // last_row function does not ignore the null value @@ -6014,28 +5925,7 @@ int32_t cacheLastRowFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - if (colDataIsNull_s(pInputCol, i)) { - pInfo->isNull = true; - } else { - if (IS_VAR_DATA_TYPE(type)) { - bytes = varDataTLen(data); - pInfo->bytes = bytes; - } - - memcpy(pInfo->buf, data, bytes); - } - - pInfo->ts = cts; - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } - - pInfo->hasResult = true; + doSaveLastrow(pCtx, data, i, cts, pInfo); } } From 339a0086c052a3881e9b284f834e440974ff7cd0 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 17:42:57 +0800 Subject: [PATCH 050/124] docs: refactor cluster chapter --- docs/zh/10-cluster/01-deploy.md | 26 +--- docs/zh/10-cluster/02-cluster-mgmt.md | 145 +++------------------ docs/zh/10-cluster/03-ha-and-lb.md | 87 ------------- docs/zh/10-cluster/03-high-availability.md | 33 +++++ 4 files changed, 56 insertions(+), 235 deletions(-) delete mode 100644 docs/zh/10-cluster/03-ha-and-lb.md create mode 100644 docs/zh/10-cluster/03-high-availability.md diff --git a/docs/zh/10-cluster/01-deploy.md b/docs/zh/10-cluster/01-deploy.md index b44d2942f2..9dbfb50f18 100644 --- a/docs/zh/10-cluster/01-deploy.md +++ b/docs/zh/10-cluster/01-deploy.md @@ -10,7 +10,7 @@ title: 集群部署 ### 第一步 -如果搭建集群的物理节点中，存有之前的测试数据、装过 1.X 的版本，或者装过其他版本的 TDengine，请先将其删除，并清空所有数据（如果需要保留原有数据，请联系涛思交付团队进行旧版本升级、数据迁移），具体步骤请参考博客[《TDengine 多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html)。 +如果搭建集群的物理节点中，存有之前的测试数据，或者装过其他版本的 TDengine，请先将其删除，并清空所有数据（如果需要保留原有数据，请联系涛思交付团队进行旧版本升级、数据迁移），具体步骤请参考博客[《TDengine 多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html)。 :::note 因为 FQDN 的信息会写进文件，如果之前没有配置或者更改 FQDN，且启动了 TDengine。请一定在确保数据无用或者备份的前提下，清理一下之前的数据（rm -rf /var/lib/taos/\*）； @@ -54,30 +54,16 @@ fqdn h1.taosdata.com // 配置本数据节点的端口号，缺省是 6030 serverPort 6030 -// 副本数为偶数的时候，需要配置，请参考《Arbitrator 的使用》的部分 -arbitrator ha.taosdata.com:6042 -``` - 一定要修改的参数是 firstEp 和 fqdn。在每个数据节点，firstEp 需全部配置成一样，但 fqdn 一定要配置成其所在数据节点的值。其他参数可不做任何修改，除非你很清楚为什么要修改。 -加入到集群中的数据节点 dnode，涉及集群相关的下表 9 项参数必须完全相同，否则不能成功加入到集群中。 +加入到集群中的数据节点 dnode，下表中涉及集群相关的参数必须完全相同，否则不能成功加入到集群中。 | **#** | **配置参数名称** | **含义** | | ----- | ------------------ | ------------------------------------------- | -| 1 | numOfMnodes | 系统中管理节点个数 | -| 2 | mnodeEqualVnodeNum | 一个 mnode 等同于 vnode 消耗的个数 | -| 3 | offlineThreshold | dnode 离线阈值，超过该时间将导致 Dnode 离线 | -| 4 | statusInterval | dnode 向 mnode 报告状态时长 | -| 5 | arbitrator | 系统中裁决器的 End Point | -| 6 | timezone | 时区 | -| 7 | balance | 是否启动负载均衡 | -| 8 | maxTablesPerVnode | 每个 vnode 中能够创建的最大表个数 | -| 9 | maxVgroupsPerDb | 每个 DB 中能够使用的最大 vgroup 个数 | - -:::note -在 2.0.19.0 及更早的版本中，除以上 9 项参数外，dnode 加入集群时，还会要求 locale 和 charset 参数的取值也一致。 - -::: +| 1 | statusInterval | dnode 向 mnode 报告状态时长 | +| 2 | timezone | 时区 | +| 3 | locale | 系统区位信息及编码格式 | +| 4 | charset | 字符集编码 | ## 启动集群 diff --git a/docs/zh/10-cluster/02-cluster-mgmt.md b/docs/zh/10-cluster/02-cluster-mgmt.md index 6ab8ec091b..f02a105826 100644 --- a/docs/zh/10-cluster/02-cluster-mgmt.md +++ b/docs/zh/10-cluster/02-cluster-mgmt.md @@ -24,15 +24,15 @@ SHOW DNODES; ``` taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.008298s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | trd01:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | +Query OK, 1 rows affected (0.006684s) ``` ## 查看虚拟节点组 -为充分利用多核技术，并提供 scalability，数据需要分片处理。因此 TDengine 会将一个 DB 的数据切分成多份，存放在多个 vnode 里。这些 vnode 可能分布在多个数据节点 dnode 里，这样就实现了水平扩展。一个 vnode 仅仅属于一个 DB，但一个 DB 可以有多个 vnode。vnode 所在的数据节点是 mnode 根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 +为充分利用多核技术，并提供横向扩展能力，数据需要分片处理。因此 TDengine 会将一个 DB 的数据切分成多份，存放在多个 vnode 里。这些 vnode 可能分布在多个数据节点 dnode 里，这样就实现了水平扩展。一个 vnode 仅仅属于一个 DB，但一个 DB 可以有多个 vnode。vnode 所在的数据节点是 mnode 根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 启动 CLI 程序 taos，然后执行： @@ -44,26 +44,15 @@ SHOW VGROUPS; 输出如下（具体内容仅供参考，取决于实际的集群配置） ``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.008298s) - taos> use db; Database changed. taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | compacting | -========================================================================================== - 14 | 38000 | ready | 1 | 1 | master | 0 | - 15 | 38000 | ready | 1 | 1 | master | 0 | - 16 | 38000 | ready | 1 | 1 | master | 0 | - 17 | 38000 | ready | 1 | 1 | master | 0 | - 18 | 37001 | ready | 1 | 1 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | status | nfiles | file_size | tsma | +================================================================================================================================================================================================ + 2 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | + 3 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | + 4 | db | 0 | 1 | leader | NULL | NULL | NULL | NULL | NULL | NULL | NULL | 0 | Query OK, 8 row(s) in set (0.001154s) ``` @@ -77,35 +66,21 @@ CREATE DNODE "fqdn:port"; 将新数据节点的 End Point 添加进集群的 EP 列表。“fqdn:port“需要用双引号引起来，否则出错。一个数据节点对外服务的 fqdn 和 port 可以通过配置文件 taos.cfg 进行配置，缺省是自动获取。【强烈不建议用自动获取方式来配置 FQDN，可能导致生成的数据节点的 End Point 不是所期望的】 -示例如下： -``` -taos> create dnode "localhost:7030"; -Query OK, 0 of 0 row(s) in database (0.008203s) - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 0 | 0 | offline | any | 2022-04-19 08:11:42.158 | status not received | -Query OK, 2 row(s) in set (0.001017s) -``` - -在上面的示例中可以看到新创建的 dnode 的状态为 offline，待该 dnode 被启动并连接上配置文件中指定的 firstEp后再次查看，得到如下结果（示例） +然后启动新加入的数据节点的 taosd 进程，再通过 taos 查看数据节点状态： ``` taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 3 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 6 | 8 | ready | any | 2022-04-19 08:14:59.165 | | -Query OK, 2 row(s) in set (0.001316s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | trd01:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | + 2 | trd04:6030 | 0 | 1024 | ready | 2022-07-15 16:56:13.670 | | +Query OK, 2 rows affected (0.007031s) ``` 从中可以看到两个 dnode 状态都为 ready - ## 删除数据节点 -启动 CLI 程序 taos，然后执行： +先停止要删除的数据节点的 taosd 进程，然后启动 CLI 程序 taos，执行： ```sql DROP DNODE "fqdn:port"; @@ -117,26 +92,6 @@ DROP DNODE dnodeId; 通过 “fqdn:port” 或 dnodeID 来指定一个具体的节点都是可以的。其中 fqdn 是被删除的节点的 FQDN，port 是其对外服务器的端口号；dnodeID 可以通过 SHOW DNODES 获得。 -示例如下： -``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | - 2 | localhost:7030 | 0 | 0 | offline | any | 2022-04-19 08:11:42.158 | status not received | -Query OK, 2 row(s) in set (0.001017s) - -taos> drop dnode 2; -Query OK, 0 of 0 row(s) in database (0.000518s) - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.001137s) -``` - -上面的示例中，初次执行 `show dnodes` 列出了两个 dnode, 执行 `drop dnode 2` 删除其中 ID 为 2 的 dnode 之后再次执行 `show dnodes`，可以看到只剩下 ID 为 1 的 dnode 。 :::warning @@ -147,70 +102,4 @@ dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增 ::: -## 手动迁移数据节点 - -手动将某个 vnode 迁移到指定的 dnode。 - -启动 CLI 程序 taos，然后执行： - -```sql -ALTER DNODE BALANCE "VNODE:-DNODE:"; -``` - -其中：source-dnodeId 是源 dnodeId，也就是待迁移的 vnode 所在的 dnodeID；vgId 可以通过 SHOW VGROUPS 获得，列表的第一列；dest-dnodeId 是目标 dnodeId。 - -首先执行 `show vgroups` 查看 vgroup 的分布情况 -``` -taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | compacting | -========================================================================================== - 14 | 38000 | ready | 1 | 3 | master | 0 | - 15 | 38000 | ready | 1 | 3 | master | 0 | - 16 | 38000 | ready | 1 | 3 | master | 0 | - 17 | 38000 | ready | 1 | 3 | master | 0 | - 18 | 37001 | ready | 1 | 3 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | -Query OK, 8 row(s) in set (0.001314s) -``` - -从中可以看到在 dnode 3 中有5个 vgroup，而 dnode 1 有 3 个 vgroup，假定我们想将其中 vgId 为18 的 vgroup 从 dnode 3 迁移到 dnode 1 - -``` -taos> alter dnode 3 balance "vnode:18-dnode:1"; - -DB error: Balance already enabled (0.00755 -``` - -上面的结果表明目前所在数据库已经启动了 balance 选项，所以无法进行手动迁移。 - -停止整个集群，将两个 dnode 的配置文件中的 balance 都设置为 0 （默认为1）之后，重新启动集群，再次执行 ` alter dnode` 和 `show vgroups` 命令如下 -``` -taos> alter dnode 3 balance "vnode:18-dnode:1"; -Query OK, 0 row(s) in set (0.000575s) - -taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | v2_dnode | v2_status | compacting | -================================================================================================================= - 14 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 15 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 16 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 17 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 18 | 37001 | ready | 2 | 1 | slave | 3 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | -Query OK, 8 row(s) in set (0.001242s) -``` - -从上面的输出可以看到 vgId 为 18 的 vnode 被从 dnode 3 迁移到了 dnode 1。 - -:::warning - -只有在集群的自动负载均衡选项关闭时（balance 设置为 0），才允许手动迁移。 -只有处于正常工作状态的 vnode 才能被迁移：master/slave；当处于 offline/unsynced/syncing 状态时，是不能迁移的。 -迁移前，务必核实目标 dnode 的资源足够：CPU、内存、硬盘。 - -::: diff --git a/docs/zh/10-cluster/03-ha-and-lb.md b/docs/zh/10-cluster/03-ha-and-lb.md deleted file mode 100644 index 3d15feb11c..0000000000 --- a/docs/zh/10-cluster/03-ha-and-lb.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -title: 高可用与负载均衡 ---- - -## Vnode 的高可用性 - -TDengine 通过多副本的机制来提供系统的高可用性，包括 vnode 和 mnode 的高可用性。 - -vnode 的副本数是与 DB 关联的，一个集群里可以有多个 DB，根据运营的需求，每个 DB 可以配置不同的副本数。创建数据库时，通过参数 replica 指定副本数（缺省为 1）。如果副本数为 1，系统的可靠性无法保证，只要数据所在的节点宕机，就将无法提供服务。集群的节点数必须大于等于副本数，否则创建表时将返回错误“more dnodes are needed”。比如下面的命令将创建副本数为 3 的数据库 demo： - -```sql -CREATE DATABASE demo replica 3; -``` - -一个 DB 里的数据会被切片分到多个 vnode group，vnode group 里的 vnode 数目就是 DB 的副本数，同一个 vnode group 里各 vnode 的数据是完全一致的。为保证高可用性，vnode group 里的 vnode 一定要分布在不同的数据节点 dnode 里（实际部署时，需要在不同的物理机上），只要一个 vnode group 里超过半数的 vnode 处于工作状态，这个 vnode group 就能正常的对外服务。 - -一个数据节点 dnode 里可能有多个 DB 的数据，因此一个 dnode 离线时，可能会影响到多个 DB。如果一个 vnode group 里的一半或一半以上的 vnode 不工作，那么该 vnode group 就无法对外服务，无法插入或读取数据，这样会影响到它所属的 DB 的一部分表的读写操作。 - -因为 vnode 的引入，无法简单地给出结论：“集群中过半数据节点 dnode 工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为 3，只有三个 dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个数据节点不工作，那整个集群就无法正常工作了。 - -## Mnode 的高可用性 - -TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管理的，为保证 mnode 的高可用，可以配置多个 mnode 副本，副本数由系统配置参数 numOfMnodes 决定，有效范围为 1-3。为保证元数据的强一致性，mnode 副本之间是通过同步的方式进行数据复制的。 - -一个集群有多个数据节点 dnode，但一个 dnode 至多运行一个 mnode 实例。多个 dnode 情况下，哪个 dnode 可以作为 mnode 呢？这是完全由系统根据整个系统资源情况，自动指定的。用户可通过 CLI 程序 taos，在 TDengine 的 console 里，执行如下命令： - -```sql -SHOW MNODES; -``` - -来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（master，slave，unsynced 或 offline）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。如果 numOfMnodes 配置为 2，启动第二个 dnode 时，该 dnode 也将运行一个 mnode 实例。 - -为保证 mnode 服务的高可用性，numOfMnodes 必须设置为 2 或更大。因为 mnode 保存的元数据必须是强一致的，如果 numOfMnodes 大于 2，复制参数 quorum 自动设为 2，也就是说，至少要保证有两个副本写入数据成功，才通知客户端应用写入成功。 - -:::note -一个 TDengine 高可用系统，无论是 vnode 还是 mnode，都必须配置多个副本。 - -::: - -## 负载均衡 - -有三种情况，将触发负载均衡，而且都无需人工干预。 - -当一个新数据节点添加进集群时，系统将自动触发负载均衡，一些节点上的数据将被自动转移到新数据节点上，无需任何人工干预。 -当一个数据节点从集群中移除时，系统将自动把该数据节点上的数据转移到其他数据节点，无需任何人工干预。 -如果一个数据节点过热（数据量过大），系统将自动进行负载均衡，将该数据节点的一些 vnode 自动挪到其他节点。 -当上述三种情况发生时，系统将启动各个数据节点的负载计算，从而决定如何挪动。 - -:::tip -负载均衡由参数 balance 控制，它决定是否启动自动负载均衡，0 表示禁用，1 表示启用自动负载均衡。 - -::: - -## 数据节点离线处理 - -如果一个数据节点离线，TDengine 集群将自动检测到。有如下两种情况： - -该数据节点离线超过一定时间（taos.cfg 里配置参数 offlineThreshold 控制时长），系统将自动把该数据节点删除，产生系统报警信息，触发负载均衡流程。如果该被删除的数据节点重新上线时，它将无法加入集群，需要系统管理员重新将其添加进集群才会开始工作。 - -离线后，在 offlineThreshold 的时长内重新上线，系统将自动启动数据恢复流程，等数据完全恢复后，该节点将开始正常工作。 - -:::note -如果一个虚拟节点组（包括 mnode 组）里所归属的每个数据节点都处于离线或 unsynced 状态，必须等该虚拟节点组里的所有数据节点都上线、都能交换状态信息后，才能选出 Master，该虚拟节点组才能对外提供服务。比如整个集群有 3 个数据节点，副本数为 3，如果 3 个数据节点都宕机，然后 2 个数据节点重启，是无法工作的，只有等 3 个数据节点都重启成功，才能对外服务。 - -::: - -## Arbitrator 的使用 - -如果副本数为偶数，当一个 vnode group 里一半或超过一半的 vnode 不工作时，是无法从中选出 master 的。同理，一半或超过一半的 mnode 不工作时，是无法选出 mnode 的 master 的，因为存在“split brain”问题。 - -为解决这个问题，TDengine 引入了 Arbitrator 的概念。Arbitrator 模拟一个 vnode 或 mnode 在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含 Arbitrator 在内，超过半数的 vnode 或 mnode 工作，那么该 vnode group 或 mnode 组就可以正常的提供数据插入或查询服务。比如对于副本数为 2 的情形，如果一个节点 A 离线，但另外一个节点 B 正常，而且能连接到 Arbitrator，那么节点 B 就能正常工作。 - -总之，在目前版本下，TDengine 建议在双副本环境要配置 Arbitrator，以提升系统的可用性。 - -Arbitrator 的执行程序名为 tarbitrator。该程序对系统资源几乎没有要求，只需要保证有网络连接，找任何一台 Linux 服务器运行它即可。以下简要描述安装配置的步骤： - -请点击 安装包下载，在 TDengine Arbitrator Linux 一节中，选择合适的版本下载并安装。 -该应用的命令行参数 -p 可以指定其对外服务的端口号，缺省是 6042。 - -修改每个 taosd 实例的配置文件，在 taos.cfg 里将参数 arbitrator 设置为 tarbitrator 程序所对应的 End Point。（如果该参数配置了，当副本数为偶数时，系统将自动连接配置的 Arbitrator。如果副本数为奇数，即使配置了 Arbitrator，系统也不会去建立连接。） - -在配置文件中配置了的 Arbitrator，会出现在 SHOW DNODES 指令的返回结果中，对应的 role 列的值会是“arb”。 -查看集群 Arbitrator 的状态【2.0.14.0 以后支持】 - -```sql -SHOW DNODES; -``` diff --git a/docs/zh/10-cluster/03-high-availability.md b/docs/zh/10-cluster/03-high-availability.md new file mode 100644 index 0000000000..1ec413437f --- /dev/null +++ b/docs/zh/10-cluster/03-high-availability.md @@ -0,0 +1,33 @@ +--- +title: 高可用与负载均衡 +--- + +## Vnode 的高可用性 + +TDengine 通过多副本的机制来提供系统的高可用性，包括 vnode 和 mnode 的高可用性。 + +vnode 的副本数是与 DB 关联的，一个集群里可以有多个 DB，根据运营的需求，每个 DB 可以配置不同的副本数。创建数据库时，通过参数 replica 指定副本数（缺省为 1）。如果副本数为 1，系统的可靠性无法保证，只要数据所在的节点宕机，就将无法提供服务。集群的节点数必须大于等于副本数，否则创建表时将返回错误“more dnodes are needed”。比如下面的命令将创建副本数为 3 的数据库 demo： + +```sql +CREATE DATABASE demo replica 3; +``` + +一个 DB 里的数据会被切片分到多个 vnode group，vnode group 里的 vnode 数目就是 DB 的副本数，同一个 vnode group 里各 vnode 的数据是完全一致的。为保证高可用性，vnode group 里的 vnode 一定要分布在不同的数据节点 dnode 里（实际部署时，需要在不同的物理机上），只要一个 vnode group 里超过半数的 vnode 处于工作状态，这个 vnode group 就能正常的对外服务。 + +一个数据节点 dnode 里可能有多个 DB 的数据，因此一个 dnode 离线时，可能会影响到多个 DB。如果一个 vnode group 里的一半或一半以上的 vnode 不工作，那么该 vnode group 就无法对外服务，无法插入或读取数据，这样会影响到它所属的 DB 的一部分表的读写操作。 + +因为 vnode 的引入，无法简单地给出结论：“集群中过半数据节点 dnode 工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为 3，只有三个 dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个数据节点不工作，那整个集群就无法正常工作了。 + +## Mnode 的高可用性 + +TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管理的，为保证 mnode 的高可用，可以配置多个 mnode 副本，在集群启动时只有一个 mnode，用户可以通过 `create mnode`来增加新的mnode。用户可以通过该命令自主决定哪几个 dnode 会承担 mnode 的角色。为保证元数据的强一致性，在有多个 mnode 时，mnode 副本之间是通过同步的方式进行数据复制的。 + +一个集群有多个数据节点 dnode，但一个 dnode 至多运行一个 mnode 实例。用户可通过 CLI 程序 taos，在 TDengine 的 console 里，执行如下命令： + +```sql +SHOW MNODES; +``` + +来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（leader, follower, candidate）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。 + +在 TDengine 3.0 及以后的版本中，数据同步采用 RAFT 协议，所以 mnode 的数量应该被设置为 1 个或者 3 个。 \ No newline at end of file From 232fb94fe0f7cbd546f26eaa79fbf373dc32a784 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 15 Jul 2022 17:51:32 +0800 Subject: [PATCH 051/124] chore: add libtaos ws submodule for3.0 (#14954) * chore: add libtaos-ws for 3.0 * chore: update taosws-rs * chore: add libtaosws to install/remove script * chore: update taosws-rs * chore: update taosws-rs * chore: update taos-tools, taosws-rs for 3.0 * fix: packaging/tools/make_install.sh for 3.0 * chore: update taos-tools * chore: fix release script for 3.0 * chore: update taosws-rs for 3.0 * chore: add taows-rs submodule for 3.0 --- .gitmodules | 2 +- tools/taosws-rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9d649bcdd3..31c211bf89 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,4 +21,4 @@ url = https://github.com/taosdata/taosadapter.git [submodule "tools/taosws-rs"] path = tools/taosws-rs - url = https://github.com/taosdata/taosws-rs.git + url = https://github.com/taosdata/taosws-rs diff --git a/tools/taosws-rs b/tools/taosws-rs index 7a94ffab45..c5fded266d 160000 --- a/tools/taosws-rs +++ b/tools/taosws-rs @@ -1 +1 @@ -Subproject commit 7a94ffab45f08e16f09b3f430fe75d717054adb6 +Subproject commit c5fded266d3b10508e38bf3285bb7ecf798bc343 From 7696cdeb72612fe7ccf5e6ea33c87e1b4e4eaf17 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 17:52:39 +0800 Subject: [PATCH 052/124] doc: add a section for load balance --- docs/zh/10-cluster/04-load-balance.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 docs/zh/10-cluster/04-load-balance.md diff --git a/docs/zh/10-cluster/04-load-balance.md b/docs/zh/10-cluster/04-load-balance.md new file mode 100644 index 0000000000..5e79d6b90e --- /dev/null +++ b/docs/zh/10-cluster/04-load-balance.md @@ -0,0 +1,15 @@ +--- +title: 负载均衡 +--- + +TDengine 中的负载均衡主要指对时序数据的处理的负载均衡。TDengine 采用 Hash 一致性算法将一个数据库中的所有表和子表的数据均衡分散在属于该数据库的所有 vgroups 中，每张表或子表只能由一个 vgroups 处理，一个 vgroups 可能负责处理多个表或子表。 + +创建数据库时可以指定其中的 vgroups 的数量： + +```sql +create database db0 vgroups 100; +``` + +如何指定合适的 vgroups 的数量，这取决于系统资源。假定系统中只计划建立一个数据库，则 vgroups 由集群中所有 dnode 所能使用的资源决定。原则上可用的 CPU 和 Memory 越多，可建立的 vgroups 也越多。但也要考虑到磁盘性能，过多的 vgroups 在磁盘性能达到上限后反而会拖累整个系统的性能。假如系统中会建立多个数据库，则多个数据库的 vgoups 之和取决于系统中可用资源的数量。要综合考虑多个数据库之间表的数量、写入频率、数据量等多个因素在多个数据库之间分配 vgroups。实际中建议首先根据系统资源配置选择一个初始的 vgroups 数量，比如 CPU 总核数的 2 倍，以此为起点通过测试找到最佳的 vgroups 数量配置，此为系统中的 vgroups 总数。如果有多个数据库的话，再根据各个数据库的表数和数据量对 vgroups 进行分配。 + +通过以上措施可以最大限度地在整个 TDengine 集群中实现负载均衡，负载均衡也能反过来提升系统总的数据处理能力。 \ No newline at end of file From cbd2d8f8a2876f7b7c503fe27b1bf9f471012c6d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 15 Jul 2022 17:54:45 +0800 Subject: [PATCH 053/124] fix(query): add spread function scalar version TD-17344 --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 42 +++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index df3e27225f..e0beaff262 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -106,6 +106,7 @@ int32_t stddevScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 74c0e54640..208022ad1a 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2115,6 +2115,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getSpreadFuncEnv, .initFunc = spreadFunctionSetup, .processFunc = spreadFunction, + .sprocessFunc = spreadScalarFunction, .finalizeFunc = spreadFinalize, .invertFunc = NULL, .combineFunc = spreadCombine, diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 8808d17454..9a8f570a52 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2338,3 +2338,45 @@ int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return percentileScalarFunction(pInput, inputNum, pOutput); } + +int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int32_t type = GET_PARAM_TYPE(pInput); + + double min, max; + SET_DOUBLE_VAL(&min, DBL_MAX); + SET_DOUBLE_VAL(&max, -DBL_MAX); + + bool hasNull = false; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + hasNull = true; + break; + } + + char *in = pInputData->pData; + + double val = 0; + GET_TYPED_DATA(val, double, type, in); + + if (val < GET_DOUBLE_VAL(&min)) { + SET_DOUBLE_VAL(&min, val); + } + + if (val > GET_DOUBLE_VAL(&max)) { + SET_DOUBLE_VAL(&max, val); + } + } + + if (hasNull) { + colDataAppendNULL(pOutputData, 0); + } else { + double result = max - min; + colDataAppend(pOutputData, 0, (char *)&result, false); + } + + pOutput->numOfRows = 1; + return TSDB_CODE_SUCCESS; +} From ce6b52cd72a1064988254e129ab80a6dd202f70a Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 17:57:38 +0800 Subject: [PATCH 054/124] doc: refine load balance --- docs/zh/10-cluster/04-load-balance.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/zh/10-cluster/04-load-balance.md b/docs/zh/10-cluster/04-load-balance.md index 5e79d6b90e..9b582f417d 100644 --- a/docs/zh/10-cluster/04-load-balance.md +++ b/docs/zh/10-cluster/04-load-balance.md @@ -12,4 +12,6 @@ create database db0 vgroups 100; 如何指定合适的 vgroups 的数量，这取决于系统资源。假定系统中只计划建立一个数据库，则 vgroups 由集群中所有 dnode 所能使用的资源决定。原则上可用的 CPU 和 Memory 越多，可建立的 vgroups 也越多。但也要考虑到磁盘性能，过多的 vgroups 在磁盘性能达到上限后反而会拖累整个系统的性能。假如系统中会建立多个数据库，则多个数据库的 vgoups 之和取决于系统中可用资源的数量。要综合考虑多个数据库之间表的数量、写入频率、数据量等多个因素在多个数据库之间分配 vgroups。实际中建议首先根据系统资源配置选择一个初始的 vgroups 数量，比如 CPU 总核数的 2 倍，以此为起点通过测试找到最佳的 vgroups 数量配置，此为系统中的 vgroups 总数。如果有多个数据库的话，再根据各个数据库的表数和数据量对 vgroups 进行分配。 +此外，对于任意数据库的 vgroups，TDengine 都是尽可能将其均衡分散在多个 dnode 上。在多副本情况下（replica 3），这种均衡分布尤其复杂，TDengine 的分布策略会尽量避免任意一个 dnode 成为写入的瓶颈。 + 通过以上措施可以最大限度地在整个 TDengine 集群中实现负载均衡，负载均衡也能反过来提升系统总的数据处理能力。 \ No newline at end of file From d99b682f765a1f8407801e757e1e42fc5c767f34 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 15 Jul 2022 17:58:37 +0800 Subject: [PATCH 055/124] fix: multi-rows function validity check --- include/libs/nodes/querynodes.h | 1 + source/libs/parser/src/parTranslater.c | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index d58d5d4b8b..f8c7024591 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -258,6 +258,7 @@ typedef struct SSelectStmt { bool hasAggFuncs; bool hasRepeatScanFuncs; bool hasIndefiniteRowsFunc; + bool hasMultiRowsFunc; bool hasSelectFunc; bool hasSelectValFunc; bool hasOtherVectorFunc; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0d2959f21d..33ab1166b0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1058,7 +1058,8 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (hasInvalidFuncNesting(pFunc->pParameterList)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } - if (isSelectStmt(pCxt->pCurrStmt) && ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc) { + if (!isSelectStmt(pCxt->pCurrStmt) || ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || + ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } @@ -1093,7 +1094,8 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod return TSDB_CODE_SUCCESS; } if (!isSelectStmt(pCxt->pCurrStmt) || SQL_CLAUSE_SELECT != pCxt->currClause || - ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs) { + ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs || + ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } if (NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pWindow || NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { @@ -1111,7 +1113,8 @@ static int32_t translateMultiRowsFunc(STranslateContext* pCxt, SFunctionNode* pF return TSDB_CODE_SUCCESS; } if (!isSelectStmt(pCxt->pCurrStmt) || SQL_CLAUSE_SELECT != pCxt->currClause || - ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs) { + ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasAggFuncs || + ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } if (hasInvalidFuncNesting(pFunc->pParameterList)) { @@ -1204,6 +1207,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); pSelect->hasIndefiniteRowsFunc = pSelect->hasIndefiniteRowsFunc ? true : fmIsIndefiniteRowsFunc(pFunc->funcId); + pSelect->hasMultiRowsFunc = pSelect->hasMultiRowsFunc ? true : fmIsMultiRowsFunc(pFunc->funcId); if (fmIsSelectFunc(pFunc->funcId)) { pSelect->hasSelectFunc = true; ++(pSelect->selectFuncNum); From 9394e3380f6f4d8da734bc0789a2ab0a10dd5648 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 15 Jul 2022 17:48:48 +0800 Subject: [PATCH 056/124] feat(stream): recover from failure --- include/common/tcommon.h | 10 +- include/common/tmsg.h | 29 ++++- include/common/tmsgdef.h | 1 + include/libs/executor/executor.h | 2 + include/libs/stream/tstream.h | 49 +++++++- source/common/src/tmsg.c | 39 +++++- source/dnode/mnode/impl/inc/mndDef.h | 18 +++ source/dnode/mnode/impl/inc/mndTrans.h | 1 + source/dnode/mnode/impl/src/mndDef.c | 2 + source/dnode/mnode/impl/src/mndScheduler.c | 1 + source/dnode/mnode/impl/src/mndStream.c | 66 ++++++++++- source/dnode/mnode/sdb/inc/sdb.h | 25 ++-- source/libs/executor/inc/executorimpl.h | 10 ++ source/libs/executor/src/executorMain.c | 9 ++ source/libs/executor/src/scanoperator.c | 33 ++++-- source/libs/stream/src/stream.c | 9 +- source/libs/stream/src/streamData.c | 2 +- source/libs/stream/src/streamExec.c | 27 ++++- source/libs/stream/src/streamRecover.c | 131 +++++++++++++++++++++ source/libs/stream/src/streamTask.c | 2 + 20 files changed, 423 insertions(+), 43 deletions(-) create mode 100644 source/libs/stream/src/streamRecover.c diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5a7b32d20b..444a17c7f8 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -57,7 +57,7 @@ enum { // STREAM_INPUT__TABLE_SCAN, STREAM_INPUT__TQ_SCAN, STREAM_INPUT__DATA_RETRIEVE, - STREAM_INPUT__TRIGGER, + STREAM_INPUT__GET_RES, STREAM_INPUT__CHECKPOINT, STREAM_INPUT__DROP, }; @@ -155,10 +155,10 @@ typedef struct SQueryTableDataCond { int32_t numOfCols; SColumnInfo* colList; int32_t type; // data block load type: -// int32_t numOfTWindows; - STimeWindow twindows; - int64_t startVersion; - int64_t endVersion; + // int32_t numOfTWindows; + STimeWindow twindows; + int64_t startVersion; + int64_t endVersion; } SQueryTableDataCond; int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index c8e13fce3d..49feba4d3c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1968,7 +1968,7 @@ typedef struct SVCreateTbReq { int8_t type; union { struct { - char* name; // super table name + char* name; // super table name tb_uid_t suid; SArray* tagName; uint8_t* pTag; @@ -2437,9 +2437,6 @@ typedef struct { int8_t igNotExists; } SMDropStreamReq; -int32_t tSerializeSMDropStreamReq(void* buf, int32_t bufLen, const SMDropStreamReq* pReq); -int32_t tDeserializeSMDropStreamReq(void* buf, int32_t bufLen, SMDropStreamReq* pReq); - typedef struct { int8_t reserved; } SMDropStreamRsp; @@ -2454,6 +2451,27 @@ typedef struct { int8_t reserved; } SVDropStreamTaskRsp; +int32_t tSerializeSMDropStreamReq(void* buf, int32_t bufLen, const SMDropStreamReq* pReq); +int32_t tDeserializeSMDropStreamReq(void* buf, int32_t bufLen, SMDropStreamReq* pReq); + +typedef struct { + char name[TSDB_STREAM_FNAME_LEN]; + int8_t igNotExists; +} SMRecoverStreamReq; + +typedef struct { + int8_t reserved; +} SMRecoverStreamRsp; + +typedef struct { + int64_t recoverObjUid; + int32_t taskId; + int32_t hasCheckPoint; +} SMVStreamGatherInfoReq; + +int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq); +int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq); + typedef struct { int64_t leftForVer; int32_t vgId; @@ -2876,7 +2894,8 @@ static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp } static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI16(buf, &pRsp->resMsgType); buf = taosDecodeFixedI32(buf, &pRsp->metaRspLen); buf = taosDecodeBinary(buf, &pRsp->metaRsp, pRsp->metaRspLen); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 488bc6346e..a24eda33bb 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -131,6 +131,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "create-stream", SCMCreateStreamReq, SCMCreateStreamRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "alter-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "drop-stream", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_RECOVER_STREAM, "recover-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_INDEX, "create-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_INDEX, "drop-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_INDEX, "get-index", NULL, NULL) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index dd64c5bf71..8b0a836ad2 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -192,6 +192,8 @@ int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner); int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem); +int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer); + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 071c539ff3..d03fab0ebc 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -34,6 +34,10 @@ typedef struct SStreamTask SStreamTask; enum { TASK_STATUS__NORMAL = 0, TASK_STATUS__DROPPING, + TASK_STATUS__FAIL, + TASK_STATUS__STOP, + TASK_STATUS__PREPARE_RECOVER, + TASK_STATUS__RECOVERING, }; enum { @@ -72,6 +76,7 @@ typedef struct { int8_t type; int32_t srcVgId; + int32_t childId; int64_t sourceVer; SArray* blocks; // SArray @@ -222,6 +227,8 @@ typedef struct { int32_t nodeId; int32_t childId; int32_t taskId; + int64_t checkpointVer; + int64_t processedVer; SEpSet epSet; } SStreamChildEpInfo; @@ -232,6 +239,7 @@ typedef struct SStreamTask { int8_t execType; int8_t sinkType; int8_t dispatchType; + int8_t isStreamDistributed; int16_t dispatchMsgType; int8_t taskStatus; @@ -242,6 +250,13 @@ typedef struct SStreamTask { int32_t nodeId; SEpSet epSet; + // used for semi or single task, + // while final task should have processedVer for each child + int64_t recoverSnapVer; + int64_t startVer; + int64_t checkpointVer; + int64_t processedVer; + // children info SArray* childEpInfo; // SArray @@ -316,12 +331,12 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem } else if (pItem->type == STREAM_INPUT__CHECKPOINT) { taosWriteQitem(pTask->inputQueue->queue, pItem); // qStreamInput(pTask->exec.executor, pItem); - } else if (pItem->type == STREAM_INPUT__TRIGGER) { + } else if (pItem->type == STREAM_INPUT__GET_RES) { taosWriteQitem(pTask->inputQueue->queue, pItem); // qStreamInput(pTask->exec.executor, pItem); } - if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { + if (pItem->type != STREAM_INPUT__GET_RES && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE, TASK_TRIGGER_STATUS__ACTIVE); } @@ -420,6 +435,36 @@ typedef struct { int8_t inputStatus; } SStreamTaskRecoverRsp; +int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq); +int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq); + +int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp); +int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pRsp); + +typedef struct { + int64_t streamId; + int32_t taskId; +} SMStreamTaskRecoverReq; + +typedef struct { + int64_t streamId; + int32_t taskId; +} SMStreamTaskRecoverRsp; + +int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq); +int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq); + +int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp); +int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pRsp); + +typedef struct { + int64_t streamId; +} SPStreamTaskRecoverReq; + +typedef struct { + int8_t reserved; +} SPStreamTaskRecoverRsp; + int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9ebfa78b80..788f2cdfba 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4823,6 +4823,35 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq * return 0; } +int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMRecoverStreamReq(void *buf, int32_t bufLen, SMRecoverStreamReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->ast); @@ -4945,8 +4974,8 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { if (tEncodeTag(pCoder, (const STag *)pReq->ctb.pTag) < 0) return -1; int32_t len = taosArrayGetSize(pReq->ctb.tagName); if (tEncodeI32(pCoder, len) < 0) return -1; - for (int32_t i = 0; i < len; i++){ - char* name = taosArrayGet(pReq->ctb.tagName, i); + for (int32_t i = 0; i < len; i++) { + char *name = taosArrayGet(pReq->ctb.tagName, i); if (tEncodeCStr(pCoder, name) < 0) return -1; } } else if (pReq->type == TSDB_NORMAL_TABLE) { @@ -4982,9 +5011,9 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { int32_t len = 0; if (tDecodeI32(pCoder, &len) < 0) return -1; pReq->ctb.tagName = taosArrayInit(len, TSDB_COL_NAME_LEN); - if(pReq->ctb.tagName == NULL) return -1; - for (int32_t i = 0; i < len; i++){ - char name[TSDB_COL_NAME_LEN] = {0}; + if (pReq->ctb.tagName == NULL) return -1; + for (int32_t i = 0; i < len; i++) { + char name[TSDB_COL_NAME_LEN] = {0}; char *tmp = NULL; if (tDecodeCStr(pCoder, &tmp) < 0) return -1; strcpy(name, tmp); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 7ac991451e..5cf6e26ee8 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -559,6 +559,7 @@ typedef struct { // info int64_t uid; int8_t status; + int8_t isDistributed; // config int8_t igExpired; int8_t trigger; @@ -586,6 +587,23 @@ typedef struct { int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); int32_t tDecodeSStreamObj(SDecoder* pDecoder, SStreamObj* pObj); +typedef struct { + char streamName[TSDB_STREAM_FNAME_LEN]; + int64_t uid; + int64_t streamUid; + SArray* childInfo; // SArray +} SStreamCheckpointObj; + +#if 0 +typedef struct { + int64_t uid; + int64_t streamId; + int8_t isDistributed; + int8_t status; + int8_t stage; +} SStreamRecoverObj; +#endif + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 1497bba11c..6e367fbe24 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -27,6 +27,7 @@ typedef enum { TRANS_STOP_FUNC_TEST = 2, TRANS_START_FUNC_MQ_REB = 3, TRANS_STOP_FUNC_MQ_REB = 4, + TRANS_FUNC_RECOVER_STREAM_STEP_NEXT = 5, } ETrnFunc; typedef enum { diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 727b20ef8a..c26424e049 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -27,6 +27,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->isDistributed) < 0) return -1; if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1; if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; @@ -72,6 +73,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->isDistributed) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index aa6e7192fc..ec0ea90f46 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -319,6 +319,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); ASSERT(totLevel <= 2); pStream->tasks = taosArrayInit(totLevel, sizeof(void*)); + pStream->isDistributed = totLevel == 2; bool hasExtraSink = false; bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5777df4fa6..8248a26c28 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -36,7 +36,7 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); -/*static int32_t mndProcessDropStreamInRsp(SRpcMsg *pRsp);*/ +static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq); static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -55,6 +55,7 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq); + mndSetMsgHandle(pMnode, TDMT_MND_RECOVER_STREAM, mndProcessRecoverStreamReq); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp); @@ -672,6 +673,69 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } +static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SStreamObj *pStream = NULL; + /*SDbObj *pDb = NULL;*/ + /*SUserObj *pUser = NULL;*/ + + SMRecoverStreamReq recoverReq = {0}; + if (tDeserializeSMRecoverStreamReq(pReq->pCont, pReq->contLen, &recoverReq) < 0) { + ASSERT(0); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + pStream = mndAcquireStream(pMnode, recoverReq.name); + + if (pStream == NULL) { + if (recoverReq.igNotExists) { + mDebug("stream:%s, not exist, ignore not exist is set", recoverReq.name); + sdbRelease(pMnode->pSdb, pStream); + return 0; + } else { + terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; + return -1; + } + } + + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { + return -1; + } + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); + if (pTrans == NULL) { + mError("stream:%s, failed to recover since %s", recoverReq.name, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + return -1; + } + mDebug("trans:%d, used to drop stream:%s", pTrans->id, recoverReq.name); + + // broadcast to recover all tasks + if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { + mError("stream:%s, failed to recover task since %s", recoverReq.name, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + return -1; + } + + // update stream status + if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + sdbRelease(pMnode->pSdb, pStream); + return -1; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare recover stream trans since %s", pTrans->id, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + return -1; + } + + sdbRelease(pMnode->pSdb, pStream); + + return TSDB_CODE_ACTION_IN_PROGRESS; +} + int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index be56d901de..4f2486d551 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -137,17 +137,18 @@ typedef enum { SDB_USER = 7, SDB_AUTH = 8, SDB_ACCT = 9, - SDB_STREAM = 10, - SDB_OFFSET = 11, - SDB_SUBSCRIBE = 12, - SDB_CONSUMER = 13, - SDB_TOPIC = 14, - SDB_VGROUP = 15, - SDB_SMA = 16, - SDB_STB = 17, - SDB_DB = 18, - SDB_FUNC = 19, - SDB_MAX = 20 + SDB_STREAM_CK = 10, + SDB_STREAM = 11, + SDB_OFFSET = 12, + SDB_SUBSCRIBE = 13, + SDB_CONSUMER = 14, + SDB_TOPIC = 15, + SDB_VGROUP = 16, + SDB_SMA = 17, + SDB_STB = 18, + SDB_DB = 19, + SDB_FUNC = 20, + SDB_MAX = 21 } ESdbType; typedef struct SSdbRaw { @@ -309,7 +310,7 @@ void sdbRelease(SSdb *pSdb, void *pObj); * @return void* The next iterator of the table. */ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj); -void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) ; +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status); /** * @brief Cancel a traversal diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1b11941849..99845713a9 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -139,6 +139,12 @@ typedef struct STaskIdInfo { char* str; } STaskIdInfo; +enum { + STREAM_RECOVER_STEP__NONE = 0, + STREAM_RECOVER_STEP__PREPARE, + STREAM_RECOVER_STEP__SCAN, +}; + typedef struct { //TODO remove prepareStatus STqOffsetVal prepareStatus; // for tmq @@ -147,6 +153,10 @@ typedef struct { SSDataBlock* pullOverBlk; // for streaming SWalFilterCond cond; int64_t lastScanUid; + int8_t recoverStep; + SQueryTableDataCond tableCond; + int64_t recoverStartVer; + int64_t recoverEndVer; } SStreamTaskInfo; typedef struct SExecTaskInfo { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index d910b8be34..37d2521e5d 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -261,6 +261,15 @@ int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) { } #endif +int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + pTaskInfo->streamInfo.recoverStartVer = startVer; + pTaskInfo->streamInfo.recoverEndVer = endVer; + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE; + return 0; +} + void* qExtractReaderFromStreamScanner(void* scanner) { SStreamScanInfo* pInfo = scanner; return (void*)pInfo->tqReader; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index f5b6b0c7a6..dcb34c621d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -517,14 +517,12 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); - int32_t code; - if (pInfo->dataReader != NULL) { - code = tsdbReaderReset(pInfo->dataReader, &pInfo->cond); - ASSERT(code == 0); - } else { - code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, - GET_TASKID(pTaskInfo)); - ASSERT(code == 0); + tsdbReaderClose(pInfo->dataReader); + + int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, + GET_TASKID(pTaskInfo)); + if (code != 0) { + // TODO } } @@ -1264,6 +1262,24 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return NULL; } + if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE) { + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + memcpy(&pTSInfo->cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond)); + pTSInfo->scanTimes = 0; + pTSInfo->currentGroupId = -1; + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN; + } + + if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN) { + SSDataBlock* pBlock = doTableScan(pInfo->pTableScanOp); + if (pBlock != NULL) { + return pBlock; + } + // TODO fill in bloom filter + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; + return NULL; + } + size_t total = taosArrayGetSize(pInfo->pBlockLists); // TODO: refactor if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { @@ -1556,6 +1572,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys goto _error; } taosArrayDestroy(tableIdList); + memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->cond, sizeof(SQueryTableDataCond)); } // create the pseduo columns info diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 566d9209a8..2a96db3bfc 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -57,7 +57,7 @@ void streamTriggerByTimer(void* param, void* tmrId) { if (atomic_load_8(&pTask->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { SStreamTrigger* trigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM); if (trigger == NULL) return; - trigger->type = STREAM_INPUT__TRIGGER; + trigger->type = STREAM_INPUT__GET_RES; trigger->pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (trigger->pBlock == NULL) { taosFreeQitem(trigger); @@ -183,8 +183,11 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S // 2.1. idle: exec // 2.2. executing: return // 2.3. closing: keep trying +#if 0 if (pTask->execType != TASK_EXEC__NONE) { - streamExec(pTask, pTask->pMsgCb); +#endif + streamExec(pTask, pTask->pMsgCb); +#if 0 } else { ASSERT(pTask->sinkType != TASK_SINK__NONE); while (1) { @@ -195,11 +198,13 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S } } } +#endif // 3. handle output // 3.1 check and set status // 3.2 dispatch / sink if (pTask->dispatchType != TASK_DISPATCH__NONE) { + ASSERT(pTask->sinkType == TASK_SINK__NONE); streamDispatch(pTask, pTask->pMsgCb); } diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 8e16b23e56..dbf6350c93 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -112,7 +112,7 @@ int32_t streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { void streamFreeQitem(SStreamQueueItem* data) { int8_t type = data->type; - if (type == STREAM_INPUT__TRIGGER) { + if (type == STREAM_INPUT__GET_RES) { blockDataDestroy(((SStreamTrigger*)data)->pBlock); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 1e46a9622b..cffd519d2a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -20,7 +20,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) // set input SStreamQueueItem* pItem = (SStreamQueueItem*)data; - if (pItem->type == STREAM_INPUT__TRIGGER) { + if (pItem->type == STREAM_INPUT__GET_RES) { SStreamTrigger* pTrigger = (SStreamTrigger*)data; qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK, false); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { @@ -73,6 +73,15 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) return 0; } +static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock* pBlock) { + ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); + int32_t childId = pBlock->childId; + int64_t ver = pBlock->sourceVer; + SStreamChildEpInfo* pChildInfo = taosArrayGetP(pTask->childEpInfo, childId); + pChildInfo->processedVer = ver; + return 0; +} + static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { int32_t cnt = 0; void* data = NULL; @@ -84,14 +93,17 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { } if (data == NULL) { data = qItem; + if (qItem->type == STREAM_INPUT__DATA_BLOCK) { + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + } streamQueueProcessSuccess(pTask->inputQueue); - continue; } else { if (streamAppendQueueItem(data, qItem) < 0) { streamQueueProcessFail(pTask->inputQueue); break; } else { cnt++; + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ streamQueueProcessSuccess(pTask->inputQueue); taosArrayDestroy(((SStreamDataBlock*)qItem)->blocks); taosFreeQitem(qItem); @@ -106,6 +118,12 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { if (data == NULL) return pRes; + if (pTask->execType == TASK_EXEC__NONE) { + ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK); + streamTaskOutput(pTask, data); + return pRes; + } + qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); streamTaskExecImpl(pTask, data, pRes); qDebug("stream task %d exec end", pTask->taskId); @@ -125,6 +143,11 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { taosFreeQitem(qRes); return NULL; } + if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; + qRes->childId = pTask->selfChildId; + qRes->sourceVer = pSubmit->ver; + } /*streamQueueProcessSuccess(pTask->inputQueue);*/ pRes = taosArrayInit(0, sizeof(SSDataBlock)); } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c new file mode 100644 index 0000000000..4175f60446 --- /dev/null +++ b/source/libs/stream/src/streamRecover.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "streamInc.h" + +int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->inputStatus) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->inputStatus) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t streamProcessFailRecoverReq(SStreamTask* pTask, SMStreamTaskRecoverReq* pReq, SRpcMsg* pRsp) { + if (pTask->taskStatus != TASK_STATUS__FAIL) { + return 0; + } + + if (pTask->isStreamDistributed) { + if (pTask->isDataScan) { + pTask->taskStatus = TASK_STATUS__PREPARE_RECOVER; + } else if (pTask->execType != TASK_EXEC__NONE) { + pTask->taskStatus = TASK_STATUS__PREPARE_RECOVER; + bool hasCheckpoint = false; + int32_t childSz = taosArrayGetSize(pTask->childEpInfo); + for (int32_t i = 0; i < childSz; i++) { + SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i); + if (pEpInfo->checkpointVer == -1) { + hasCheckpoint = true; + break; + } + } + if (hasCheckpoint) { + // load from checkpoint + } else { + // recover child + } + } + } else { + if (pTask->isDataScan) { + if (pTask->checkpointVer != -1) { + // load from checkpoint + } else { + // reset stream query task info + // TODO get snapshot ver + pTask->recoverSnapVer = -1; + qStreamPrepareRecover(pTask->exec.executor, pTask->startVer, pTask->recoverSnapVer); + pTask->taskStatus = TASK_STATUS__RECOVERING; + } + } + } + + if (pTask->taskStatus == TASK_STATUS__RECOVERING) { + streamProcessRunReq(pTask); + } + return 0; +} diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 7488c009bd..2d15c31bf1 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -34,6 +34,7 @@ int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo) if (tEncodeI32(pEncoder, pInfo->taskId) < 0) return -1; if (tEncodeI32(pEncoder, pInfo->nodeId) < 0) return -1; if (tEncodeI32(pEncoder, pInfo->childId) < 0) return -1; + if (tEncodeI64(pEncoder, pInfo->processedVer) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1; return 0; } @@ -42,6 +43,7 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) { if (tDecodeI32(pDecoder, &pInfo->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pInfo->nodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pInfo->childId) < 0) return -1; + if (tDecodeI64(pDecoder, &pInfo->processedVer) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1; return 0; } From 7e45878581e0a06ca2c19eefc122cf11f8a026b9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 15 Jul 2022 10:03:18 +0000 Subject: [PATCH 057/124] more vnode snapshot --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 342 +++++++++++---------- 1 file changed, 182 insertions(+), 160 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index d9a67ce487..932cf9f9ae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -488,52 +488,42 @@ _err: return code; } -static int32_t tsdbSnapWriteTableDataAhead(STsdbSnapWriter* pWriter, TABLEID id) { +static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* pBlockIdx) { int32_t code = 0; - if (pWriter->pDataFReader == NULL) goto _exit; + code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL); + if (code) goto _err; - while (true) { - if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break; + // SBlockData + SBlock block; + tMapDataReset(&pWriter->mBlockW); + for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) { + tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock); - SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); - int32_t c = tTABLEIDCmprFn(pBlockIdx, &id); + if (block.last) { + code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); + if (code) goto _err; - if (c >= 0) break; - - pWriter->iBlockIdx++; - - code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL); - if (code) goto _err; - - SBlock block; - tMapDataReset(&pWriter->mBlockW); - for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) { - tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock); - - if (block.last) { - code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); - if (code) goto _err; - - tBlockReset(&block); - block.last = 1; - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, - pWriter->cmprAlg); - if (code) goto _err; - } - - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + tBlockReset(&block); + block.last = 1; + code = + tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, pWriter->cmprAlg); if (code) goto _err; } - SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; - code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, &blockIdx); + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); if (code) goto _err; + } - if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + // SBlock + SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; + code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, &blockIdx); + if (code) goto _err; + + // SBlockIdx + if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } _exit: @@ -543,14 +533,135 @@ _err: return code; } -static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) { +static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { + int32_t code = 0; + SBlockData* pBlockData = &pWriter->bData; + int32_t iRow = 0; + TSDBROW row; + TSDBROW* pRow = &row; + + // correct schema + code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData); + if (code) goto _err; + + // loop to merge + *pRow = tsdbRowFromBlockData(pBlockData, iRow); + while (true) { + if (pRow == NULL) break; + + if (pWriter->pBlockData) { + ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); + + int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow)); + + ASSERT(c); + + if (c < 0) { + code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + if (code) goto _err; + + iRow++; + if (iRow < pWriter->pBlockData->nRow) { + *pRow = tsdbRowFromBlockData(pBlockData, iRow); + } else { + pRow = NULL; + } + } else if (c > 0) { + code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL); + if (code) goto _err; + + pWriter->iRow++; + if (pWriter->iRow >= pWriter->pBlockData->nRow) { + pWriter->pBlockData = NULL; + } + } + } else { + TSDBKEY key = TSDBROW_KEY(pRow); + + while (true) { + if (pWriter->iBlock >= pWriter->mBlock.nItem) break; + + SBlock block; + int32_t c; + + tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); + + if (block.last) { + pWriter->pBlockData = &pWriter->bDataR; + + code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); + if (code) goto _err; + pWriter->iRow = 0; + + pWriter->iBlock++; + break; + } + + c = tsdbKeyCmprFn(&block.maxKey, &key); + + ASSERT(c); + + if (c > 0) break; + + if (pWriter->bDataW.nRow) { + pWriter->blockW.last = 0; + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + &pWriter->blockW, pWriter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + if (code) goto _err; + + tBlockReset(&pWriter->blockW); + tBlockDataClearData(&pWriter->bDataW); + } + + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + if (code) goto _err; + + pWriter->iBlock++; + } + + if (pWriter->pBlockData) continue; + + code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + if (code) goto _err; + + iRow++; + if (iRow < pBlockData->nRow) { + *pRow = tsdbRowFromBlockData(pBlockData, iRow); + } else { + pRow = NULL; + } + } + + _check_write: + if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue; + + _write_block: + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdx, &pWriter->blockW, + pWriter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + if (code) goto _err; + + tBlockReset(&pWriter->blockW); + tBlockDataClearData(&pWriter->bDataW); + } + + return code; + +_err: + return code; +} + +static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { int32_t code = 0; SBlockData* pBlockData = &pWriter->bData; TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); TSDBKEY keyLast = tBlockDataLastKey(pBlockData); - // TABLE ==================================== - // end last table write if should if (pWriter->pBlockIdxW) { int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id); @@ -569,23 +680,33 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) { // start new table data write if need if (pWriter->pBlockIdxW == NULL) { // write table data ahead - code = tsdbSnapWriteTableDataAhead(pWriter, id); - if (code) goto _err; + while (true) { + if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break; + + SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); + int32_t c = tTABLEIDCmprFn(pBlockIdx, &id); + + if (c >= 0) break; + + code = tsdbSnapMoveWriteTableData(pWriter, pBlockIdx); + if (code) goto _err; + + pWriter->iBlockIdx++; + } // reader if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { ASSERT(pWriter->pDataFReader); - pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlock); - int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id); - if (c) { - ASSERT(c > 0); - pWriter->pBlockIdx = NULL; - } else { + SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlock); + int32_t c = tTABLEIDCmprFn(pBlockIdx, &id); + + ASSERT(c >= 0); + + if (c == 0) { + pWriter->pBlockIdx = pBlockIdx; pWriter->iBlockIdx++; } - } else { - pWriter->pBlockIdx = NULL; } if (pWriter->pBlockIdx) { @@ -611,107 +732,8 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) { ASSERT(pWriter->pBlockIdxW && pWriter->pBlockIdxW->suid == id.suid && pWriter->pBlockIdxW->uid == id.uid); ASSERT(pWriter->pBlockIdx == NULL || (pWriter->pBlockIdx->suid == id.suid && pWriter->pBlockIdx->uid == id.uid)); - // BLOCK ==================================== - int32_t iRow = 0; - TSDBROW* pRow = &tsdbRowFromBlockData(pBlockData, iRow); - while (true) { - if (pRow == NULL) break; - - if (pWriter->pBlockData) { - ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); - - int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow)); - - ASSERT(c); - - if (c < 0) { - code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); - if (code) goto _err; - - iRow++; - if (iRow < pWriter->pBlockData->nRow) { - pRow = &tsdbRowFromBlockData(pBlockData, iRow); - } else { - pRow = NULL; - } - } else if (c > 0) { - code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL); - if (code) goto _err; - - pWriter->iRow++; - if (pWriter->iRow >= pWriter->pBlockData->nRow) { - pWriter->pBlockData = NULL; - } - } - } else { - SBlock block; - - if (pWriter->iBlock < pWriter->mBlock.nItem) { - tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); - int32_t c; - - c = tsdbKeyCmprFn(&block.maxKey, &TSDBROW_KEY(pRow)); - - ASSERT(c); - - if (c < 0) { - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); - if (code) goto _err; - - pWriter->iBlock++; - continue; - } - - c = tsdbKeyCmprFn(&block.minKey, &TSDBROW_KEY(pRow)); - - ASSERT(c); - - if (c > 0) { - code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); - if (code) goto _err; - - iRow++; - if (iRow < pWriter->pBlockData->nRow) { - pRow = &tsdbRowFromBlockData(pBlockData, iRow); - } else { - pRow = NULL; - } - - goto _check_write; - } - - pWriter->pBlockData = &pWriter->bDataR; - code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); - if (code) goto _err; - - pWriter->iRow = 0; - } - - code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); - if (code) goto _err; - - iRow++; - if (iRow < pWriter->pBlockData->nRow) { - pRow = &tsdbRowFromBlockData(pBlockData, iRow); - } else { - pRow = NULL; - } - } - - _check_write: - if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue; - - _write_block: - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdx, &pWriter->blockW, - pWriter->cmprAlg); - if (code) goto _err; - - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); - if (code) goto _err; - - tBlockReset(&pWriter->blockW); - tBlockDataClearData(&pWriter->bDataW); - } + code = tsdbSnapWriteTableDataImpl(pWriter); + if (code) goto _err; _exit: return code; @@ -723,11 +745,10 @@ _err: } static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - STsdb* pTsdb = pWriter->pTsdb; - SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; - TABLEID id = *(TABLEID*)(&pHdr[1]); - int64_t n; + int32_t code = 0; + STsdb* pTsdb = pWriter->pTsdb; + TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); + int64_t n; // decode SBlockData* pBlockData = &pWriter->bData; @@ -742,13 +763,13 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision)); if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) { // end last file data write if need - code = tsdbSnapWriteDataEnd(pWriter); // todo + code = tsdbSnapWriteDataEnd(pWriter); if (code) goto _err; pWriter->fid = fid; // read - SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ); // todo: check nState is valid + SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ); if (pSet) { code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet); if (code) goto _err; @@ -763,8 +784,9 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 pWriter->pBlockIdx = NULL; tMapDataReset(&pWriter->mBlock); pWriter->iBlock = 0; - tBlockDataReset(&pWriter->bDataR); + pWriter->pBlockData = NULL; pWriter->iRow = 0; + tBlockDataReset(&pWriter->bDataR); // write SDFileSet wSet; @@ -789,12 +811,12 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 if (code) goto _err; taosArrayClear(pWriter->aBlockIdxW); - pWriter->pBlockIdxW = NULL; tMapDataReset(&pWriter->mBlockW); + pWriter->pBlockIdxW = NULL; tBlockDataReset(&pWriter->bDataW); } - code = tsdbSnapWriteDataImpl(pWriter, id); + code = tsdbSnapWriteTableData(pWriter, id); if (code) goto _err; tsdbInfo("vgId:%d vnode snapshot tsdb write data, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d", From b86ec034dbea631c54542b6556a9a6f3903900d4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 15 Jul 2022 18:05:51 +0800 Subject: [PATCH 058/124] refactor(sync): modify elect/heartbeat timer --- source/dnode/vnode/src/vnd/vnodeSync.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 38b9df52b8..81177fd5d8 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -584,8 +584,8 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { } setPingTimerMS(pVnode->sync, 5000); - setElectTimerMS(pVnode->sync, 500); - setHeartbeatTimerMS(pVnode->sync, 100); + setElectTimerMS(pVnode->sync, 1300); + setHeartbeatTimerMS(pVnode->sync, 900); return 0; } From f91d6dca9e1370d9ae6a7dd2a41e90ac7e17832e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Jul 2022 18:07:45 +0800 Subject: [PATCH 059/124] add more debug --- source/client/src/clientImpl.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5e620d1060..ba75e3b85f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1276,7 +1276,12 @@ int32_t doProcessMsgFromServer(void* param) { assert(pMsg->info.ahandle != NULL); STscObj* pTscObj = NULL; - tscDebug("processMsgFromServer message: %s, code: %s", TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code)); + STraceId* trace = &pMsg->info.traceId; + char tbuf[40] = {0}; + TRACE_TO_STR(trace, tbuf); + + tscDebug("processMsgFromServer message: %s, code: %s, gtid: %s", TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), + tbuf); if (pSendInfo->requestObjRefId != 0) { SRequestObj* pRequest = (SRequestObj*)taosAcquireRef(clientReqRefPool, pSendInfo->requestObjRefId); From d02a6090ef755d7322dbaba43ff8aa9b292c3d1d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 15 Jul 2022 18:08:43 +0800 Subject: [PATCH 060/124] fix: the follower of mnode handles transactions incorrectly --- source/dnode/mnode/impl/src/mndSync.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 3c3864b620..418bb1be0a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -56,23 +56,24 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM sdbSetApplyInfo(pMnode->pSdb, cbMeta.index, cbMeta.term, cbMeta.lastConfigIndex); } - if (pMgmt->transId == transId && transId != 0) { + if (transId <= 0) { + mError("trans:%d, invalid commit msg", transId); + } else if (transId == pMgmt->transId) { if (pMgmt->errCode != 0) { mError("trans:%d, failed to propose since %s", transId, tstrerror(pMgmt->errCode)); } pMgmt->transId = 0; tsem_post(&pMgmt->syncSem); } else { -#if 1 - mError("trans:%d, invalid commit msg since trandId not match with %d", transId, pMgmt->transId); -#else STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { + mDebug("trans:%d, execute in mnode which not leader", transId); mndTransExecute(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans); + // sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA); + } else { + mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); } - // sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA); -#endif } } From d754cd14f4af50a982a4d36c711dab4ee9d03bcb Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 18:09:01 +0800 Subject: [PATCH 061/124] doc: fix wrong title --- docs/zh/10-cluster/03-high-availability.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/10-cluster/03-high-availability.md b/docs/zh/10-cluster/03-high-availability.md index 1ec413437f..b8f3b25edf 100644 --- a/docs/zh/10-cluster/03-high-availability.md +++ b/docs/zh/10-cluster/03-high-availability.md @@ -1,5 +1,5 @@ --- -title: 高可用与负载均衡 +title: 高可用 --- ## Vnode 的高可用性 From 0048bd8cb634b027d1c504196dcd901791cc350b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 15 Jul 2022 18:09:27 +0800 Subject: [PATCH 062/124] fix: fix redirect issue --- source/client/src/clientImpl.c | 2 +- source/libs/qworker/src/qwDbg.c | 4 ++-- source/libs/qworker/src/qworker.c | 6 ++---- source/libs/scheduler/inc/schInt.h | 3 ++- source/libs/scheduler/src/schRemote.c | 2 +- source/libs/scheduler/src/schTask.c | 6 ++++-- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 5e620d1060..90d16d17a5 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1276,7 +1276,7 @@ int32_t doProcessMsgFromServer(void* param) { assert(pMsg->info.ahandle != NULL); STscObj* pTscObj = NULL; - tscDebug("processMsgFromServer message: %s, code: %s", TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code)); + tscDebug("processMsgFromServer handle %p, message: %s, code: %s", pMsg->info.handle, TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code)); if (pSendInfo->requestObjRefId != 0) { SRequestObj* pRequest = (SRequestObj*)taosAcquireRef(clientReqRefPool, pSendInfo->requestObjRefId); diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index cd54c5e5f9..7906617a3f 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -188,8 +188,8 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t msgType) { return; } - //FETCH OR QUERY USE DIFFERENT CONNINFO - qwBuildAndSendErrorRsp(msgType + 1, ctx->dataConnInfo, TSDB_CODE_RPC_BROKEN_LINK); + SRpcHandleInfo *pConn = ((msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) ? &ctx->dataConnInfo : &ctx->ctrlConnInfo); + qwBuildAndSendErrorRsp(msgType + 1, pConn, TSDB_CODE_RPC_BROKEN_LINK); qwDropTask(QW_FPARAMS()); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index a671585014..c763645a07 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -426,7 +426,7 @@ _return: qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC); } - if (QW_PHASE_POST_QUERY == phase) { + if (QW_PHASE_POST_QUERY == phase && ctx) { ctx->queryRsped = true; qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); @@ -730,11 +730,9 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { if (QW_QUERY_RUNNING(ctx)) { QW_ERR_JRET(qwKillTaskHandle(ctx)); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP); - } else if (QW_GET_PHASE(ctx) > 0) { + } else { QW_ERR_JRET(qwDropTask(QW_FPARAMS())); dropped = true; - } else { - // task not started } if (!dropped) { diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index bc0270635d..4979a41f17 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -58,7 +58,7 @@ typedef enum { #define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000 #define SCH_MAX_TASK_TIMEOUT_USEC 60000000 -#define SCH_MAX_CANDIDATE_EP_NUM TSDB_MAX_REPLICA +#define SCH_MAX_CANDIDATE_EP_NUM (TSDB_MAX_REPLICA + 100) @@ -211,6 +211,7 @@ typedef struct SSchTask { int32_t maxExecTimes; // task max exec times int32_t maxRetryTimes; // task max retry times int32_t retryTimes; // task retry times + bool waitRetry; // wait for retry int32_t execId; // task current execute index SSchLevel *level; // level SRWLatch planLock; // task update plan lock diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index ec07ee85fd..41d9f46a87 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1010,7 +1010,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); persistHandle = true; - //SCH_SET_TASK_HANDLE(pTask, rpcAllocHandle()); + SCH_SET_TASK_HANDLE(pTask, rpcAllocHandle()); break; } case TDMT_SCH_FETCH: diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 9483ecd6eb..a46b293965 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -125,8 +125,8 @@ int32_t schDropTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_ SCH_TASK_DLOG("execId %d removed from execNodeList", execId); } - if (execId != pTask->execId) { // ignore it - SCH_TASK_DLOG("execId %d is not current execId %d", execId, pTask->execId); + if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it + SCH_TASK_DLOG("execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry); SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); } @@ -335,6 +335,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 return TSDB_CODE_SUCCESS; } + pTask->waitRetry = true; schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); SCH_ERR_JRET(schRemoveTaskFromExecList(pJob, pTask)); @@ -790,6 +791,7 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); pTask->execId++; pTask->retryTimes++; + pTask->waitRetry = false; SCH_TASK_DLOG("start to launch task, execId %d, retry %d", pTask->execId, pTask->retryTimes); From b1826ef736a52071b07e30ca71a299b64d19d738 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 18:15:06 +0800 Subject: [PATCH 063/124] fix(query): fix a typo --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a921e4a70b..12b796c5ca 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2682,7 +2682,7 @@ static void saveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFun } } -static void doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t currentTs, int32_t type, char* pData) { +static void doSaveCurrentVal(SqlFunctionCtx* pCtx, int32_t rowIndex, int64_t currentTs, int32_t type, char* pData) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo); From 72e4a8f5f5a14b256324336e7f520985b7b33af0 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Fri, 15 Jul 2022 18:24:16 +0800 Subject: [PATCH 064/124] doc: refine according to comments --- docs/zh/10-cluster/03-high-availability.md | 4 ++-- docs/zh/10-cluster/04-load-balance.md | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/zh/10-cluster/03-high-availability.md b/docs/zh/10-cluster/03-high-availability.md index b8f3b25edf..36f9300366 100644 --- a/docs/zh/10-cluster/03-high-availability.md +++ b/docs/zh/10-cluster/03-high-availability.md @@ -20,7 +20,7 @@ CREATE DATABASE demo replica 3; ## Mnode 的高可用性 -TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管理的，为保证 mnode 的高可用，可以配置多个 mnode 副本，在集群启动时只有一个 mnode，用户可以通过 `create mnode`来增加新的mnode。用户可以通过该命令自主决定哪几个 dnode 会承担 mnode 的角色。为保证元数据的强一致性，在有多个 mnode 时，mnode 副本之间是通过同步的方式进行数据复制的。 +TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管理的，为保证 mnode 的高可用，可以配置多个 mnode 副本，在集群启动时只有一个 mnode，用户可以通过 `create mnode` 来增加新的 mnode。用户可以通过该命令自主决定哪几个 dnode 会承担 mnode 的角色。为保证元数据的强一致性，在有多个 mnode 时，mnode 副本之间是通过同步的方式进行数据复制的。 一个集群有多个数据节点 dnode，但一个 dnode 至多运行一个 mnode 实例。用户可通过 CLI 程序 taos，在 TDengine 的 console 里，执行如下命令： @@ -30,4 +30,4 @@ SHOW MNODES; 来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（leader, follower, candidate）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。 -在 TDengine 3.0 及以后的版本中，数据同步采用 RAFT 协议，所以 mnode 的数量应该被设置为 1 个或者 3 个。 \ No newline at end of file +在 TDengine 3.0 及以后的版本中，数据同步采用 RAFT 协议，所以 mnode 的数量应该被设置为 1 个或者 3 个。 diff --git a/docs/zh/10-cluster/04-load-balance.md b/docs/zh/10-cluster/04-load-balance.md index 9b582f417d..c1a72cf006 100644 --- a/docs/zh/10-cluster/04-load-balance.md +++ b/docs/zh/10-cluster/04-load-balance.md @@ -14,4 +14,6 @@ create database db0 vgroups 100; 此外，对于任意数据库的 vgroups，TDengine 都是尽可能将其均衡分散在多个 dnode 上。在多副本情况下（replica 3），这种均衡分布尤其复杂，TDengine 的分布策略会尽量避免任意一个 dnode 成为写入的瓶颈。 -通过以上措施可以最大限度地在整个 TDengine 集群中实现负载均衡，负载均衡也能反过来提升系统总的数据处理能力。 \ No newline at end of file +通过以上措施可以最大限度地在整个 TDengine 集群中实现负载均衡，负载均衡也能反过来提升系统总的数据处理能力。 + +在初始的负载均衡建立起来之后，如果由于删库、删表等动作，特别是删库动作会导致属于它的 vnode 都被删除，这有可能会造成一定程度的负载失衡，在后续版本中会提供重新平衡的方法。但如果有新的数据库建立，TDengine 也能够一定程度自我再平衡而无须人工干预。 From 9273caf5b7b3fe2c7cc6cc72def24e99240df342 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 15 Jul 2022 18:47:16 +0800 Subject: [PATCH 065/124] feat: update taostools for3.0 (#14964) * feat: update taos-tools for 3.0 [TD-14141] * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index bd496f76b6..d807c3ffa6 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit bd496f76b64931c66da2f8b0f24143a98a881cde +Subproject commit d807c3ffa6f750f7765e102917d1328cadf21c13 From 684b550f5e23170ba07fed1981f18a5ec2254931 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 15 Jul 2022 20:17:21 +0800 Subject: [PATCH 066/124] test: add test case to index --- .../2-query/json_tag_large_tables.py | 672 ++++++++++++++++++ 1 file changed, 672 insertions(+) create mode 100644 tests/system-test/2-query/json_tag_large_tables.py diff --git a/tests/system-test/2-query/json_tag_large_tables.py b/tests/system-test/2-query/json_tag_large_tables.py new file mode 100644 index 0000000000..fc41858580 --- /dev/null +++ b/tests/system-test/2-query/json_tag_large_tables.py @@ -0,0 +1,672 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, db_test.stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import imp +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import json +import os + + +class TDTestCase: + def caseDescription(self): + ''' + Json tag test case, include create table with json tag, select json tag and query with json tag in where condition, besides, include json tag in group by/order by/join/subquery. + case1: [TD-12452] fix error if json tag is NULL + case2: [TD-12389] describe child table, tag length error if the tag is json tag + ''' + return + + def init(self, conn, logSql): + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + # os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + def run(self): + # tdSql.prepare() + tdSql.execute('drop database if exists db') + tdSql.execute('create database db vgroups 1') + tdSql.execute('use db') + print("============== STEP 1 ===== prepare data & validate json string") + i = 0 + tdSql.execute("create table if not exists jsons1(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + while i <= 100000: + f = "insert into jsons1_{} using jsons1 tags('{\"tag1\":\"fff\",\"tag2\":{}, \"tag3\":true}') values(1591060618000, 1, false, 'json1', '你是') (1591060608000, 23, true, '等等', 'json')".format + sql = f(i, i) + tdSql.execute(sql) + i = i + 1 + + # test duplicate key using the first one. elimate empty key + #tdSql.execute("CREATE TABLE if not exists jsons1_8 using jsons1 tags('{\"tag1\":null, \"tag1\":true, \"tag1\":45, \"1tag$\":2, \" \":90, \"\":32}')") tdSql.query("select jtag from jsons1_8") tdSql.checkRows(0); + #tdSql.query("select ts,jtag from jsons1 order by ts limit 2,3") + #tdSql.checkData(0, 0, '2020-06-02 09:17:08.000') + #tdSql.checkData(0, 1, '{"tag1":5,"tag2":"beijing"}') + #tdSql.checkData(1, 0, '2020-06-02 09:17:48.000') + #tdSql.checkData(1, 1, '{"tag1":false,"tag2":"beijing"}') + #tdSql.checkData(2, 0, '2020-06-02 09:18:48.000') + #tdSql.checkData(2, 1, '{"tag1":null,"tag2":"shanghai","tag3":"hello"}') + + #tdSql.query("select ts,jtag->'tag1' from jsons1 order by ts limit 2,3") + #tdSql.checkData(0, 0, '2020-06-02 09:17:08.000') + #tdSql.checkData(0, 1, '5.000000000') + #tdSql.checkData(1, 0, '2020-06-02 09:17:48.000') + #tdSql.checkData(1, 1, 'false') + #tdSql.checkData(2, 0, '2020-06-02 09:18:48.000') + #tdSql.checkData(2, 1, 'null') + + ## test empty json string, save as jtag is NULL + #tdSql.execute("insert into jsons1_9 using jsons1 tags('\t') values (1591062328000, 24, NULL, '你就会', '2sdw')") + #tdSql.execute("CREATE TABLE if not exists jsons1_10 using jsons1 tags('')") + #tdSql.execute("CREATE TABLE if not exists jsons1_11 using jsons1 tags(' ')") + #tdSql.execute("CREATE TABLE if not exists jsons1_12 using jsons1 tags('{}')") + #tdSql.execute("CREATE TABLE if not exists jsons1_13 using jsons1 tags('null')") + + ## test invalidate json + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('\"efwewf\"')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('3333')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags(76)") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags(hell)") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('33.33')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('false')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('[1,true]')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{222}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"fe\"}')") + + ## test invalidate json key, key must can be printed assic char + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":[1,true]}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":{}}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"。loc\":\"fff\"}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"\t\":\"fff\"}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"试试\":\"fff\"}')") + + ## test invalidate json value, value number can not be inf,nan TD-12166 + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":1.8e308}')") + #tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":-1.8e308}')") + + ##test length limit + #char1= ''.join(['abcd']*64) + #char3= ''.join(['abcd']*1021) + #print(len(char3)) # 4084 + #tdSql.error("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s1\":5}')" % char1) # len(key)=257 + #tdSql.execute("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s\":5}')" % char1) # len(key)=256 + #tdSql.error("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TSSSS\":\"%s\"}')" % char3) # len(object)=4096 + #tdSql.execute("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TSSS\":\"%s\"}')" % char3) # len(object)=4095 + #tdSql.execute("drop table if exists jsons1_15") + #tdSql.execute("drop table if exists jsons1_16") + + #print("============== STEP 2 ===== alter table json tag") + #tdSql.error("ALTER STABLE jsons1 add tag tag2 nchar(20)") + #tdSql.error("ALTER STABLE jsons1 drop tag jtag") + #tdSql.error("ALTER TABLE jsons1 MODIFY TAG jtag nchar(128)") + + #tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag='{\"tag1\":\"femail\",\"tag2\":35,\"tag3\":true}'") + #tdSql.query("select jtag from jsons1_1") + #tdSql.checkData(0, 0, '{"tag1":"femail","tag2":35,"tag3":true}') + #tdSql.execute("ALTER TABLE jsons1 rename TAG jtag jtag_new") + #tdSql.execute("ALTER TABLE jsons1 rename TAG jtag_new jtag") + + #tdSql.execute("create table st(ts timestamp, i int) tags(t int)") + #tdSql.error("ALTER STABLE st add tag jtag json") + #tdSql.error("ALTER STABLE st add column jtag json") + + #print("============== STEP 3 ===== query table") + ## test error syntax + #tdSql.error("select * from jsons1 where jtag->tag1='beijing'") + #tdSql.error("select -> from jsons1") + #tdSql.error("select * from jsons1 where contains") + #tdSql.error("select * from jsons1 where jtag->") + #tdSql.error("select jtag->location from jsons1") + #tdSql.error("select jtag contains location from jsons1") + #tdSql.error("select * from jsons1 where jtag contains location") + #tdSql.query("select * from jsons1 where jtag contains''") + #tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'") + + ## test function error + #tdSql.error("select avg(jtag->'tag1') from jsons1") + #tdSql.error("select avg(jtag) from jsons1") + #tdSql.error("select min(jtag->'tag1') from jsons1") + #tdSql.error("select min(jtag) from jsons1") + #tdSql.error("select ceil(jtag->'tag1') from jsons1") + #tdSql.error("select ceil(jtag) from jsons1") + + + ##test scalar operation + #tdSql.query("select jtag contains 'tag1',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' not like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' match 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1' nmatch 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + #tdSql.query("select jtag->'tag1',jtag->'tag1'>='a' from jsons1 order by jtag->'tag1'") + #tdSql.checkRows(9) + + ## test select normal column + #tdSql.query("select dataint from jsons1 order by dataint") + #tdSql.checkRows(9) + #tdSql.checkData(1, 0, 1) + + ## test select json tag + #tdSql.query("select * from jsons1") + #tdSql.checkRows(9) + #tdSql.query("select jtag from jsons1") + #tdSql.checkRows(9) + #tdSql.query("select * from jsons1 where jtag is null") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag is not null") + #tdSql.checkRows(8) + + ## test jtag is NULL + #tdSql.query("select jtag from jsons1_9") + #tdSql.checkData(0, 0, None) + + ## test select json tag->'key', value is string + #tdSql.query("select jtag->'tag1' from jsons1_1") + #tdSql.checkData(0, 0, '"femail"') + #tdSql.query("select jtag->'tag2' from jsons1_6") + #tdSql.checkData(0, 0, '""') + ## test select json tag->'key', value is int + #tdSql.query("select jtag->'tag2' from jsons1_1") + #tdSql.checkData(0, 0, "35.000000000") + ## test select json tag->'key', value is bool + #tdSql.query("select jtag->'tag3' from jsons1_1") + #tdSql.checkData(0, 0, "true") + ## test select json tag->'key', value is null + #tdSql.query("select jtag->'tag1' from jsons1_4") + #tdSql.checkData(0, 0, None) + ## test select json tag->'key', value is double + #tdSql.query("select jtag->'tag1' from jsons1_5") + #tdSql.checkData(0, 0, "1.232000000") + ## test select json tag->'key', key is not exist + #tdSql.query("select jtag->'tag10' from jsons1_4") + #tdSql.checkData(0, 0, None) + + #tdSql.query("select jtag->'tag1' from jsons1") + #tdSql.checkRows(9) + ## test header name + #res = tdSql.getColNameList("select jtag->'tag1' from jsons1") + #cname_list = [] + #cname_list.append("jtag->'tag1'") + #tdSql.checkColNameList(res, cname_list) + + + ## test where with json tag + #tdSql.query("select * from jsons1_1 where jtag is not null") + #tdSql.query("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'") + #tdSql.error("select * from jsons1 where jtag->'tag1'={}") + + ## test json error + #tdSql.error("select jtag + 1 from jsons1") + #tdSql.error("select jtag > 1 from jsons1") + #tdSql.error("select jtag like \"1\" from jsons1") + #tdSql.error("select jtag in (\"1\") from jsons1") + ##tdSql.error("select jtag from jsons1 where jtag > 1") + ##tdSql.error("select jtag from jsons1 where jtag like 'fsss'") + ##tdSql.error("select jtag from jsons1 where jtag in (1)") + + + ## where json value is string + #tdSql.query("select * from jsons1 where jtag->'tag2'='beijing'") + #tdSql.checkRows(2) + #tdSql.query("select dataint,tbname,jtag->'tag1',jtag from jsons1 where jtag->'tag2'='beijing' order by dataint") + #tdSql.checkRows(2) + #tdSql.checkData(0, 0, 2) + #tdSql.checkData(0, 1, 'jsons1_2') + #tdSql.checkData(0, 2, "5.000000000") + #tdSql.checkData(0, 3, '{"tag1":5,"tag2":"beijing"}') + #tdSql.checkData(1, 0, 3) + #tdSql.checkData(1, 1, 'jsons1_3') + #tdSql.checkData(1, 2, 'false') + + + #tdSql.query("select * from jsons1 where jtag->'tag1'='beijing'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'='收到货'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag2'>'beijing'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag2'>='beijing'") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag2'<'beijing'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag2'<='beijing'") + #tdSql.checkRows(4) + #tdSql.query("select * from jsons1 where jtag->'tag2'!='beijing'") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag2'=''") + #tdSql.checkRows(2) + + ## where json value is int + #tdSql.query("select * from jsons1 where jtag->'tag1'=5") + #tdSql.checkRows(1) + #tdSql.checkData(0, 1, 2) + #tdSql.query("select * from jsons1 where jtag->'tag1'=10") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'<54") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'<=11") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'>4") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'>=5") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=5") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=55") + #tdSql.checkRows(3) + + ## where json value is double + #tdSql.query("select * from jsons1 where jtag->'tag1'=1.232") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'<1.232") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'<=1.232") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'>1.23") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'>=1.232") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=1.232") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=3.232") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'/0=3") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'/5=1") + #tdSql.checkRows(1) + + ## where json value is bool + #tdSql.query("select * from jsons1 where jtag->'tag1'=true") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'!=false") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'>false") + #tdSql.checkRows(0) + + ## where json value is null + #tdSql.query("select * from jsons1 where jtag->'tag1'=null") + #tdSql.checkRows(0) + + ## where json key is null + #tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3") + #tdSql.checkRows(0) + + ## where json value is not exist + #tdSql.query("select * from jsons1 where jtag->'tag1' is null") + #tdSql.checkData(0, 0, 'jsons1_9') + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag4' is null") + #tdSql.checkRows(9) + #tdSql.query("select * from jsons1 where jtag->'tag3' is not null") + #tdSql.checkRows(3) + + ## test contains + #tdSql.query("select * from jsons1 where jtag contains 'tag1'") + #tdSql.checkRows(8) + #tdSql.query("select * from jsons1 where jtag contains 'tag3'") + #tdSql.checkRows(4) + #tdSql.query("select * from jsons1 where jtag contains 'tag_no_exist'") + #tdSql.checkRows(0) + + ## test json tag in where condition with and/or + #tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='beijing'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false or jtag->'tag2'='beijing'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='shanghai'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='shanghai'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=13 or jtag->'tag2'>35") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1'=13 or jtag->'tag2'>35") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag1' is not null and jtag contains 'tag3'") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1'='femail' and jtag contains 'tag3'") + #tdSql.checkRows(2) + + + ## test with between and + #tdSql.query("select * from jsons1 where jtag->'tag1' between 1 and 30") + #tdSql.checkRows(3) + #tdSql.query("select * from jsons1 where jtag->'tag1' between 'femail' and 'beijing'") + #tdSql.checkRows(2) + + ## test with tbname/normal column + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3' and dataint=3") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3' and dataint=23") + #tdSql.checkRows(1) + + + ## test where condition like + #tdSql.query("select * from jsons1 where jtag->'tag2' like 'bei%'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1' like 'fe%' and jtag->'tag2' is not null") + #tdSql.checkRows(2) + + ## test where condition in no support in + #tdSql.error("select * from jsons1 where jtag->'tag1' in ('beijing')") + + ## test where condition match/nmath + #tdSql.query("select * from jsons1 where jtag->'tag1' match 'ma'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1' match 'ma$'") + #tdSql.checkRows(0) + #tdSql.query("select * from jsons1 where jtag->'tag2' match 'jing$'") + #tdSql.checkRows(2) + #tdSql.query("select * from jsons1 where jtag->'tag1' match '收到'") + #tdSql.checkRows(1) + #tdSql.query("select * from jsons1 where jtag->'tag1' nmatch 'ma'") + #tdSql.checkRows(1) + + ## test distinct + #tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')") + #tdSql.query("select distinct jtag->'tag1' from jsons1") + #tdSql.checkRows(7) + ## tdSql.query("select distinct jtag from jsons1") + ## tdSql.checkRows(9) + + ##test dumplicate key with normal colomn + #tdSql.execute("INSERT INTO jsons1_15 using jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"datastr\":\"是是是\"}') values(1591060828000, 4, false, 'jjsf', \"你就会\")") + #tdSql.query("select * from jsons1 where jtag->'datastr' match '是' and datastr match 'js'") + #tdSql.checkRows(1) + ## tdSql.query("select tbname,jtag->'tbname' from jsons1 where jtag->'tbname'='tt' and tbname='jsons1_14'") + ## tdSql.checkRows(1) + + ## test join + #tdSql.execute("create table if not exists jsons2(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + #tdSql.execute("insert into jsons2_1 using jsons2 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 2, false, 'json2', '你是2')") + #tdSql.execute("insert into jsons2_2 using jsons2 tags('{\"tag1\":5,\"tag2\":null}') values (1591060628000, 2, true, 'json2', 'sss')") + + #tdSql.execute("create table if not exists jsons3(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + #tdSql.execute("insert into jsons3_1 using jsons3 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 3, false, 'json3', '你是3')") + #tdSql.execute("insert into jsons3_2 using jsons3 tags('{\"tag1\":5,\"tag2\":\"beijing\"}') values (1591060638000, 2, true, 'json3', 'sss')") + #tdSql.query("select 'sss',33,a.jtag->'tag3' from jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") + #tdSql.checkData(0, 0, "sss") + #tdSql.checkData(0, 2, "true") + + #res = tdSql.getColNameList("select 'sss',33,a.jtag->'tag3' from jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'") + #cname_list = [] + #cname_list.append("'sss'") + #cname_list.append("33") + #cname_list.append("a.jtag->'tag3'") + #tdSql.checkColNameList(res, cname_list) + ## + ## test group by & order by json tag + #tdSql.query("select ts,jtag->'tag1' from jsons1 partition by jtag->'tag1' order by jtag->'tag1' desc") + #tdSql.checkRows(11) + #tdSql.checkData(0, 1, '"femail"') + #tdSql.checkData(2, 1, '"收到货"') + #tdSql.checkData(7, 1, "false") + + + #tdSql.error("select count(*) from jsons1 group by jtag") + #tdSql.error("select count(*) from jsons1 partition by jtag") + #tdSql.error("select count(*) from jsons1 group by jtag order by jtag") + #tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'") + #tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag") + #tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") + #tdSql.checkRows(8) + #tdSql.checkData(0, 0, 2) + #tdSql.checkData(0, 1, '"femail"') + #tdSql.checkData(1, 0, 2) + #tdSql.checkData(1, 1, '"收到货"') + #tdSql.checkData(2, 0, 1) + #tdSql.checkData(2, 1, "11.000000000") + #tdSql.checkData(5, 0, 1) + #tdSql.checkData(5, 1, "false") + + #tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc") + #tdSql.checkRows(8) + #tdSql.checkData(0, 1, None) + #tdSql.checkData(2, 0, 1) + #tdSql.checkData(2, 1, "false") + #tdSql.checkData(5, 0, 1) + #tdSql.checkData(5, 1, "11.000000000") + #tdSql.checkData(7, 0, 2) + #tdSql.checkData(7, 1, '"femail"') + + ## test stddev with group by json tag + #tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + #tdSql.checkRows(8) + #tdSql.checkData(0, 1, None) + #tdSql.checkData(4, 0, 0) + #tdSql.checkData(4, 1, "5.000000000") + #tdSql.checkData(7, 0, 11) + #tdSql.checkData(7, 1, '"femail"') + + #res = tdSql.getColNameList("select stddev(dataint),jsons1.jtag->'tag1' from jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'") + #cname_list = [] + #cname_list.append("stddev(dataint)") + #cname_list.append("jsons1.jtag->'tag1'") + #tdSql.checkColNameList(res, cname_list) + + ## test top/bottom with group by json tag + #tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + #tdSql.checkRows(11) + #tdSql.checkData(0, 1, None) + ##tdSql.checkData(2, 0, 24) + ##tdSql.checkData(3, 0, 3) + ##tdSql.checkData(3, 1, "false") + ##tdSql.checkData(8, 0, 2) + ##tdSql.checkData(10, 1, '"femail"') + + ## test having + ## tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1") + ## tdSql.checkRows(3) + + ## subquery with json tag + #tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint") + #tdSql.checkRows(11) + #tdSql.checkData(1, 1, 1) + #tdSql.checkData(5, 0, '{"tag1":false,"tag2":"beijing"}') + + #tdSql.error("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)") + #tdSql.error("select t->'tag1' from (select jtag->'tag1' as t, dataint from jsons1)") + ## tdSql.query("select ts,jtag->'tag1' from (select jtag->'tag1',tbname,ts from jsons1 order by ts)") + ## tdSql.checkRows(11) + ## tdSql.checkData(1, 1, "jsons1_1") + ## tdSql.checkData(1, 2, '"femail"') + + ## union all + #tdSql.query("select jtag->'tag1' from jsons1 union all select jtag->'tag2' from jsons2") + #tdSql.checkRows(13) + #tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag2' from jsons2_1") + #tdSql.checkRows(3) + + #tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag1' from jsons2_1") + #tdSql.checkRows(3) + #tdSql.query("select dataint,jtag->'tag1',tbname from jsons1 union all select dataint,jtag->'tag1',tbname from jsons2") + #tdSql.checkRows(13) + #tdSql.query("select dataint,jtag,tbname from jsons1 union all select dataint,jtag,tbname from jsons2") + #tdSql.checkRows(13) + + ##show create table + #tdSql.query("show create table jsons1") + #tdSql.checkData(0, 1, 'CREATE STABLE `jsons1` (`ts` TIMESTAMP, `dataint` INT, `databool` BOOL, `datastr` NCHAR(50), `datastrbin` VARCHAR(150)) TAGS (`jtag` JSON) WATERMARK 5000a, 5000a') + + ##test aggregate function:count/avg/twa/irate/sum/stddev/leastsquares + #tdSql.query("select count(*) from jsons1 where jtag is not null") + #tdSql.checkData(0, 0, 10) + #tdSql.query("select avg(dataint) from jsons1 where jtag is not null") + #tdSql.checkData(0, 0, 5.3) + ## tdSql.query("select twa(dataint) from jsons1 where jtag is not null") + ## tdSql.checkData(0, 0, 28.386363636363637) + ## tdSql.query("select irate(dataint) from jsons1 where jtag is not null") + + #tdSql.query("select sum(dataint) from jsons1 where jtag->'tag1' is not null") + #tdSql.checkData(0, 0, 45) + #tdSql.query("select stddev(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 4.496912521) + #tdSql.query("SELECT LEASTSQUARES(dataint, 1, 1) from jsons1 where jtag is not null") + + ##test selection function:min/max/first/last/top/bottom/percentile/apercentile/last_row/interp + #tdSql.query("select min(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 1) + #tdSql.query("select max(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 11) + #tdSql.query("select first(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 2) + #tdSql.query("select last(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 11) + #tdSql.query("select top(dataint,100) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + #tdSql.query("select bottom(dataint,100) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + ##tdSql.query("select percentile(dataint,20) from jsons1 where jtag->'tag1'>1") + #tdSql.query("select apercentile(dataint, 50) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 1.5) + ## tdSql.query("select last_row(dataint) from jsons1 where jtag->'tag1'>1") + ## tdSql.query("select interp(dataint) from jsons1 where ts = '2020-06-02 09:17:08.000' and jtag->'tag1'>1") + + ##test calculation function:diff/derivative/spread/ceil/floor/round/ + #tdSql.query("select diff(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(2) + ## tdSql.checkData(0, 0, -1) + ## tdSql.checkData(1, 0, 10) + #tdSql.query("select derivative(dataint, 10m, 0) from jsons1 where jtag->'tag1'>1") + ## tdSql.checkData(0, 0, -2) + #tdSql.query("select spread(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkData(0, 0, 10) + #tdSql.query("select ceil(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + #tdSql.query("select floor(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + #tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1") + #tdSql.checkRows(3) + + ##math function + #tdSql.query("select sin(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select cos(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select tan(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select asin(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select acos(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select atan(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select ceil(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select floor(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select abs(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select pow(dataint,5) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select log(dataint,10) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select sqrt(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select HISTOGRAM(dataint,'user_input','[1, 33, 555, 7777]',1) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select csum(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select mavg(dataint,1) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select statecount(dataint,'GE',10) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select stateduration(dataint,'GE',0) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select sample(dataint,3) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select HYPERLOGLOG(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + #tdSql.query("select twa(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + + ## function not ready + #tdSql.query("select tail(dataint,1) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + #tdSql.query("select unique(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select mode(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + #tdSql.query("select irate(dataint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + + ##str function + #tdSql.query("select upper(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select ltrim(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select lower(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select rtrim(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CHAR_LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select SUBSTR(dataStr,5) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CONCAT(dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CONCAT_ws('adad!@!@%$^$%$^$%^a',dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select CAST(dataStr as bigint) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + + ##time function + #tdSql.query("select now() from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select today() from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMEZONE() from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TO_ISO8601(ts) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TO_UNIXTIMESTAMP(datastr) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMETRUNCATE(ts,1s) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMEDIFF(ts,_c0) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select TIMEDIFF(ts,1u) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(3) + #tdSql.query("select ELAPSED(ts,1h) from jsons1 where jtag->'tag1'>1;") + #tdSql.checkRows(1) + + ##test TD-12077 + #tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')") + #tdSql.query("select jtag->'tag3' from jsons1_16") + #tdSql.checkData(0, 0, '-2.111000000') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + From dce0f06e31de54564c04a4eb0ad2af93772e1980 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 15 Jul 2022 20:40:48 +0800 Subject: [PATCH 067/124] fix: fix redirect issue --- source/dnode/mnode/impl/src/mndMain.c | 2 ++ source/libs/qworker/inc/qwInt.h | 4 ++-- source/libs/qworker/src/qwDbg.c | 11 ++++++++++- source/libs/qworker/src/qworker.c | 21 +++++++++++++++------ source/libs/scheduler/src/schTask.c | 10 ++++++++++ 5 files changed, 39 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 29e68ce4e8..7151c4a026 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -42,6 +42,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "qworker.h" static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -375,6 +376,7 @@ void mndPreClose(SMnode *pMnode) { void mndClose(SMnode *pMnode) { if (pMnode != NULL) { mDebug("start to close mnode"); + qWorkerDestroy((void **)&pMnode->pQuery); mndCleanupSteps(pMnode, -1); taosMemoryFreeClear(pMnode->path); taosMemoryFreeClear(pMnode); diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 3c6c02ede6..56af7ee1e0 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -132,7 +132,7 @@ typedef struct SQWTaskCtx { int8_t taskType; int8_t explain; int8_t needFetch; - int32_t queryType; + int32_t msgType; int32_t fetchType; int32_t execId; @@ -380,7 +380,7 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt); int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore); int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SEpSet *pEpSet); int32_t qwAddTaskCtx(QW_FPARAMS_DEF); -int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx); +int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped); #ifdef __cplusplus diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 7906617a3f..4e174018f4 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -147,7 +147,7 @@ int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int return TSDB_CODE_SUCCESS; } -int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx) { +int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) { if (gQWDebug.tmp) { if (TDMT_SCH_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { SEpSet epSet = {0}; @@ -162,16 +162,25 @@ int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx) { ctx->phase = QW_PHASE_POST_QUERY; qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, &epSet); + *rsped = true; return TSDB_CODE_SUCCESS; } if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { QW_SET_PHASE(ctx, QW_PHASE_POST_QUERY); qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); + *rsped = true; + return TSDB_CODE_SUCCESS; + } + + if ((TDMT_SCH_FETCH == qwMsg->msgType) && (0 == taosRand() % 3)) { + qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); + *rsped = true; return TSDB_CODE_SUCCESS; } } + *rsped = false; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index c763645a07..7e12a3a424 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -428,7 +428,14 @@ _return: if (QW_PHASE_POST_QUERY == phase && ctx) { ctx->queryRsped = true; - qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); + + bool rsped = false; + SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; + qwDbgResponseRedirect(&qwMsg, ctx, &rsped); + if (!rsped) { + qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); + } + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); } @@ -476,8 +483,6 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); - qwDbgResponseRedirect(qwMsg, ctx); - _return: if (ctx) { @@ -505,7 +510,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) { ctx->taskType = qwMsg->msgInfo.taskType; ctx->explain = qwMsg->msgInfo.explain; ctx->needFetch = qwMsg->msgInfo.needFetch; - ctx->queryType = qwMsg->msgType; + ctx->msgType = qwMsg->msgType; //QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); @@ -650,7 +655,7 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - ctx->queryType = qwMsg->msgType; + ctx->msgType = qwMsg->msgType; SOutputData sOutput = {0}; QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); @@ -702,7 +707,11 @@ _return: } if (code || rsp) { - qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); + bool rsped = false; + qwDbgResponseRedirect(qwMsg, ctx, &rsped); + if (!rsped) { + qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); + } QW_TASK_DLOG("%s send, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1), qwMsg->connInfo.handle, code, tstrerror(code), dataLen); } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index a46b293965..bfba81b118 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -138,7 +138,17 @@ int32_t schUpdateTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int3 return TSDB_CODE_SUCCESS; } + if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it + SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry); + return TSDB_CODE_SUCCESS; + } + SSchNodeInfo *nodeInfo = taosHashGet(pTask->execNodes, &execId, sizeof(execId)); + if (NULL == nodeInfo) { // ignore it + SCH_TASK_DLOG("handle not updated since execId %d already not exist, current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry); + return TSDB_CODE_SUCCESS; + } + nodeInfo->handle = handle; SCH_TASK_DLOG("handle updated to %p for execId %d", handle, execId); From e840cdaaace17846e7430af636556a608fc14d2e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Jul 2022 20:56:29 +0800 Subject: [PATCH 068/124] fix(query): set correct sort buffer format --- source/libs/executor/src/tsort.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 8779fe5415..9b15cc7adb 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -621,7 +621,7 @@ static int32_t createInitialSources(SSortHandle* pHandle) { pHandle->sortElapsed += el; // All sorted data can fit in memory, external memory sort is not needed. Return to directly - if (size <= sortBufSize) { + if (size <= sortBufSize && pHandle->pBuf == NULL) { pHandle->cmpParam.numOfSources = 1; pHandle->inMemSort = true; From 989159f0ba4510c2a07eb1dfad0796b9f2854269 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 15 Jul 2022 13:02:24 +0000 Subject: [PATCH 069/124] vnod snapshot done --- source/dnode/vnode/src/tsdb/tsdbRead.c | 23 ++- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 184 +++++++++++++++------ 2 files changed, 145 insertions(+), 62 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f0aea0cefb..7b4127e591 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -63,10 +63,10 @@ typedef struct SBlockLoadSuppInfo { } SBlockLoadSuppInfo; typedef struct SFilesetIter { - int32_t numOfFiles; // number of total files - int32_t index; // current accessed index in the list - SArray* pFileList; // data file list - int32_t order; + int32_t numOfFiles; // number of total files + int32_t index; // current accessed index in the list + SArray* pFileList; // data file list + int32_t order; } SFilesetIter; typedef struct SFileDataBlockInfo { @@ -830,8 +830,8 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int32_t code = tsdbReadColData(pReader->pFileReader, &pBlockScanInfo->blockIdx, pBlock, pSupInfo->colIds, numOfCols, - pBlockData, NULL, NULL); + int32_t code = tsdbReadColData(pReader->pFileReader, &pBlockScanInfo->blockIdx, pBlock, pSupInfo->colIds, numOfCols, + pBlockData, NULL, NULL); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1991,7 +1991,7 @@ static TSDBKEY getCurrentKeyInBuf(SDataBlockIter* pBlockIter, STsdbReader* pRead static int32_t moveToNextFile(STsdbReader* pReader, int32_t* numOfBlocks) { SReaderStatus* pStatus = &pReader->status; - SArray* pIndexList = taosArrayInit(4, sizeof(SBlockIdx)); + SArray* pIndexList = taosArrayInit(4, sizeof(SBlockIdx)); while (1) { bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader); @@ -3006,14 +3006,14 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { code = doLoadFileBlockData(pReader, &pStatus->blockIter, pBlockScanInfo, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { - tBlockDataClear(&pStatus->fileBlockData); + tBlockDataClear(&pStatus->fileBlockData, 1); terrno = code; return NULL; } copyBlockDataToSDataBlock(pReader, pBlockScanInfo); - tBlockDataClear(&pStatus->fileBlockData); + tBlockDataClear(&pStatus->fileBlockData, 1); return pReader->pResBlock->pDataBlock; } @@ -3132,8 +3132,8 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa hasNext = (pBlockIter->numOfBlocks > 0); } -// tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, -// pReader->pFileGroup->fid, pReader->idStr); + // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, + // pReader->pFileGroup->fid, pReader->idStr); } return code; @@ -3204,4 +3204,3 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 return TSDB_CODE_SUCCESS; } - diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 932cf9f9ae..e561bb9d22 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -411,30 +411,6 @@ static int32_t tsdbSnapCommit(STsdbSnapWriter* pWriter) { return code; } -static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { - int32_t code = 0; - STsdb* pTsdb = pWriter->pTsdb; - - if (pWriter->pDataFWriter == NULL) goto _exit; - - // TODO - - code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 0); - if (code) goto _err; - - if (pWriter->pDataFReader) { - code = tsdbDataFReaderClose(&pWriter->pDataFReader); - if (code) goto _err; - } - -_exit: - return code; - -_err: - tsdbError("vgId:%d tsdb snapshot writer data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; int32_t iRow = 0; // todo @@ -456,14 +432,41 @@ _err: static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; - ASSERT(pWriter->pBlockIdxW != NULL); + ASSERT(pWriter->pDataFWriter); + + if (pWriter->pBlockIdxW == NULL) goto _exit; + + // consume remain rows + if (pWriter->pBlockData) { + ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); + while (pWriter->iRow < pWriter->pBlockData->nRow) { + code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL); + if (code) goto _err; + + if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) { + pWriter->blockW.last = 0; + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + &pWriter->blockW, pWriter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + if (code) goto _err; + + tBlockReset(&pWriter->blockW); + tBlockDataClearData(&pWriter->bDataW); + } + + pWriter->iRow++; + } + } // write remain data if has if (pWriter->bDataW.nRow > 0) { - if (pWriter->bDataW.nRow >= pWriter->minRow) { - pWriter->blockW.last = 0; - } else { - pWriter->blockW.last = 1; + pWriter->blockW.last = 0; + if (pWriter->bDataW.nRow < pWriter->minRow) { + if (pWriter->iBlock > pWriter->mBlock.nItem) { + pWriter->blockW.last = 1; + } } code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, @@ -474,14 +477,40 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { if (code) goto _err; } + while (true) { + if (pWriter->iBlock >= pWriter->mBlock.nItem) break; + + SBlock block; + tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); + + if (block.last) { + code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); + if (code) goto _err; + + tBlockReset(&block); + block.last = 1; + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block, + pWriter->cmprAlg); + if (code) goto _err; + } + + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + if (code) goto _err; + + pWriter->iBlock++; + } + + // SBlock code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); if (code) goto _err; + // SBlockIdx if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } +_exit: return code; _err: @@ -601,25 +630,39 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { ASSERT(c); - if (c > 0) break; + if (c < 0) { + if (pWriter->bDataW.nRow) { + pWriter->blockW.last = 0; + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + &pWriter->blockW, pWriter->cmprAlg); + if (code) goto _err; - if (pWriter->bDataW.nRow) { - pWriter->blockW.last = 0; - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - &pWriter->blockW, pWriter->cmprAlg); + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + if (code) goto _err; + + tBlockReset(&pWriter->blockW); + tBlockDataClearData(&pWriter->bDataW); + } + + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); if (code) goto _err; - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); - if (code) goto _err; + pWriter->iBlock++; + } else { + c = tsdbKeyCmprFn(&tBlockDataLastKey(pBlockData), &block.minKey); - tBlockReset(&pWriter->blockW); - tBlockDataClearData(&pWriter->bDataW); + ASSERT(c); + + if (c < 0) break; + + pWriter->pBlockData = &pWriter->bDataR; + code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); + if (code) goto _err; + pWriter->iRow = 0; + + pWriter->iBlock++; + break; } - - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); - if (code) goto _err; - - pWriter->iBlock++; } if (pWriter->pBlockData) continue; @@ -744,6 +787,45 @@ _err: return code; } +static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { + int32_t code = 0; + STsdb* pTsdb = pWriter->pTsdb; + + if (pWriter->pDataFWriter == NULL) goto _exit; + + code = tsdbSnapWriteTableDataEnd(pWriter); + if (code) goto _err; + + while (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { + code = tsdbSnapMoveWriteTableData(pWriter, (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx)); + if (code) goto _err; + + pWriter->iBlockIdx++; + } + + code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW, NULL); + if (code) goto _err; + + code = tsdbFSStateUpsertDFileSet(pTsdb->fs->nState, tsdbDataFWriterGetWSet(pWriter->pDataFWriter)); + if (code) goto _err; + + code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1); + if (code) goto _err; + + if (pWriter->pDataFReader) { + code = tsdbDataFReaderClose(&pWriter->pDataFReader); + if (code) goto _err; + } + +_exit: + tsdbError("vgId:%d vnode snapshot tsdb writer data end", TD_VID(pTsdb->pVnode)); + return code; + +_err: + tsdbError("vgId:%d vnode snapshot tsdb writer data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; @@ -845,23 +927,23 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 } // writer - SDelFile delFile = {.commitID = pTsdb->pVnode->state.commitID, .offset = 0, .size = 0}; + SDelFile delFile = {.commitID = pWriter->commitID, .offset = 0, .size = 0}; code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb); if (code) goto _err; } // process the del data - TABLEID id = {0}; // todo + TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); while (true) { SDelIdx* pDelIdx = NULL; - int64_t n = 0; + int64_t n = sizeof(SSnapDataHdr) + sizeof(TABLEID); SDelData delData; SDelIdx delIdx; int8_t toBreak = 0; if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR)) { - pDelIdx = taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); + pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); } if (pDelIdx) { @@ -914,7 +996,7 @@ _exit: return code; _err: - tsdbError("vgId:%d tsdb snapshot write del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb write del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -923,6 +1005,7 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { STsdb* pTsdb = pWriter->pTsdb; if (pWriter->pDelFWriter == NULL) goto _exit; + for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) { SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); @@ -954,10 +1037,11 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { } _exit: + tsdbInfo("vgId:%d vnode snapshot tsdb write del end", TD_VID(pTsdb->pVnode)); return code; _err: - tsdbError("vgId:%d tsdb snapshow write del end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb write del end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -1076,7 +1160,7 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) // del data if (pHdr->type == 2) { - code = tsdbSnapWriteDel(pWriter, pData + 1, nData - 1); + code = tsdbSnapWriteDel(pWriter, pData, nData); if (code) goto _err; } From 725fa46142cfe04d498beb83532e2e2d7a32ec86 Mon Sep 17 00:00:00 2001 From: tomchon Date: Fri, 15 Jul 2022 21:08:17 +0800 Subject: [PATCH 070/124] test:add test case of tsbs query --- tests/system-test/2-query/tsbsQuery.py | 40 ++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 5fe025ea07..b52b0bffef 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -40,9 +40,9 @@ class TDTestCase: f"insert into rct{j} values ( {ts+i*10000}, {80+i}, {90+i}, {85+i}, {30+i*10}, {1.2*i}, {221+i*2}, {20+i*0.2}, {1500+i*20}, {150+i*2},{5+i} )" ) tdSql.execute( - f"insert into dct{j} values ( {ts+i*10000}, {1+i*0.1},{1400+i*15}, {1+i},{1500+i*20}, {150+i*2},{5+i} )" + f"insert into dct{j} values ( {ts+i*10000}, {1+i*0.1},{1400+i*15}, {i},{1500+i*20}, {150+i*2},{5+i} )" ) - + tdSql.execute("insert into dct9 (ts,fuel_state) values('2021-07-13 14:06:33.123Z',1.2) ;") # def check_avg(self ,origin_query , check_query): # avg_result = tdSql.getResult(origin_query) # origin_result = tdSql.getResult(check_query) @@ -60,6 +60,7 @@ class TDTestCase: def tsbsIotQuery(self): + tdSql.execute("use db_tsbs") # test interval and partition @@ -88,11 +89,38 @@ class TDTestCase: # test partition interval Pseudo time-column tdSql.query("SELECT count(ms1)/144 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;") - #PR TD-17413 - tdSql.query("SELECT ms1 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-01T00:01:01Z';") - tdSql.checkRows(0) + # 1 high-load: + # tdSql.query("SELECT ts,name,driver,current_load,load_capacity FROM (SELECT last(ts) as ts,name,driver, current_load,load_capacity FROM diagnostics WHERE fleet = 'South' partition by name,driver) WHERE current_load>= (0.9 * load_capacity) partition by name ORDER BY name desc, ts DESC;") - # test + # tdSql.query("SELECT ts,name,driver,current_load,load_capacity FROM (SELECT last(ts) as ts,name,driver, current_load,load_capacity FROM diagnostics WHERE fleet = 'South' partition by name,driver) WHERE current_load>= (0.9 * load_capacity) partition by name ORDER BY name ;") + + # 2 stationary-trucks + tdSql.query("select name,driver from (SELECT name,driver,fleet ,avg(velocity) as mean_velocity FROM readings WHERE ts > '2016-01-01T15:07:21Z' AND ts <= '2016-01-01T16:17:21Z' partition BY name,driver,fleet interval(10m) LIMIT 1)") + tdSql.query("select name,driver from (SELECT name,driver,fleet ,avg(velocity) as mean_velocity FROM readings WHERE ts > '2016-01-01T15:07:21Z' AND ts <= '2016-01-01T16:17:21Z' partition BY name,driver,fleet interval(10m) LIMIT 1) WHERE fleet = 'West' AND mean_velocity < 1000 partition BY name") + + # 3 long-driving-sessions + # tdSql.query("SELECT name,driver FROM(SELECT name,driver,count(*) AS ten_min FROM(SELECT _wstart as ts,name,driver,avg(velocity) as mean_velocity FROM readings where ts > '2016-01-01T00:00:34Z' AND ts <= '2016-01-01T04:00:34Z' partition BY name,driver interval(10m)) WHERE mean_velocity > 1 GROUP BY name,driver) WHERE ten_min > 22 ;") + + + #4 long-daily-sessions + tdSql.query("SELECT name,driver FROM(SELECT name,driver,count(*) AS ten_min FROM(SELECT name,driver,avg(velocity) as mean_velocity FROM readings WHERE fleet ='West' AND ts > '2016-01-01T12:31:37Z' AND ts <= '2016-01-05T12:31:37Z' partition BY name,driver interval(10m) ) WHERE mean_velocity > 1 GROUP BY name,driver) WHERE ten_min > 60") + + # 5. avg-daily-driving-duration + tdSql.query("select _wstart as ts,fleet,name,driver,count(mv)/6 as hours_driven from ( select _wstart as ts,fleet,name,driver,avg(velocity) as mv from readings where ts > '2016-01-01T00:00:00Z' and ts < '2016-01-05T00:00:01Z' partition by fleet,name,driver interval(10m)) where ts > '2016-01-01T00:00:00Z' and ts < '2016-01-05T00:00:01Z' partition by fleet,name,driver interval(1d) ;") + + + # 6. avg-daily-driving-session + #taosc core dumped + tdSql.execute("create table random_measure2_1 (ts timestamp,ela float, name binary(40))") + tdSql.query("SELECT ts,diff(mv) AS difka FROM (SELECT ts,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name,ts interval(10m) fill(value,0)) GROUP BY name,ts;") + tdSql.query("SELECT _wstart,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0)") + + # 7. avg-load + tdSql.query("SELECT fleet, model,avg(ml) AS mean_load_percentage FROM (SELECT fleet, model,current_load/load_capacity AS ml FROM diagnostics partition BY name, fleet, model) partition BY fleet, model order by fleet ;") + + # 8. daily-activity + tdSql.query(" SELECT model,ms1 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m) fill(value,0)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;") + tdSql.query("SELECT _wstart,model,fleet,count(ms1)/144 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m) fill(value,0)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1 partition by model, fleet interval(1d) ;") def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdLog.printNoPrefix("==========step1:create database and table,insert data ==============") self.prepareData() From 280ed4f65d88c470c4dfbfa957246b385359427c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 15 Jul 2022 21:29:53 +0800 Subject: [PATCH 071/124] feat(stream): pipelined exec for recovering --- include/libs/stream/tstream.h | 5 ++ source/dnode/mnode/impl/src/mndStream.c | 90 ++++++++++++++++++++++++- source/dnode/snode/inc/sndInt.h | 9 --- source/libs/stream/inc/streamInc.h | 2 + source/libs/stream/src/streamExec.c | 57 ++++++++++++++++ source/libs/stream/src/streamRecover.c | 6 +- 6 files changed, 156 insertions(+), 13 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index d03fab0ebc..5c4d8ce250 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -31,6 +31,11 @@ extern "C" { typedef struct SStreamTask SStreamTask; +enum { + STREAM_STATUS__NORMAL = 0, + STREAM_STATUS__RECOVER, +}; + enum { TASK_STATUS__NORMAL = 0, TASK_STATUS__DROPPING, diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 8248a26c28..5e8867fdfb 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -177,7 +177,7 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream taosWLockLatch(&pOldStream->lock); - // TODO handle update + pOldStream->status = pNewStream->status; taosWUnLockLatch(&pOldStream->lock); return 0; @@ -395,6 +395,20 @@ int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr return 0; } +static int32_t mndSetStreamRecover(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream) { + SStreamObj streamObj = {0}; + memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN); + streamObj.status = STREAM_STATUS__RECOVER; + SSdbRaw *pCommitRaw = mndStreamActionEncode(&streamObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + return 0; +} + static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { SStbObj *pStb = NULL; SDbObj *pDb = NULL; @@ -492,6 +506,76 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return 0; } +static int32_t mndPersistTaskRecoverReq(STrans *pTrans, SStreamTask *pTask) { + SMStreamTaskRecoverReq *pReq = taosMemoryCalloc(1, sizeof(SMStreamTaskRecoverReq)); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pReq->streamId = pTask->streamId; + pReq->taskId = pTask->taskId; + int32_t len; + int32_t code; + tEncodeSize(tEncodeSMStreamTaskRecoverReq, pReq, len, code); + if (code != 0) { + return -1; + } + void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + if (buf == NULL) { + return -1; + } + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + tEncodeSMStreamTaskRecoverReq(&encoder, pReq); + ((SMsgHead *)buf)->vgId = pTask->nodeId; + + STransAction action = {0}; + memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + action.pCont = buf; + action.contLen = sizeof(SMsgHead) + len; + action.msgType = TDMT_STREAM_TASK_RECOVER; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(buf); + return -1; + } + return 0; +} + +int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { + if (pStream->isDistributed) { + int32_t lv = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < lv; i++) { + SArray *pTasks = taosArrayGetP(pStream->tasks, i); + int32_t sz = taosArrayGetSize(pTasks); + SStreamTask *pTask = taosArrayGetP(pTasks, 0); + if (!pTask->isDataScan && pTask->execType != TASK_EXEC__NONE) { + ASSERT(sz == 1); + if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { + return -1; + } + } else { + continue; + } + } + } else { + int32_t lv = taosArrayGetSize(pStream->tasks); + for (int32_t i = 0; i < lv; i++) { + SArray *pTasks = taosArrayGetP(pStream->tasks, i); + int32_t sz = taosArrayGetSize(pTasks); + for (int32_t j = 0; j < sz; j++) { + SStreamTask *pTask = taosArrayGetP(pTasks, j); + if (!pTask->isDataScan) break; + ASSERT(pTask->execType != TASK_EXEC__NONE); + if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { + return -1; + } + } + } + } + return 0; +} + int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t lv = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < lv; i++) { @@ -712,14 +796,14 @@ static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { mDebug("trans:%d, used to drop stream:%s", pTrans->id, recoverReq.name); // broadcast to recover all tasks - if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { + if (mndRecoverStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to recover task since %s", recoverReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); return -1; } // update stream status - if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + if (mndSetStreamRecover(pMnode, pTrans, pStream) < 0) { sdbRelease(pMnode->pSdb, pStream); return -1; } diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 8916e2a31c..1f0019ef46 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -30,15 +30,6 @@ extern "C" { #endif -enum { - STREAM_STATUS__RUNNING = 1, - STREAM_STATUS__STOPPED, - STREAM_STATUS__CREATING, - STREAM_STATUS__STOPING, - STREAM_STATUS__RESTORING, - STREAM_STATUS__DELETING, -}; - typedef struct { SHashObj* pHash; // taskId -> SStreamTask } SStreamMeta; diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index f9f4e62774..d10ea76c83 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -33,6 +33,8 @@ typedef struct { static SStreamGlobalEnv streamEnv; int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb); +int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum); + int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData); int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index cffd519d2a..9d22da8662 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -82,6 +82,63 @@ static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock return 0; } +int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { + ASSERT(pTask->execType != TASK_EXEC__NONE); + + void* exec = pTask->exec.executor; + + while (1) { + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + if (pRes == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t batchCnt = 0; + while (1) { + SSDataBlock* output = NULL; + uint64_t ts = 0; + if (qExecTask(exec, &output, &ts) < 0) { + ASSERT(0); + } + if (output == NULL) break; + + SSDataBlock block = {0}; + assignOneDataBlock(&block, output); + block.info.childId = pTask->selfChildId; + taosArrayPush(pRes, &block); + + if (++batchCnt >= batchNum) break; + } + if (taosArrayGetSize(pRes) == 0) { + taosArrayDestroy(pRes); + break; + } + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + if (qRes == NULL) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + return -1; + } + + qRes->type = STREAM_INPUT__DATA_BLOCK; + qRes->blocks = pRes; + qRes->childId = pTask->selfChildId; + + if (streamTaskOutput(pTask, qRes) < 0) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + taosFreeQitem(qRes); + return -1; + } + + if (pTask->dispatchType != TASK_DISPATCH__NONE) { + ASSERT(pTask->sinkType == TASK_SINK__NONE); + streamDispatch(pTask, pTask->pMsgCb); + } + } + + return 0; +} + static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { int32_t cnt = 0; void* data = NULL; diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 4175f60446..87b27daf60 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -125,7 +125,11 @@ int32_t streamProcessFailRecoverReq(SStreamTask* pTask, SMStreamTaskRecoverReq* } if (pTask->taskStatus == TASK_STATUS__RECOVERING) { - streamProcessRunReq(pTask); + if (streamPipelineExec(pTask, 10) < 0) { + // set fail + return -1; + } } + return 0; } From 352db0febbfca2948786e16c66eb4b2092d11cb1 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sat, 16 Jul 2022 09:28:15 +0800 Subject: [PATCH 072/124] feat(stream): optimize interval plan --- source/common/src/tdatablock.c | 2 +- source/libs/executor/inc/executorimpl.h | 16 +- source/libs/executor/src/scanoperator.c | 20 ++- source/libs/executor/src/timewindowoperator.c | 21 ++- source/libs/planner/src/planSpliter.c | 3 +- tests/script/tsim/stream/sliding.sim | 139 ++++++++++++++++++ 6 files changed, 183 insertions(+), 18 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 47cfa5d410..cd21de52ed 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1752,7 +1752,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; int32_t len = 0; - len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id:%" PRIu64 "| uid:%ld\n", flag, + len += snprintf(dumpBuf + len, size - len, "%s |block type %d |child id %d|group id:%" PRIu64 "| uid:%ld|======\n", "dumpBlockData", (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId, pDataBlock->info.uid); if (len >= size - 1) return dumpBuf; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f110b1c92e..cd50a13edd 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -371,6 +371,13 @@ typedef struct SessionWindowSupporter { uint8_t parentType; } SessionWindowSupporter; +typedef struct STimeWindowSupp { + int8_t calTrigger; + int64_t waterMark; + TSKEY maxTs; + SColumnInfoData timeWindowData; // query time window info for scalar function execution. +} STimeWindowAggSupp; + typedef struct SStreamScanInfo { uint64_t tableUid; // queried super table uid SExprInfo* pPseudoExpr; @@ -407,6 +414,7 @@ typedef struct SStreamScanInfo { SSDataBlock* pDeleteDataRes; // delete data SSDataBlock int32_t deleteDataIndex; STimeWindow updateWin; + STimeWindowAggSupp twAggSup; // status for tmq // SSchemaWrapper schema; @@ -452,13 +460,6 @@ typedef struct SAggSupporter { int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row } SAggSupporter; -typedef struct STimeWindowSupp { - int8_t calTrigger; - int64_t waterMark; - TSKEY maxTs; - SColumnInfoData timeWindowData; // query time window info for scalar function execution. -} STimeWindowAggSupp; - typedef struct SIntervalAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; // basic info @@ -952,6 +953,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); +bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index afb5e5ed45..a81a796fa7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -802,7 +802,12 @@ static bool isStateWindow(SStreamScanInfo* pInfo) { static bool isIntervalWindow(SStreamScanInfo* pInfo) { return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || - pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL; + pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || + pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; +} + +static bool isSignleIntervalWindow(SStreamScanInfo* pInfo) { + return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; } static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) { @@ -1130,9 +1135,14 @@ static void setUpdateData(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlo static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); - TSKEY* ts = (TSKEY*)pColDataInfo->pData; + TSKEY* tsCol = (TSKEY*)pColDataInfo->pData; for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { - if (updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, ts[rowId]) && out) { + SResultRowInfo dumyInfo; + dumyInfo.cur.pageId = -1; + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[rowId], &pInfo->interval, TSDB_ORDER_ASC); + // must check update info first. + bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); + if ( (update || (isSignleIntervalWindow(pInfo) && isCloseWindow(&win, &pInfo->twAggSup)) ) && out) { taosArrayPush(pInfo->tsArray, &rowId); } } @@ -1413,6 +1423,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->tsArrayIndex = 0; checkUpdateData(pInfo, true, pInfo->pRes, true); setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlockInfo->window.ekey); if (pInfo->pUpdateRes->info.rows > 0) { if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) { pInfo->updateResIndex = 0; @@ -1584,13 +1595,14 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateRes = createResDataBlock(pDescNode); pInfo->pCondition = pScanPhyNode->node.pConditions; pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; + pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; pInfo->groupId = 0; pInfo->pPullDataRes = createPullDataBlock(); pInfo->pStreamScanOp = pOperator; pInfo->deleteDataIndex = 0; pInfo->pDeleteDataRes = createPullDataBlock(); pInfo->updateWin = (STimeWindow){.skey = INT64_MAX, .ekey = INT64_MAX}; + pInfo->twAggSup = *pTwSup; pOperator->name = "StreamScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index acc56af5cd..71eb221c64 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -652,8 +652,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } void printDataBlock(SSDataBlock* pBlock, const char* flag) { - if (pBlock == NULL) { - qDebug("======printDataBlock Block is Null"); + if (!pBlock || pBlock->info.rows == 0) { + qDebug("======printDataBlock: Block is Null or Empty"); return; } char* pBuf = NULL; @@ -1355,7 +1355,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, int32_t size = taosArrayGetSize(chAy); qDebug("window %" PRId64 " wait child size:%d", win.skey, size); for (int32_t i = 0; i < size; i++) { - qDebug("window %" PRId64 " wait chid id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); + qDebug("window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); } continue; } else if (pPullDataMap) { @@ -1626,6 +1626,12 @@ SSDataBlock* createDeleteBlock() { return pBlock; } +void initIntervalDownStream(SOperatorInfo* downstream, uint8_t type) { + ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + SStreamScanInfo* pScanInfo = downstream->info; + pScanInfo->sessionSup.parentType = type; +} + SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, @@ -1701,6 +1707,10 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) { + initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL); + } + code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -2476,12 +2486,14 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc } else { int32_t index = -1; SArray* chArray = NULL; + int32_t chId = 0; if (chIds) { chArray = *(void**)chIds; - int32_t chId = getChildIndex(pSDataBlock); + chId = getChildIndex(pSDataBlock); index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); } if (index != -1 && pSDataBlock->info.type == STREAM_PULL_DATA) { + qDebug("======delete child id %d", chId); taosArrayRemove(chArray, index); if (taosArrayGetSize(chArray) == 0) { // pull data is over @@ -3010,6 +3022,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num pDummy[i].functionId = pCtx[i].functionId; } } + void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark, uint8_t type) { ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 871bcf015b..ae0ccb1c51 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -198,8 +198,7 @@ static bool stbSplHasGatherExecFunc(const SNodeList* pFuncs) { } static bool stbSplIsMultiTbScan(bool streamQuery, SScanLogicNode* pScan) { - return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1) || - (streamQuery && TSDB_SUPER_TABLE == pScan->tableType); + return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1); } static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index 1ffca2a67b..f34a50de9d 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -366,4 +366,143 @@ if $data32 != 8 then goto loop1 endi +sql drop database IF EXISTS test2; +sql drop stream IF EXISTS streams21; +sql drop stream IF EXISTS streams22; + +sql create database test2 vgroups 2; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams21 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); +sql create stream streams22 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,1.1); +sql insert into t1 values(1648791233002,3,3,3,2.1); +sql insert into t1 values(1648791243003,4,4,4,3.1); +sql insert into t1 values(1648791213004,4,5,5,4.1); + +sql insert into t2 values(1648791213000,1,6,6,1.0); +sql insert into t2 values(1648791223001,2,7,7,1.1); +sql insert into t2 values(1648791233002,3,8,8,2.1); +sql insert into t2 values(1648791243003,4,9,9,3.1); +sql insert into t2 values(1648791213004,4,10,10,4.1); + +$loop_count = 0 + +loop2: +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt; + +# row 0 +if $data01 != 2 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 5 then + print =====data02=$data02 + goto loop2 +endi + +# row 1 +if $data11 != 1 then + print =====data11=$data11 + goto loop2 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop2 +endi + +# row 2 +if $data21 != 1 then + print =====data21=$data21 + goto loop2 +endi + +if $data22 != 3 then + print =====data22=$data22 + goto loop2 +endi + +# row 3 +if $data31 != 1 then + print =====data31=$data31 + goto loop2 +endi + +if $data32 != 4 then + print =====data32=$data32 + goto loop2 +endi + +print step 6 + +$loop_count = 0 + +loop3: +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt2; + +# row 0 +if $data01 != 4 then + print =====data01=$data01 + # goto loop3 +endi + +if $data02 != 10 then + print =====data02=$data02 + goto loop3 +endi + +# row 1 +if $data11 != 2 then + print =====data11=$data11 + goto loop3 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop3 +endi + +# row 2 +if $data21 != 2 then + print =====data21=$data21 + goto loop3 +endi + +if $data22 != 6 then + print =====data22=$data22 + goto loop3 +endi + +# row 3 +if $data31 != 2 then + print =====data31=$data31 + goto loop3 +endi + +if $data32 != 8 then + print =====data32=$data32 + goto loop3 +endi + system sh/stop_dnodes.sh \ No newline at end of file From af24f2f85d414b636b0e48e0108f3f3a86a7267f Mon Sep 17 00:00:00 2001 From: huolibo Date: Sat, 16 Jul 2022 09:32:31 +0800 Subject: [PATCH 073/124] fix(driver): tmq jdbc get meta --- .../jni/com_taosdata_jdbc_tmq_TMQConnector.h | 2 +- source/client/src/TMQConnector.c | 20 +++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h index b3b098e460..197cd78006 100644 --- a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h +++ b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h @@ -161,7 +161,7 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableNam * Signature: (JJLcom/taosdata/jdbc/TSDBResultSetBlockData;ILjava/util/List;)I */ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(JNIEnv *, jobject, jlong, jlong, - jobject, jint, jobject); + jobject, jobject); #ifdef __cplusplus } diff --git a/source/client/src/TMQConnector.c b/source/client/src/TMQConnector.c index 1d84dcf7a2..3755a591e3 100644 --- a/source/client/src/TMQConnector.c +++ b/source/client/src/TMQConnector.c @@ -278,7 +278,7 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableNam } JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(JNIEnv *env, jobject jobj, jlong con, - jlong res, jobject rowobj, jint flag, + jlong res, jobject rowobj, jobject arrayListObj) { TAOS *tscon = (TAOS *)con; int32_t code = check_for_params(jobj, con, res); @@ -309,16 +309,14 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( TAOS_FIELD *fields = taos_fetch_fields(tres); jniDebug("jobj:%p, conn:%p, resultset:%p, fields size is %d", jobj, tscon, tres, numOfFields); - if (flag) { - for (int i = 0; i < numOfFields; ++i) { - jobject metadataObj = (*env)->NewObject(env, g_metadataClass, g_metadataConstructFp); - (*env)->SetIntField(env, metadataObj, g_metadataColtypeField, fields[i].type); - (*env)->SetIntField(env, metadataObj, g_metadataColsizeField, fields[i].bytes); - (*env)->SetIntField(env, metadataObj, g_metadataColindexField, i); - jstring metadataObjColname = (*env)->NewStringUTF(env, fields[i].name); - (*env)->SetObjectField(env, metadataObj, g_metadataColnameField, metadataObjColname); - (*env)->CallBooleanMethod(env, arrayListObj, g_arrayListAddFp, metadataObj); - } + for (int i = 0; i < numOfFields; ++i) { + jobject metadataObj = (*env)->NewObject(env, g_metadataClass, g_metadataConstructFp); + (*env)->SetIntField(env, metadataObj, g_metadataColtypeField, fields[i].type); + (*env)->SetIntField(env, metadataObj, g_metadataColsizeField, fields[i].bytes); + (*env)->SetIntField(env, metadataObj, g_metadataColindexField, i); + jstring metadataObjColname = (*env)->NewStringUTF(env, fields[i].name); + (*env)->SetObjectField(env, metadataObj, g_metadataColnameField, metadataObjColname); + (*env)->CallBooleanMethod(env, arrayListObj, g_arrayListAddFp, metadataObj); } (*env)->CallVoidMethod(env, rowobj, g_blockdataSetNumOfRowsFp, (jint)numOfRows); From 52327cc2fb578a4a40c91c2d7f7ad995bbb174cf Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 16 Jul 2022 09:44:52 +0800 Subject: [PATCH 074/124] fix: multi-rows function validity check --- source/libs/parser/src/parTranslater.c | 5 +++-- tests/system-test/2-query/statecount.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 33ab1166b0..3164623465 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1058,8 +1058,9 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (hasInvalidFuncNesting(pFunc->pParameterList)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } - if (!isSelectStmt(pCxt->pCurrStmt) || ((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || - ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc) { + // The auto-generated COUNT function in the DELETE statement is legal + if (isSelectStmt(pCxt->pCurrStmt) && + (((SSelectStmt*)pCxt->pCurrStmt)->hasIndefiniteRowsFunc || ((SSelectStmt*)pCxt->pCurrStmt)->hasMultiRowsFunc)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py index 90e8bebab4..91e2aa9e47 100644 --- a/tests/system-test/2-query/statecount.py +++ b/tests/system-test/2-query/statecount.py @@ -335,7 +335,7 @@ class TDTestCase: # tdSql.checkRows(21) # group by - tdSql.query("select statecount(c1,'GT',1) from ct1 group by c1") + tdSql.error("select statecount(c1,'GT',1) from ct1 group by c1") tdSql.error("select statecount(c1,'GT',1) from ct1 group by tbname") # super table From be8e5325b0aae42e627deffc381582938d3f84c6 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 10:47:32 +0800 Subject: [PATCH 075/124] change submodule ref point --- tools/taos-tools | 2 +- tools/taosadapter | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/taos-tools b/tools/taos-tools index 1163c0f60a..50b68d85f7 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 1163c0f60aa65d6cc58283247c8bf8c56ba43b92 +Subproject commit 50b68d85f7cbaf7a9adfa4082e88ca758770f75e diff --git a/tools/taosadapter b/tools/taosadapter index c3815951fc..d9f9e69960 160000 --- a/tools/taosadapter +++ b/tools/taosadapter @@ -1 +1 @@ -Subproject commit c3815951fc80617ecd171f3743b8b4a4d0bc712e +Subproject commit d9f9e69960385eec7e3f7d0582b932d143014ae1 From 5bbbb5d214bb7d8792316559f47772351d9ea4f8 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 10:51:13 +0800 Subject: [PATCH 076/124] restore submodule refpoint --- tools/taos-tools | 2 +- tools/taosadapter | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/taos-tools b/tools/taos-tools index 50b68d85f7..bd496f76b6 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 50b68d85f7cbaf7a9adfa4082e88ca758770f75e +Subproject commit bd496f76b64931c66da2f8b0f24143a98a881cde diff --git a/tools/taosadapter b/tools/taosadapter index d9f9e69960..df8678f070 160000 --- a/tools/taosadapter +++ b/tools/taosadapter @@ -1 +1 @@ -Subproject commit d9f9e69960385eec7e3f7d0582b932d143014ae1 +Subproject commit df8678f070e3f707faf59baebec90065f6e1268b From 8228808a2ad80dc66a66c08924f46219527fd953 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 16 Jul 2022 03:12:47 +0000 Subject: [PATCH 077/124] more vnode snapshot writer --- source/dnode/vnode/src/meta/metaSnapshot.c | 20 +++-------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 23 +++++-------- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 40 ++++++++-------------- 3 files changed, 26 insertions(+), 57 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 900416f1b8..2143a3f085 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -131,18 +131,6 @@ struct SMetaSnapWriter { int64_t ever; }; -static int32_t metaSnapRollback(SMetaSnapWriter* pWriter) { - int32_t code = 0; - // TODO - return code; -} - -static int32_t metaSnapCommit(SMetaSnapWriter* pWriter) { - int32_t code = 0; - // TODO - return code; -} - int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) { int32_t code = 0; SMetaSnapWriter* pWriter; @@ -171,10 +159,9 @@ int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) { SMetaSnapWriter* pWriter = *ppWriter; if (rollback) { - code = metaSnapRollback(pWriter); - if (code) goto _err; + ASSERT(0); } else { - code = metaSnapCommit(pWriter); + code = metaCommit(pWriter->pMeta); if (code) goto _err; } taosMemoryFree(pWriter); @@ -199,9 +186,10 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) code = metaHandleEntry(pMeta, &metaEntry); if (code) goto _err; + tDecoderClear(pDecoder); return code; _err: - metaError("vgId:%d meta snapshot write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); + metaError("vgId:%d vnode snapshot meta write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index e561bb9d22..8ca921c6e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -399,18 +399,6 @@ struct STsdbSnapWriter { SArray* aDelIdxW; }; -static int32_t tsdbSnapRollback(STsdbSnapWriter* pWriter) { - int32_t code = 0; - // TODO - return code; -} - -static int32_t tsdbSnapCommit(STsdbSnapWriter* pWriter) { - int32_t code = 0; - // TODO - return code; -} - static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; int32_t iRow = 0; // todo @@ -738,6 +726,7 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { } // reader + pWriter->pBlockIdx = NULL; if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { ASSERT(pWriter->pDataFReader); @@ -1104,6 +1093,9 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr goto _err; } + code = tsdbFSBegin(pTsdb->fs); + if (code) goto _err; + *ppWriter = pWriter; return code; @@ -1118,7 +1110,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { STsdbSnapWriter* pWriter = *ppWriter; if (rollback) { - code = tsdbSnapRollback(pWriter); + code = tsdbFSRollback(pWriter->pTsdb->fs); if (code) goto _err; } else { code = tsdbSnapWriteDataEnd(pWriter); @@ -1127,7 +1119,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { code = tsdbSnapWriteDelEnd(pWriter); if (code) goto _err; - code = tsdbSnapCommit(pWriter); + code = tsdbFSCommit(pWriter->pTsdb->fs); if (code) goto _err; } @@ -1137,7 +1129,8 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { return code; _err: - tsdbError("vgId:%d tsdb snapshot writer close failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb writer close failed since %s", TD_VID(pWriter->pTsdb->pVnode), + tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 674beb51ec..c42c080fb8 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -149,18 +149,6 @@ struct SVSnapWriter { STsdbSnapWriter *pTsdbSnapWriter; }; -static int32_t vnodeSnapRollback(SVSnapWriter *pWriter) { - int32_t code = 0; - // TODO - return code; -} - -static int32_t vnodeSnapCommit(SVSnapWriter *pWriter) { - int32_t code = 0; - // TODO - return code; -} - int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) { int32_t code = 0; SVSnapWriter *pWriter = NULL; @@ -188,15 +176,15 @@ _err: int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) { int32_t code = 0; - // TODO + if (pWriter->pMetaSnapWriter) { + code = metaSnapWriterClose(&pWriter->pMetaSnapWriter, rollback); + if (code) goto _err; + } - // if (rollback) { - // code = vnodeSnapRollback(pWriter); - // if (code) goto _err; - // } else { - // code = vnodeSnapCommit(pWriter); - // if (code) goto _err; - // } + if (pWriter->pTsdbSnapWriter) { + code = tsdbSnapWriterClose(&pWriter->pTsdbSnapWriter, rollback); + if (code) goto _err; + } _exit: vInfo("vgId:%d vnode snapshot writer closed, rollback:%d", TD_VID(pWriter->pVnode), rollback); @@ -223,13 +211,13 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { if (pHdr->type == 0) { // meta - // if (pWriter->pMetaSnapWriter == NULL) { - // code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter); - // if (code) goto _err; - // } + if (pWriter->pMetaSnapWriter == NULL) { + code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter); + if (code) goto _err; + } - // code = metaSnapWrite(pWriter->pMetaSnapWriter, pData , nData); - // if (code) goto _err; + code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData); + if (code) goto _err; } else { // tsdb From 598c57e05966b82e577c366bf94d33875c82a2e1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 11:26:04 +0800 Subject: [PATCH 078/124] feat(query): add irate function scalar version TD-17344 --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 26 ++++++++++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index e0beaff262..d743a9ac02 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -107,6 +107,7 @@ int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPa int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index c458e2caec..99bcc2cb59 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2217,6 +2217,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getIrateFuncEnv, .initFunc = irateFuncSetup, .processFunc = irateFunction, + .sprocessFunc = irateScalarFunction, .finalizeFunc = irateFinalize }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 9a8f570a52..87d1f0c683 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2380,3 +2380,29 @@ int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara pOutput->numOfRows = 1; return TSDB_CODE_SUCCESS; } + +int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int32_t type = GET_PARAM_TYPE(pInput); + //int64_t count = 0, sum = 0, qSum = 0; + bool hasNull = false; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + hasNull = true; + break; + } + } + + double *out = (double *)pOutputData->pData; + if (hasNull) { + colDataAppendNULL(pOutputData, 0); + } else { + *out = 0; + } + + pOutput->numOfRows = 1; + return TSDB_CODE_SUCCESS; +} From 566ab93f839db501d4a9bef816aa4b283aa0db52 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 11:37:05 +0800 Subject: [PATCH 079/124] add derivative scalar version --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index d743a9ac02..582fe5a09b 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -107,6 +107,7 @@ int32_t leastSQRScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPa int32_t percentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 99bcc2cb59..b9e2b6feaf 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2207,6 +2207,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getDerivativeFuncEnv, .initFunc = derivativeFuncSetup, .processFunc = derivativeFunction, + .sprocessFunc = derivativeScalarFunction, .finalizeFunc = functionFinalize }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 87d1f0c683..5ccc23c2d4 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2381,6 +2381,11 @@ int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara return TSDB_CODE_SUCCESS; } +int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + pOutput->numOfRows = 0; + return TSDB_CODE_SUCCESS; +} + int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; From ff07377fa37a68f51f4cc59c6340ea7b6de942b0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 16 Jul 2022 03:40:20 +0000 Subject: [PATCH 080/124] finish vnode snapshot --- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 8ca921c6e2..4886d874a9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -641,14 +641,15 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { ASSERT(c); - if (c < 0) break; + if (c > 0) { + pWriter->pBlockData = &pWriter->bDataR; + code = + tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); + if (code) goto _err; + pWriter->iRow = 0; - pWriter->pBlockData = &pWriter->bDataR; - code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); - if (code) goto _err; - pWriter->iRow = 0; - - pWriter->iBlock++; + pWriter->iBlock++; + } break; } } @@ -684,6 +685,8 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { return code; _err: + tsdbError("vgId:%d vnode snapshot tsdb write table data impl failed since %s", TD_VID(pWriter->pTsdb->pVnode), + tstrerror(code)); return code; } From 0ed38565c1814266c9fef41073bad5703624e56c Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Sat, 16 Jul 2022 11:44:11 +0800 Subject: [PATCH 081/124] doc: english version of cluster chapter and some refine for chinese version --- docs/en/10-cluster/01-deploy.md | 48 ++++------ docs/en/10-cluster/02-cluster-mgmt.md | 101 +++------------------ docs/en/10-cluster/03-ha-and-lb.md | 81 ----------------- docs/en/10-cluster/03-high-availability.md | 30 ++++++ docs/en/10-cluster/04-load-balance.md | 20 ++++ docs/zh/10-cluster/01-deploy.md | 21 +++-- docs/zh/10-cluster/02-cluster-mgmt.md | 10 +- docs/zh/10-cluster/03-high-availability.md | 2 +- docs/zh/10-cluster/04-load-balance.md | 8 +- 9 files changed, 104 insertions(+), 217 deletions(-) delete mode 100644 docs/en/10-cluster/03-ha-and-lb.md create mode 100644 docs/en/10-cluster/03-high-availability.md create mode 100644 docs/en/10-cluster/04-load-balance.md diff --git a/docs/en/10-cluster/01-deploy.md b/docs/en/10-cluster/01-deploy.md index 200da1be3f..d998fd6ad0 100644 --- a/docs/en/10-cluster/01-deploy.md +++ b/docs/en/10-cluster/01-deploy.md @@ -6,9 +6,9 @@ title: Deployment ### Step 1 -The FQDN of all hosts must be setup properly. For e.g. FQDNs may have to be configured in the /etc/hosts file on each host. You must confirm that each FQDN can be accessed from any other host. For e.g. you can do this by using the `ping` command. +The FQDN of all hosts must be setup properly. All FQDNs need to be configured in the /etc/hosts file on each host. You must confirm that each FQDN can be accessed from any other host, you can do this by using the `ping` command. -To get the hostname on any host, the command `hostname -f` can be executed. `ping ` command can be executed on each host to check whether any other host is accessible from it. If any host is not accessible, the network configuration, like /etc/hosts or DNS configuration, needs to be checked and revised, to make any two hosts accessible to each other. +The command `hostname -f` can be executed to get the hostname on any host. `ping ` command can be executed on each host to check whether any other host is accessible from it. If any host is not accessible, the network configuration, like /etc/hosts or DNS configuration, needs to be checked and revised, to make any two hosts accessible to each other. :::note @@ -20,7 +20,7 @@ To get the hostname on any host, the command `hostname -f` can be executed. `pin ### Step 2 -If any previous version of TDengine has been installed and configured on any host, the installation needs to be removed and the data needs to be cleaned up. For details about uninstalling please refer to [Install and Uninstall](/operation/pkg-install). To clean up the data, please use `rm -rf /var/lib/taos/\*` assuming the `dataDir` is configured as `/var/lib/taos`. +If any previous version of TDengine has been installed and configured on any host, the installation needs to be removed and the data needs to be cleaned up. For details about uninstalling please refer to [Install and Uninstall](/operation/pkg-install). To clean up the data, please use `rm -rf /var/lib/taos/\*` assuming the `dataDir` is configured as `/var/lib/taos`. :::note @@ -54,22 +54,12 @@ serverPort 6030 For all the dnodes in a TDengine cluster, the below parameters must be configured exactly the same, any node whose configuration is different from dnodes already in the cluster can't join the cluster. -| **#** | **Parameter** | **Definition** | -| ----- | ------------------ | --------------------------------------------------------------------------------- | -| 1 | numOfMnodes | The number of management nodes in the cluster | -| 2 | mnodeEqualVnodeNum | The ratio of resource consuming of mnode to vnode | -| 3 | offlineThreshold | The threshold of dnode offline, once it's reached the dnode is considered as down | -| 4 | statusInterval | The interval by which dnode reports its status to mnode | -| 5 | arbitrator | End point of the arbitrator component in the cluster | -| 6 | timezone | Timezone | -| 7 | balance | Enable load balance automatically | -| 8 | maxTablesPerVnode | Maximum number of tables that can be created in each vnode | -| 9 | maxVgroupsPerDb | Maximum number vgroups that can be used by each DB | - -:::note -Prior to version 2.0.19.0, besides the above parameters, `locale` and `charset` must also be configured the same for each dnode. - -::: +| **#** | **Parameter** | **Definition** | +| ----- | -------------- | ------------------------------------------------------------- | +| 1 | statusInterval | The time interval for which dnode reports its status to mnode | +| 2 | timezone | Time Zone where the server is located | +| 3 | locale | Location code of the system | +| 4 | charset | Character set of the system | ## Start Cluster @@ -77,19 +67,19 @@ In the following example we assume that first dnode has FQDN h1.taosdata.com and ### Start The First DNODE -The first dnode can be started following the instructions in [Get Started](/get-started/). Then TDengine CLI `taos` can be launched to execute command `show dnodes`, the output is as following for example: +Start the first dnode following the instructions in [Get Started](/get-started/). Then launch TDengine CLI `taos` and execute command `show dnodes`, the output is as following for example: ``` -Welcome to the TDengine shell from Linux, Client Version:2.0.0.0 +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. - -Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. +Server is Enterprise trial Edition, ver:3.0.0.0 and will never expire. taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | -===================================================================================== - 1 | h1.taosdata.com:6030 | 0 | 2 | ready | any | 2020-07-31 03:49:29.202 | -Query OK, 1 row(s) in set (0.006385s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | h1.taosdata.com:6030 | 0 | 1024 | ready | 2022-07-16 10:50:42.673 | | +Query OK, 1 rows affected (0.007984s) taos> ``` @@ -100,7 +90,7 @@ From the above output, it is shown that the end point of the started dnode is "h There are a few steps necessary to add other dnodes in the cluster. -Let's assume we are starting the second dnode with FQDN, h2.taosdata.com. First we make sure the configuration is correct. +Let's assume we are starting the second dnode with FQDN, h2.taosdata.com. Firstly we make sure the configuration is correct. ```c // firstEp is the end point to connect to when any dnode starts @@ -114,7 +104,7 @@ serverPort 6030 ``` -Second, we can start `taosd` as instructed in [Get Started](/get-started/). +Secondly, we can start `taosd` as instructed in [Get Started](/get-started/). Then, on the first dnode i.e. h1.taosdata.com in our example, use TDengine CLI `taos` to execute the following command to add the end point of the dnode in the cluster. In the command "fqdn:port" should be quoted using double quotes. diff --git a/docs/en/10-cluster/02-cluster-mgmt.md b/docs/en/10-cluster/02-cluster-mgmt.md index bd3386c411..19ee034127 100644 --- a/docs/en/10-cluster/02-cluster-mgmt.md +++ b/docs/en/10-cluster/02-cluster-mgmt.md @@ -39,31 +39,25 @@ USE SOME_DATABASE; SHOW VGROUPS; ``` -The example output is below: - -``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | localhost:6030 | 9 | 8 | ready | any | 2022-04-15 08:27:09.359 | | -Query OK, 1 row(s) in set (0.008298s) +The output is like below: taos> use db; Database changed. taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | compacting | +vgId | tables | status | onlines | v1_dnode | v1_status | compacting | ========================================================================================== - 14 | 38000 | ready | 1 | 1 | leader | 0 | - 15 | 38000 | ready | 1 | 1 | leader | 0 | - 16 | 38000 | ready | 1 | 1 | leader | 0 | - 17 | 38000 | ready | 1 | 1 | leader | 0 | - 18 | 37001 | ready | 1 | 1 | leader | 0 | - 19 | 37000 | ready | 1 | 1 | leader | 0 | - 20 | 37000 | ready | 1 | 1 | leader | 0 | - 21 | 37000 | ready | 1 | 1 | leader | 0 | +14 | 38000 | ready | 1 | 1 | leader | 0 | +15 | 38000 | ready | 1 | 1 | leader | 0 | +16 | 38000 | ready | 1 | 1 | leader | 0 | +17 | 38000 | ready | 1 | 1 | leader | 0 | +18 | 37001 | ready | 1 | 1 | leader | 0 | +19 | 37000 | ready | 1 | 1 | leader | 0 | +20 | 37000 | ready | 1 | 1 | leader | 0 | +21 | 37000 | ready | 1 | 1 | leader | 0 | Query OK, 8 row(s) in set (0.001154s) -``` + +```` ## Add DNODE @@ -71,7 +65,7 @@ Launch TDengine CLI `taos` and execute the command below to add the end point of ```sql CREATE DNODE "fqdn:port"; -``` +```` The example output is as below: @@ -142,72 +136,3 @@ In the above example, when `show dnodes` is executed the first time, two dnodes - dnodeID is allocated automatically and can't be manually modified. dnodeID is generated in ascending order without duplication. ::: - -## Move VNODE - -A vnode can be manually moved from one dnode to another. - -Launch TDengine CLI `taos` and execute below command: - -```sql -ALTER DNODE BALANCE "VNODE:-DNODE:"; -``` - -In the above command, `source-dnodeId` is the original dnodeId where the vnode resides, `dest-dnodeId` specifies the target dnode. vgId (vgroup ID) can be shown by `SHOW VGROUPS `. - -First `show vgroups` is executed to show the vgroup distribution. - -``` -taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | compacting | -========================================================================================== - 14 | 38000 | ready | 1 | 3 | leader | 0 | - 15 | 38000 | ready | 1 | 3 | leader | 0 | - 16 | 38000 | ready | 1 | 3 | leader | 0 | - 17 | 38000 | ready | 1 | 3 | leader | 0 | - 18 | 37001 | ready | 1 | 3 | leader | 0 | - 19 | 37000 | ready | 1 | 1 | leader | 0 | - 20 | 37000 | ready | 1 | 1 | leader | 0 | - 21 | 37000 | ready | 1 | 1 | leader | 0 | -Query OK, 8 row(s) in set (0.001314s) -``` - -It can be seen that there are 5 vgroups in dnode 3 and 3 vgroups in node 1, now we want to move vgId 18 from dnode 3 to dnode 1. Execute the below command in `taos` - -``` -taos> alter dnode 3 balance "vnode:18-dnode:1"; - -DB error: Balance already enabled (0.00755 -``` - -However, the operation fails with error message show above, which means automatic load balancing has been enabled in the current database so manual load balance can't be performed. - -Shutdown the cluster, configure `balance` parameter in all the dnodes to 0, then restart the cluster, and execute `alter dnode` and `show vgroups` as below. - -``` -taos> alter dnode 3 balance "vnode:18-dnode:1"; -Query OK, 0 row(s) in set (0.000575s) - -taos> show vgroups; - vgId | tables | status | onlines | v1_dnode | v1_status | v2_dnode | v2_status | compacting | -================================================================================================================= - 14 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | - 15 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | - 16 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | - 17 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | - 18 | 37001 | ready | 2 | 1 | follower | 3 | leader | 0 | - 19 | 37000 | ready | 1 | 1 | leader | 0 | NULL | 0 | - 20 | 37000 | ready | 1 | 1 | leader | 0 | NULL | 0 | - 21 | 37000 | ready | 1 | 1 | leader | 0 | NULL | 0 | -Query OK, 8 row(s) in set (0.001242s) -``` - -It can be seen from above output that vgId 18 has been moved from dnode 3 to dnode 1. - -:::note - -- Manual load balancing can only be performed when the automatic load balancing is disabled, i.e. `balance` is set to 0. -- Only a vnode in normal state, i.e. leader or follower, can be moved. vnode can't be moved when its in status offline, unsynced or syncing. -- Before moving a vnode, it's necessary to make sure the target dnode has enough resources: CPU, memory and disk. - -::: diff --git a/docs/en/10-cluster/03-ha-and-lb.md b/docs/en/10-cluster/03-ha-and-lb.md deleted file mode 100644 index 9780e8f6c6..0000000000 --- a/docs/en/10-cluster/03-ha-and-lb.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -sidebar_label: HA & LB -title: High Availability and Load Balancing ---- - -## High Availability of Vnode - -High availability of vnode and mnode can be achieved through replicas in TDengine. - -A TDengine cluster can have multiple databases. Each database has a number of vnodes associated with it. A different number of replicas can be configured for each DB. When creating a database, the parameter `replica` is used to specify the number of replicas. The default value for `replica` is 1. Naturally, a single replica cannot guarantee high availability since if one node is down, the data service is unavailable. Note that the number of dnodes in the cluster must NOT be lower than the number of replicas set for any DB, otherwise the `create table` operation will fail with error "more dnodes are needed". The SQL statement below is used to create a database named "demo" with 3 replicas. - -```sql -CREATE DATABASE demo replica 3; -``` - -The data in a DB is divided into multiple shards and stored in multiple vgroups. The number of vnodes in each vgroup is determined by the number of replicas set for the DB. The vnodes in each vgroup store exactly the same data. For the purpose of high availability, the vnodes in a vgroup must be located in different dnodes on different hosts. As long as over half of the vnodes in a vgroup are in an online state, the vgroup is able to provide data access. Otherwise the vgroup can't provide data access for reading or inserting data. - -There may be data for multiple DBs in a dnode. When a dnode is down, multiple DBs may be affected. While in theory, the cluster will provide data access for reading or inserting data if over half the vnodes in vgroups are online, because of the possibly complex mapping between vnodes and dnodes, it is difficult to guarantee that the cluster will work properly if over half of the dnodes are online. - -## High Availability of Mnode - -Each TDengine cluster is managed by `mnode`, which is a module of `taosd`. For the high availability of mnode, multiple mnodes can be configured using system parameter `numOfMNodes`. The valid range for `numOfMnodes` is [1,3]. To ensure data consistency between mnodes, data replication between mnodes is performed synchronously. - -There may be multiple dnodes in a cluster, but only one mnode can be started in each dnode. Which one or ones of the dnodes will be designated as mnodes is automatically determined by TDengine according to the cluster configuration and system resources. The command `show mnodes` can be executed in TDengine `taos` to show the mnodes in the cluster. - -```sql -SHOW MNODES; -``` - -The end point and role/status (leader, follower, unsynced, or offline) of all mnodes can be shown by the above command. When the first dnode is started in a cluster, there must be one mnode in this dnode. Without at least one mnode, the cluster cannot work. If `numOfMNodes` is configured to 2, another mnode will be started when the second dnode is launched. - -For the high availability of mnode, `numOfMnodes` needs to be configured to 2 or a higher value. Because the data consistency between mnodes must be guaranteed, the replica confirmation parameter `quorum` is set to 2 automatically if `numOfMNodes` is set to 2 or higher. - -:::note -If high availability is important for your system, both vnode and mnode must be configured to have multiple replicas. - -::: - -## Load Balancing - -Load balancing will be triggered in 3 cases without manual intervention. - -- When a new dnode joins the cluster, automatic load balancing may be triggered. Some data from other dnodes may be transferred to the new dnode automatically. -- When a dnode is removed from the cluster, the data from this dnode will be transferred to other dnodes automatically. -- When a dnode is too hot, i.e. too much data has been stored in it, automatic load balancing may be triggered to migrate some vnodes from this dnode to other dnodes. - -:::tip -Automatic load balancing is controlled by the parameter `balance`, 0 means disabled and 1 means enabled. This is set in the file [taos.cfg](https://docs.tdengine.com/reference/config/#balance). - -::: - -## Dnode Offline - -When a dnode is offline, it can be detected by the TDengine cluster. There are two cases: - -- The dnode comes online before the threshold configured in `offlineThreshold` is reached. The dnode is still in the cluster and data replication is started automatically. The dnode can work properly after the data sync is finished. - -- If the dnode has been offline over the threshold configured in `offlineThreshold` in `taos.cfg`, the dnode will be removed from the cluster automatically. A system alert will be generated and automatic load balancing will be triggered if `balance` is set to 1. When the removed dnode is restarted and becomes online, it will not join the cluster automatically. The system administrator has to manually join the dnode to the cluster. - -:::note -If all the vnodes in a vgroup (or mnodes in mnode group) are in offline or unsynced status, the leader node can only be voted on, after all the vnodes or mnodes in the group become online and can exchange status. Following this, the vgroup (or mnode group) is able to provide service. - -::: - -## Arbitrator - -The "arbitrator" component is used to address the special case when the number of replicas is set to an even number like 2,4 etc. If half of the vnodes in a vgroup don't work, it is impossible to vote and select a leader node. This situation also applies to mnodes if the number of mnodes is set to an even number like 2,4 etc. - -To resolve this problem, a new arbitrator component named `tarbitrator`, an abbreviation of TDengine Arbitrator, was introduced. The `tarbitrator` simulates a vnode or mnode but it's only responsible for network communication and doesn't handle any actual data access. As long as more than half of the vnode or mnode, including Arbitrator, are available the vnode group or mnode group can provide data insertion or query services normally. - -Normally, it's prudent to configure the replica number for each DB or system parameter `numOfMNodes` to be an odd number. However, if a user is very sensitive to storage space, a replica number of 2 plus arbitrator component can be used to achieve both lower cost of storage space and high availability. - -Arbitrator component is installed with the server package. For details about how to install, please refer to [Install](/operation/pkg-install). The `-p` parameter of `tarbitrator` can be used to specify the port on which it provides service. - -In the configuration file `taos.cfg` of each dnode, parameter `arbitrator` needs to be configured to the end point of the `tarbitrator` process. Arbitrator component will be used automatically if the replica is configured to an even number and will be ignored if the replica is configured to an odd number. - -Arbitrator can be shown by executing command in TDengine CLI `taos` with its role shown as "arb". - -```sql -SHOW DNODES; -``` diff --git a/docs/en/10-cluster/03-high-availability.md b/docs/en/10-cluster/03-high-availability.md new file mode 100644 index 0000000000..e2e1c6521e --- /dev/null +++ b/docs/en/10-cluster/03-high-availability.md @@ -0,0 +1,30 @@ +--- +sidebar_label: High Availability +title: High Availability +--- + +## High Availability of Vnode + +High availability of vnode can be achieved through replicas in TDengine. + +A TDengine cluster can have multiple databases. Each database has a number of vnodes associated with it. A different number of replicas can be configured for each DB. When creating a database, the parameter `replica` is used to specify the number of replicas. The default value for `replica` is 1. Naturally, a single replica cannot guarantee high availability since if one node is down, the data service is unavailable. Note that the number of dnodes in the cluster must NOT be lower than the number of replicas set for any DB, otherwise the `create table` operation will fail with error "more dnodes are needed". The SQL statement below is used to create a database named "demo" with 3 replicas. + +```sql +CREATE DATABASE demo replica 3; +``` + +The data in a DB is divided into multiple shards and stored in multiple vgroups. The number of vnodes in each vgroup is determined by the number of replicas set for the DB. The vnodes in each vgroup store exactly the same data. For the purpose of high availability, the vnodes in a vgroup must be located in different dnodes on different hosts. As long as over half of the vnodes in a vgroup are in an online state, the vgroup is able to provide data access. Otherwise the vgroup can't provide data access for reading or inserting data. + +There may be data for multiple DBs in a dnode. When a dnode is down, multiple DBs may be affected. While in theory, the cluster will provide data access for reading or inserting data if over half the vnodes in vgroups are online, because of the possibly complex mapping between vnodes and dnodes, it is difficult to guarantee that the cluster will work properly if over half of the dnodes are online. + +## High Availability of Mnode + +Each TDengine cluster is managed by `mnode`, which is a module of `taosd`. For the high availability of mnode, multiple mnodes can be configured in the system. When a TDengine cluster is started from scratch, there is only one `mnode`, then you can use command `create mnode` to and start corresponding dnode to add more mnodes. + +```sql +SHOW MNODES; +``` + +The end point and role/status (leader, follower, candidate, offline) of all mnodes can be shown by the above command. When the first dnode is started in a cluster, there must be one mnode in this dnode. Without at least one mnode, the cluster cannot work. + +From TDengine 3.0.0, RAFT procotol is used to guarantee the high availability, so the number of mnodes is should be 1 or 3. diff --git a/docs/en/10-cluster/04-load-balance.md b/docs/en/10-cluster/04-load-balance.md new file mode 100644 index 0000000000..7648398059 --- /dev/null +++ b/docs/en/10-cluster/04-load-balance.md @@ -0,0 +1,20 @@ +--- +sidebar_label: Load Balance +title: Load Balance +--- + +The load balance in TDengine is mainly about processing data series data. TDengine employes builtin hash algorithm to distribute all the tables, sub-tables and their data of a database across all the vgroups that belongs to the database. Each table or sub-table can only be handled by a single vgroup, while each vgroup can process multiple table or sub-table. + +The number of vgroup can be specified when creating a database, using the parameter `vgroups`. + +```sql +create database db0 vgroups 100; +``` + +The proper value of `vgroups` depends on available system resources. Assuming there is only one database to be created in the system, then the number of `vgroups` is determined by the available resources from all dnodes. In principle more vgroups can be created if you have more CPU and memory. Disk I/O is another important factor to consider. Once the bottleneck shows on disk I/O, more vgroups may downgrad the system performance significantly. If multiple databases are to be created in the system, then the total number of `vroups` of all the databases are dependent on the available system resources. It needs to be careful to distribute vgroups among these databases, you need to consider the number of tables, data writing frequency, size of each data row for all these databases. A recommended practice is to firstly choose a starting number for `vgroups`, for example double of the number of CPU cores, then try to adjust and optimize system configurations to find the best setting for `vgroups`, then distribute these vgroups among databases. + +Furthermode, TDengine distributes the vgroups of each database equally among all dnodes. In case of replica 3, the distrubtion is even more complex, TDengine tries its best to prevent any dnode from becoming a bottleneck. + +TDegnine utilizes the above ways to achieve load balance in a cluster, and finally achieve higher throughput. + +Once the load balance is achieved, after some operations like deleting tables or droping databases, the load across all dnodes may become inbalanced, the method of rebalance will be provided in later versions. However, even without explicit rebalancing, TDengine will try its best to achieve new balance without manual interfering when a new database is created. \ No newline at end of file diff --git a/docs/zh/10-cluster/01-deploy.md b/docs/zh/10-cluster/01-deploy.md index 9dbfb50f18..cd19f90ba1 100644 --- a/docs/zh/10-cluster/01-deploy.md +++ b/docs/zh/10-cluster/01-deploy.md @@ -72,19 +72,22 @@ serverPort 6030 按照《立即开始》里的步骤，启动第一个数据节点，例如 h1.taosdata.com，然后执行 taos，启动 taos shell，从 shell 里执行命令“SHOW DNODES”，如下所示： ``` -Welcome to the TDengine shell from Linux, Client Version:2.0.0.0 +Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 +Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. - -Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. +Server is Enterprise trial Edition, ver:3.0.0.0 and will never expire. taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | -===================================================================================== - 1 | h1.taos.com:6030 | 0 | 2 | ready | any | 2020-07-31 03:49:29.202 | -Query OK, 1 row(s) in set (0.006385s) + id | endpoint | vnodes | support_vnodes | status | create_time | note | +============================================================================================================================================ + 1 | h1.taosdata.com:6030 | 0 | 1024 | ready | 2022-07-16 10:50:42.673 | | +Query OK, 1 rows affected (0.007984s) taos> -``` + +taos> + +```` 上述命令里，可以看到刚启动的数据节点的 End Point 是：h1.taos.com:6030，就是这个新集群的 firstEp。 @@ -98,7 +101,7 @@ taos> ```sql CREATE DNODE "h2.taos.com:6030"; -``` +```` 将新数据节点的 End Point（准备工作中第四步获知的）添加进集群的 EP 列表。“fqdn:port”需要用双引号引起来，否则出错。请注意将示例的“h2.taos.com:6030” 替换为这个新数据节点的 End Point。 diff --git a/docs/zh/10-cluster/02-cluster-mgmt.md b/docs/zh/10-cluster/02-cluster-mgmt.md index f02a105826..5c490516f0 100644 --- a/docs/zh/10-cluster/02-cluster-mgmt.md +++ b/docs/zh/10-cluster/02-cluster-mgmt.md @@ -72,10 +72,11 @@ CREATE DNODE "fqdn:port"; taos> show dnodes; id | endpoint | vnodes | support_vnodes | status | create_time | note | ============================================================================================================================================ - 1 | trd01:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | - 2 | trd04:6030 | 0 | 1024 | ready | 2022-07-15 16:56:13.670 | | + 1 | localhost:6030 | 100 | 1024 | ready | 2022-07-15 16:47:47.726 | | + 2 | localhost:7030 | 0 | 1024 | ready | 2022-07-15 16:56:13.670 | | Query OK, 2 rows affected (0.007031s) ``` + 从中可以看到两个 dnode 状态都为 ready ## 删除数据节点 @@ -85,14 +86,15 @@ Query OK, 2 rows affected (0.007031s) ```sql DROP DNODE "fqdn:port"; ``` + 或者 + ```sql DROP DNODE dnodeId; ``` 通过 “fqdn:port” 或 dnodeID 来指定一个具体的节点都是可以的。其中 fqdn 是被删除的节点的 FQDN，port 是其对外服务器的端口号；dnodeID 可以通过 SHOW DNODES 获得。 - :::warning 数据节点一旦被 drop 之后，不能重新加入集群。需要将此节点重新部署（清空数据文件夹）。集群在完成 `drop dnode` 操作之前，会将该 dnode 的数据迁移走。 @@ -101,5 +103,3 @@ DROP DNODE dnodeId; dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增的，不会重复。 ::: - - diff --git a/docs/zh/10-cluster/03-high-availability.md b/docs/zh/10-cluster/03-high-availability.md index 36f9300366..ba056b6f16 100644 --- a/docs/zh/10-cluster/03-high-availability.md +++ b/docs/zh/10-cluster/03-high-availability.md @@ -28,6 +28,6 @@ TDengine 集群是由 mnode（taosd 的一个模块，管理节点）负责管 SHOW MNODES; ``` -来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（leader, follower, candidate）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。 +来查看 mnode 列表，该列表将列出 mnode 所处的 dnode 的 End Point 和角色（leader, follower, candidate, offline）。当集群中第一个数据节点启动时，该数据节点一定会运行一个 mnode 实例，否则该数据节点 dnode 无法正常工作，因为一个系统是必须有至少一个 mnode 的。 在 TDengine 3.0 及以后的版本中，数据同步采用 RAFT 协议，所以 mnode 的数量应该被设置为 1 个或者 3 个。 diff --git a/docs/zh/10-cluster/04-load-balance.md b/docs/zh/10-cluster/04-load-balance.md index c1a72cf006..2376dd3e61 100644 --- a/docs/zh/10-cluster/04-load-balance.md +++ b/docs/zh/10-cluster/04-load-balance.md @@ -2,17 +2,17 @@ title: 负载均衡 --- -TDengine 中的负载均衡主要指对时序数据的处理的负载均衡。TDengine 采用 Hash 一致性算法将一个数据库中的所有表和子表的数据均衡分散在属于该数据库的所有 vgroups 中，每张表或子表只能由一个 vgroups 处理，一个 vgroups 可能负责处理多个表或子表。 +TDengine 中的负载均衡主要指对时序数据的处理的负载均衡。TDengine 采用 Hash 一致性算法将一个数据库中的所有表和子表的数据均衡分散在属于该数据库的所有 vgroup 中，每张表或子表只能由一个 vgroup 处理，一个 vgroup 可能负责处理多个表或子表。 -创建数据库时可以指定其中的 vgroups 的数量： +创建数据库时可以指定其中的 vgroup 的数量： ```sql create database db0 vgroups 100; ``` -如何指定合适的 vgroups 的数量，这取决于系统资源。假定系统中只计划建立一个数据库，则 vgroups 由集群中所有 dnode 所能使用的资源决定。原则上可用的 CPU 和 Memory 越多，可建立的 vgroups 也越多。但也要考虑到磁盘性能，过多的 vgroups 在磁盘性能达到上限后反而会拖累整个系统的性能。假如系统中会建立多个数据库，则多个数据库的 vgoups 之和取决于系统中可用资源的数量。要综合考虑多个数据库之间表的数量、写入频率、数据量等多个因素在多个数据库之间分配 vgroups。实际中建议首先根据系统资源配置选择一个初始的 vgroups 数量，比如 CPU 总核数的 2 倍，以此为起点通过测试找到最佳的 vgroups 数量配置，此为系统中的 vgroups 总数。如果有多个数据库的话，再根据各个数据库的表数和数据量对 vgroups 进行分配。 +如何指定合适的 vgroup 的数量，这取决于系统资源。假定系统中只计划建立一个数据库，则 vgroup 数量由集群中所有 dnode 所能使用的资源决定。原则上可用的 CPU 和 Memory 越多，可建立的 vgroup 也越多。但也要考虑到磁盘性能，过多的 vgroup 在磁盘性能达到上限后反而会拖累整个系统的性能。假如系统中会建立多个数据库，则多个数据库的 vgroup 之和取决于系统中可用资源的数量。要综合考虑多个数据库之间表的数量、写入频率、数据量等多个因素在多个数据库之间分配 vgroup。实际中建议首先根据系统资源配置选择一个初始的 vgroup 数量，比如 CPU 总核数的 2 倍，以此为起点通过测试找到最佳的 vgroup 数量配置，此为系统中的 vgroup 总数。如果有多个数据库的话，再根据各个数据库的表数和数据量对 vgroup 进行分配。 -此外，对于任意数据库的 vgroups，TDengine 都是尽可能将其均衡分散在多个 dnode 上。在多副本情况下（replica 3），这种均衡分布尤其复杂，TDengine 的分布策略会尽量避免任意一个 dnode 成为写入的瓶颈。 +此外，对于任意数据库的 vgroup，TDengine 都是尽可能将其均衡分散在多个 dnode 上。在多副本情况下（replica 3），这种均衡分布尤其复杂，TDengine 的分布策略会尽量避免任意一个 dnode 成为写入的瓶颈。 通过以上措施可以最大限度地在整个 TDengine 集群中实现负载均衡，负载均衡也能反过来提升系统总的数据处理能力。 From ee5d2c1d8248a1d9c0baf8de3ba17a9d1546fb55 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 11:46:49 +0800 Subject: [PATCH 082/124] refactor --- source/libs/scalar/src/sclfunc.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 5ccc23c2d4..13b29f8ae3 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2381,17 +2381,11 @@ int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara return TSDB_CODE_SUCCESS; } -int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - pOutput->numOfRows = 0; - return TSDB_CODE_SUCCESS; -} - -int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { +int32_t nonCalcScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; int32_t type = GET_PARAM_TYPE(pInput); - //int64_t count = 0, sum = 0, qSum = 0; bool hasNull = false; for (int32_t i = 0; i < pInput->numOfRows; ++i) { @@ -2411,3 +2405,11 @@ int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam pOutput->numOfRows = 1; return TSDB_CODE_SUCCESS; } + +int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return nonCalcScalarFunction(pInput, inputNum, pOutput); +} + +int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return nonCalcScalarFunction(pInput, inputNum, pOutput); +} From 7c585d8ed9d12b0af10949d45a85a854c1af21bc Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 16 Jul 2022 11:47:26 +0800 Subject: [PATCH 083/124] fix: database options 'strict' and 'cachelast' syntax adjustments --- include/common/ttokendef.h | 4 +- include/libs/nodes/cmdnodes.h | 4 +- include/util/tdef.h | 33 ++-- source/common/src/systable.c | 6 +- source/dnode/mnode/impl/src/mndDb.c | 31 +++- source/libs/nodes/src/nodesCodeFuncs.c | 6 +- source/libs/parser/inc/parAst.h | 4 +- source/libs/parser/inc/sql.y | 10 +- source/libs/parser/src/parAstCreater.c | 18 +- source/libs/parser/src/parTokenizer.c | 4 +- source/libs/parser/src/parTranslater.c | 48 +++++- source/libs/parser/src/sql.c | 172 ++++++++++---------- source/libs/parser/test/parInitialATest.cpp | 2 +- source/libs/parser/test/parInitialCTest.cpp | 15 +- tests/pytest/alter/alter_cacheLastRow.py | 4 +- tests/pytest/query/last_cache.py | 12 +- tests/pytest/query/last_row_cache.py | 66 ++++---- tests/pytest/util/boundary.py | 2 +- tests/script/tsim/db/alter_option.sim | 12 +- tests/script/tsim/db/basic6.sim | 2 +- tests/script/tsim/db/create_all_options.sim | 2 +- tests/script/tsim/parser/last_cache.sim | 2 +- tests/script/tsim/parser/like.sim | 2 +- tests/system-test/2-query/last_row.py | 8 +- tests/system-test/6-cluster/5dnode1mnode.py | 6 +- 25 files changed, 267 insertions(+), 208 deletions(-) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index e2089f3023..0fce573af9 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -79,8 +79,8 @@ #define TK_NOT 61 #define TK_EXISTS 62 #define TK_BUFFER 63 -#define TK_CACHELAST 64 -#define TK_CACHELASTSIZE 65 +#define TK_CACHEMODEL 64 +#define TK_CACHESIZE 65 #define TK_COMP 66 #define TK_DURATION 67 #define TK_NK_VARIABLE 68 diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 5311915612..e13f85002c 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -51,7 +51,8 @@ extern "C" { typedef struct SDatabaseOptions { ENodeType type; int32_t buffer; - int8_t cacheLast; + char cacheModelStr[TSDB_CACHE_MODEL_STR_LEN]; + int8_t cacheModel; int32_t cacheLastSize; int8_t compressionLevel; int32_t daysPerFile; @@ -66,6 +67,7 @@ typedef struct SDatabaseOptions { char precisionStr[3]; int8_t precision; int8_t replica; + char strictStr[TSDB_DB_STRICT_STR_LEN]; int8_t strict; int8_t walLevel; int32_t numOfVgroups; diff --git a/include/util/tdef.h b/include/util/tdef.h index 55194d8647..3b31398063 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -53,7 +53,7 @@ extern const int32_t TYPE_BYTES[16]; #define TSDB_DATA_BIGINT_NULL 0x8000000000000000LL #define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL -#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN #define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000LL // an NAN #define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF #define TSDB_DATA_BINARY_NULL 0xFF @@ -107,9 +107,10 @@ extern const int32_t TYPE_BYTES[16]; #define TSDB_INS_USER_STABLES_DBNAME_COLID 2 -#define TSDB_TICK_PER_SECOND(precision) \ - ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1000LL \ - : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1000000LL : 1000000000LL))) +#define TSDB_TICK_PER_SECOND(precision) \ + ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI \ + ? 1000LL \ + : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1000000LL : 1000000000LL))) #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) #define T_APPEND_MEMBER(dst, ptr, type, member) \ @@ -328,15 +329,25 @@ typedef enum ELogicConditionType { #define TSDB_MIN_DB_REPLICA 1 #define TSDB_MAX_DB_REPLICA 3 #define TSDB_DEFAULT_DB_REPLICA 1 +#define TSDB_DB_STRICT_STR_LEN sizeof(TSDB_DB_STRICT_OFF_STR) +#define TSDB_DB_STRICT_OFF_STR "off" +#define TSDB_DB_STRICT_ON_STR "on" #define TSDB_DB_STRICT_OFF 0 #define TSDB_DB_STRICT_ON 1 -#define TSDB_DEFAULT_DB_STRICT 0 -#define TSDB_MIN_DB_CACHE_LAST 0 -#define TSDB_MAX_DB_CACHE_LAST 3 -#define TSDB_DEFAULT_CACHE_LAST 0 -#define TSDB_MIN_DB_CACHE_LAST_SIZE 1 // MB -#define TSDB_MAX_DB_CACHE_LAST_SIZE 65536 -#define TSDB_DEFAULT_CACHE_LAST_SIZE 1 +#define TSDB_DEFAULT_DB_STRICT TSDB_DB_STRICT_OFF +#define TSDB_CACHE_MODEL_STR_LEN sizeof(TSDB_CACHE_MODEL_LAST_VALUE_STR) +#define TSDB_CACHE_MODEL_NONE_STR "none" +#define TSDB_CACHE_MODEL_LAST_ROW_STR "last_row" +#define TSDB_CACHE_MODEL_LAST_VALUE_STR "last_value" +#define TSDB_CACHE_MODEL_BOTH_STR "both" +#define TSDB_CACHE_MODEL_NONE 0 +#define TSDB_CACHE_MODEL_LAST_ROW 1 +#define TSDB_CACHE_MODEL_LAST_VALUE 2 +#define TSDB_CACHE_MODEL_BOTH 3 +#define TSDB_DEFAULT_CACHE_MODEL TSDB_CACHE_MODEL_NONE +#define TSDB_MIN_DB_CACHE_SIZE 1 // MB +#define TSDB_MAX_DB_CACHE_SIZE 65536 +#define TSDB_DEFAULT_CACHE_SIZE 1 #define TSDB_DB_STREAM_MODE_OFF 0 #define TSDB_DB_STREAM_MODE_ON 1 #define TSDB_DEFAULT_DB_STREAM_MODE 0 diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 08997bcaf8..c9c1c1b395 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -76,7 +76,7 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "replica", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "strict", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "strict", .bytes = TSDB_DB_STRICT_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "duration", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "keep", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "buffer", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, @@ -87,9 +87,9 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "wal", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "cache_model", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "cacheModel", .bytes = TSDB_CACHE_MODEL_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "single_stable_model", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, + {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, // {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, {.name = "retention", .bytes = 60 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index b1c2b0e277..715340f688 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -293,7 +293,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1; if (pCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1; if (pCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1; - if (pCfg->cacheLastSize < TSDB_MIN_DB_CACHE_LAST_SIZE || pCfg->cacheLastSize > TSDB_MAX_DB_CACHE_LAST_SIZE) return -1; + if (pCfg->cacheLastSize < TSDB_MIN_DB_CACHE_SIZE || pCfg->cacheLastSize > TSDB_MAX_DB_CACHE_SIZE) return -1; if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1; if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1; @@ -312,7 +312,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->replications != 1 && pCfg->replications != 3) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1; - if (pCfg->cacheLast < TSDB_MIN_DB_CACHE_LAST || pCfg->cacheLast > TSDB_MAX_DB_CACHE_LAST) return -1; + if (pCfg->cacheLast < TSDB_CACHE_MODEL_NONE || pCfg->cacheLast > TSDB_CACHE_MODEL_BOTH) return -1; if (pCfg->hashMethod != 1) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; @@ -341,8 +341,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL; if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA; if (pCfg->strict < 0) pCfg->strict = TSDB_DEFAULT_DB_STRICT; - if (pCfg->cacheLast < 0) pCfg->cacheLast = TSDB_DEFAULT_CACHE_LAST; - if (pCfg->cacheLastSize <= 0) pCfg->cacheLastSize = TSDB_DEFAULT_CACHE_LAST_SIZE; + if (pCfg->cacheLast < 0) pCfg->cacheLast = TSDB_DEFAULT_CACHE_MODEL; + if (pCfg->cacheLastSize <= 0) pCfg->cacheLastSize = TSDB_DEFAULT_CACHE_SIZE; if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0; if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF; } @@ -1443,6 +1443,22 @@ char *buildRetension(SArray *pRetension) { return p1; } +static const char *getCacheModelStr(int8_t cacheModel) { + switch (cacheModel) { + case TSDB_CACHE_MODEL_NONE: + return TSDB_CACHE_MODEL_NONE_STR; + case TSDB_CACHE_MODEL_LAST_ROW: + return TSDB_CACHE_MODEL_LAST_ROW_STR; + case TSDB_CACHE_MODEL_LAST_VALUE: + return TSDB_CACHE_MODEL_LAST_VALUE_STR; + case TSDB_CACHE_MODEL_BOTH: + return TSDB_CACHE_MODEL_BOTH_STR; + default: + break; + } + return "unknown"; +} + static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, bool sysDb, ESdbStatus objStatus, bool sysinfo) { int32_t cols = 0; @@ -1491,7 +1507,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false); - const char *strictStr = pDb->cfg.strict ? "strict" : "no_strict"; + const char *strictStr = pDb->cfg.strict ? "on" : "off"; char strictVstr[24] = {0}; STR_WITH_SIZE_TO_VARSTR(strictVstr, strictStr, strlen(strictStr)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1539,8 +1555,11 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false); + const char *cacheModelStr = getCacheModelStr(pDb->cfg.cacheLast); + char cacheModelVstr[24] = {0}; + STR_WITH_SIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, strlen(cacheModelStr)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLast, false); + colDataAppend(pColInfo, rows, (const char *)cacheModelVstr, false); const char *precStr = NULL; switch (pDb->cfg.precision) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 4955384a9b..f9ebde0657 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3663,7 +3663,7 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) { } static const char* jkDatabaseOptionsBuffer = "Buffer"; -static const char* jkDatabaseOptionsCachelast = "Cachelast"; +static const char* jkDatabaseOptionsCacheModel = "CacheModel"; static const char* jkDatabaseOptionsCompressionLevel = "CompressionLevel"; static const char* jkDatabaseOptionsDaysPerFileNode = "DaysPerFileNode"; static const char* jkDatabaseOptionsDaysPerFile = "DaysPerFile"; @@ -3687,7 +3687,7 @@ static int32_t databaseOptionsToJson(const void* pObj, SJson* pJson) { int32_t code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsBuffer, pNode->buffer); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCachelast, pNode->cacheLast); + code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCacheModel, pNode->cacheModel); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDatabaseOptionsCompressionLevel, pNode->compressionLevel); @@ -3749,7 +3749,7 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) { int32_t code = tjsonGetIntValue(pJson, jkDatabaseOptionsBuffer, &pNode->buffer); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCachelast, &pNode->cacheLast); + code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCacheModel, &pNode->cacheModel); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkDatabaseOptionsCompressionLevel, &pNode->compressionLevel); diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 59fc69f768..f3ea332fe2 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -38,8 +38,8 @@ typedef struct SAstCreateContext { typedef enum EDatabaseOptionType { DB_OPTION_BUFFER = 1, - DB_OPTION_CACHELAST, - DB_OPTION_CACHELASTSIZE, + DB_OPTION_CACHEMODEL, + DB_OPTION_CACHESIZE, DB_OPTION_COMP, DB_OPTION_DAYS, DB_OPTION_FSYNC, diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index ddde20e8e9..9cab419c2e 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -172,8 +172,8 @@ exists_opt(A) ::= . db_options(A) ::= . { A = createDefaultDatabaseOptions(pCxt); } db_options(A) ::= db_options(B) BUFFER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BUFFER, &C); } -db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); } -db_options(A) ::= db_options(B) CACHELASTSIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELASTSIZE, &C); } +db_options(A) ::= db_options(B) CACHEMODEL NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHEMODEL, &C); } +db_options(A) ::= db_options(B) CACHESIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHESIZE, &C); } db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); } db_options(A) ::= db_options(B) DURATION NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } db_options(A) ::= db_options(B) DURATION NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } @@ -186,7 +186,7 @@ db_options(A) ::= db_options(B) PAGES NK_INTEGER(C). db_options(A) ::= db_options(B) PAGESIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PAGESIZE, &C); } db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); } db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); } -db_options(A) ::= db_options(B) STRICT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STRICT, &C); } +db_options(A) ::= db_options(B) STRICT NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STRICT, &C); } db_options(A) ::= db_options(B) WAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL, &C); } db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); } @@ -199,8 +199,8 @@ alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). %type alter_db_option { SAlterOption } %destructor alter_db_option { } alter_db_option(A) ::= BUFFER NK_INTEGER(B). { A.type = DB_OPTION_BUFFER; A.val = B; } -alter_db_option(A) ::= CACHELAST NK_INTEGER(B). { A.type = DB_OPTION_CACHELAST; A.val = B; } -alter_db_option(A) ::= CACHELASTSIZE NK_INTEGER(B). { A.type = DB_OPTION_CACHELASTSIZE; A.val = B; } +alter_db_option(A) ::= CACHEMODEL NK_STRING(B). { A.type = DB_OPTION_CACHEMODEL; A.val = B; } +alter_db_option(A) ::= CACHESIZE NK_INTEGER(B). { A.type = DB_OPTION_CACHESIZE; A.val = B; } alter_db_option(A) ::= FSYNC NK_INTEGER(B). { A.type = DB_OPTION_FSYNC; A.val = B; } alter_db_option(A) ::= KEEP integer_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; } alter_db_option(A) ::= KEEP variable_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a7806f5e34..0a04ff3e30 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -760,8 +760,8 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); CHECK_OUT_OF_MEM(pOptions); pOptions->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; - pOptions->cacheLast = TSDB_DEFAULT_CACHE_LAST; - pOptions->cacheLastSize = TSDB_DEFAULT_CACHE_LAST_SIZE; + pOptions->cacheModel = TSDB_DEFAULT_CACHE_MODEL; + pOptions->cacheLastSize = TSDB_DEFAULT_CACHE_SIZE; pOptions->compressionLevel = TSDB_DEFAULT_COMP_LEVEL; pOptions->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; pOptions->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; @@ -787,7 +787,7 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); CHECK_OUT_OF_MEM(pOptions); pOptions->buffer = -1; - pOptions->cacheLast = -1; + pOptions->cacheModel = -1; pOptions->cacheLastSize = -1; pOptions->compressionLevel = -1; pOptions->daysPerFile = -1; @@ -815,10 +815,10 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti case DB_OPTION_BUFFER: ((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; - case DB_OPTION_CACHELAST: - ((SDatabaseOptions*)pOptions)->cacheLast = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + case DB_OPTION_CACHEMODEL: + COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->cacheModelStr, (SToken*)pVal); break; - case DB_OPTION_CACHELASTSIZE: + case DB_OPTION_CACHESIZE: ((SDatabaseOptions*)pOptions)->cacheLastSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_COMP: @@ -858,7 +858,7 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti ((SDatabaseOptions*)pOptions)->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_STRICT: - ((SDatabaseOptions*)pOptions)->strict = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->strictStr, (SToken*)pVal); break; case DB_OPTION_WAL: ((SDatabaseOptions*)pOptions)->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); @@ -872,10 +872,6 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti case DB_OPTION_RETENTIONS: ((SDatabaseOptions*)pOptions)->pRetentions = pVal; break; - // case DB_OPTION_SCHEMALESS: - // ((SDatabaseOptions*)pOptions)->schemaless = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); - // ((SDatabaseOptions*)pOptions)->schemaless = 0; - // break; default: break; } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 0d176cef09..b9d907c600 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -52,8 +52,8 @@ static SKeyword keywordTable[] = { {"BUFSIZE", TK_BUFSIZE}, {"BY", TK_BY}, {"CACHE", TK_CACHE}, - {"CACHELAST", TK_CACHELAST}, - {"CACHELASTSIZE", TK_CACHELASTSIZE}, + {"CACHEMODEL", TK_CACHEMODEL}, + {"CACHESIZE", TK_CACHESIZE}, {"CAST", TK_CAST}, {"CLIENT_VERSION", TK_CLIENT_VERSION}, {"CLUSTER", TK_CLUSTER}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3164623465..39f60a7a3e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2938,7 +2938,7 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS pReq->compression = pStmt->pOptions->compressionLevel; pReq->replications = pStmt->pOptions->replica; pReq->strict = pStmt->pOptions->strict; - pReq->cacheLast = pStmt->pOptions->cacheLast; + pReq->cacheLast = pStmt->pOptions->cacheModel; pReq->cacheLastSize = pStmt->pOptions->cacheLastSize; pReq->schemaless = pStmt->pOptions->schemaless; pReq->ignoreExist = pStmt->ignoreExists; @@ -3019,13 +3019,31 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt return TSDB_CODE_SUCCESS; } +static int32_t checkDbCacheModelOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { + if ('\0' != pOptions->cacheModelStr[0]) { + if (0 == strcasecmp(pOptions->cacheModelStr, TSDB_CACHE_MODEL_NONE_STR)) { + pOptions->cacheModel = TSDB_CACHE_MODEL_NONE; + } else if (0 == strcasecmp(pOptions->cacheModelStr, TSDB_CACHE_MODEL_LAST_ROW_STR)) { + pOptions->cacheModel = TSDB_CACHE_MODEL_LAST_ROW; + } else if (0 == strcasecmp(pOptions->cacheModelStr, TSDB_CACHE_MODEL_LAST_VALUE_STR)) { + pOptions->cacheModel = TSDB_CACHE_MODEL_LAST_VALUE; + } else if (0 == strcasecmp(pOptions->cacheModelStr, TSDB_CACHE_MODEL_BOTH_STR)) { + pOptions->cacheModel = TSDB_CACHE_MODEL_BOTH; + } else { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STR_OPTION, "cacheModel", + pOptions->cacheModelStr); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkDbPrecisionOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { if ('\0' != pOptions->precisionStr[0]) { - if (0 == strcmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MILLI_STR)) { + if (0 == strcasecmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MILLI_STR)) { pOptions->precision = TSDB_TIME_PRECISION_MILLI; - } else if (0 == strcmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MICRO_STR)) { + } else if (0 == strcasecmp(pOptions->precisionStr, TSDB_TIME_PRECISION_MICRO_STR)) { pOptions->precision = TSDB_TIME_PRECISION_MICRO; - } else if (0 == strcmp(pOptions->precisionStr, TSDB_TIME_PRECISION_NANO_STR)) { + } else if (0 == strcasecmp(pOptions->precisionStr, TSDB_TIME_PRECISION_NANO_STR)) { pOptions->precision = TSDB_TIME_PRECISION_NANO; } else { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STR_OPTION, "precision", pOptions->precisionStr); @@ -3034,6 +3052,19 @@ static int32_t checkDbPrecisionOption(STranslateContext* pCxt, SDatabaseOptions* return TSDB_CODE_SUCCESS; } +static int32_t checkDbStrictOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { + if ('\0' != pOptions->strictStr[0]) { + if (0 == strcasecmp(pOptions->strictStr, TSDB_DB_STRICT_OFF_STR)) { + pOptions->strict = TSDB_DB_STRICT_OFF; + } else if (0 == strcasecmp(pOptions->strictStr, TSDB_DB_STRICT_ON_STR)) { + pOptions->strict = TSDB_DB_STRICT_ON; + } else { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STR_OPTION, "strict", pOptions->strictStr); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t v1, int32_t v2) { if (val >= 0 && val != v1 && val != v2) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ENUM_OPTION, pName, val, v1, v2); @@ -3100,11 +3131,10 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName int32_t code = checkRangeOption(pCxt, "buffer", pOptions->buffer, TSDB_MIN_BUFFER_PER_VNODE, TSDB_MAX_BUFFER_PER_VNODE); if (TSDB_CODE_SUCCESS == code) { - code = checkRangeOption(pCxt, "cacheLast", pOptions->cacheLast, TSDB_MIN_DB_CACHE_LAST, TSDB_MAX_DB_CACHE_LAST); + code = checkDbCacheModelOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { - code = checkRangeOption(pCxt, "cacheLastSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_LAST_SIZE, - TSDB_MAX_DB_CACHE_LAST_SIZE); + code = checkRangeOption(pCxt, "cacheSize", pOptions->cacheLastSize, TSDB_MIN_DB_CACHE_SIZE, TSDB_MAX_DB_CACHE_SIZE); } if (TSDB_CODE_SUCCESS == code) { code = checkRangeOption(pCxt, "compression", pOptions->compressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); @@ -3140,7 +3170,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbEnumOption(pCxt, "replications", pOptions->replica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbEnumOption(pCxt, "strict", pOptions->strict, TSDB_DB_STRICT_OFF, TSDB_DB_STRICT_ON); + code = checkDbStrictOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "walLevel", pOptions->walLevel, TSDB_MIN_WAL_LEVEL, TSDB_MAX_WAL_LEVEL); @@ -3225,7 +3255,7 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, pReq->fsyncPeriod = pStmt->pOptions->fsyncPeriod; pReq->walLevel = pStmt->pOptions->walLevel; pReq->strict = pStmt->pOptions->strict; - pReq->cacheLast = pStmt->pOptions->cacheLast; + pReq->cacheLast = pStmt->pOptions->cacheModel; pReq->cacheLastSize = pStmt->pOptions->cacheLastSize; pReq->replications = pStmt->pOptions->replica; return; diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 2200b43db1..57a243257b 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -384,32 +384,32 @@ static const YYACTIONTYPE yy_action[] = { /* 1630 */ 464, 1792, 468, 467, 1512, 469, 1511, 1496, 1587, 1824, /* 1640 */ 1154, 50, 196, 295, 1793, 586, 1795, 1796, 582, 1824, /* 1650 */ 577, 1153, 1586, 290, 1793, 586, 1795, 1796, 582, 1810, - /* 1660 */ 577, 1079, 632, 1078, 634, 1077, 1076, 584, 1073, 1521, - /* 1670 */ 1072, 319, 1761, 320, 583, 1071, 1070, 1516, 1514, 321, - /* 1680 */ 496, 1792, 1495, 498, 1494, 500, 1493, 502, 493, 1732, - /* 1690 */ 94, 551, 15, 1792, 1237, 1726, 140, 509, 1713, 1824, - /* 1700 */ 1711, 1712, 1710, 146, 1793, 586, 1795, 1796, 582, 1810, - /* 1710 */ 577, 1709, 1707, 56, 1699, 1247, 229, 581, 510, 227, - /* 1720 */ 214, 1810, 1761, 16, 583, 232, 339, 225, 322, 584, - /* 1730 */ 219, 78, 515, 41, 1761, 17, 583, 47, 79, 23, - /* 1740 */ 524, 1437, 84, 234, 13, 243, 236, 1419, 1951, 1824, + /* 1660 */ 577, 1079, 632, 1078, 634, 1077, 1076, 584, 1073, 1071, + /* 1670 */ 1072, 1521, 1761, 319, 583, 1070, 1516, 320, 1514, 321, + /* 1680 */ 496, 1792, 1495, 498, 1494, 500, 1493, 502, 493, 94, + /* 1690 */ 1732, 551, 509, 1792, 1237, 1726, 140, 1713, 1711, 1824, + /* 1700 */ 1712, 1710, 1709, 146, 1793, 586, 1795, 1796, 582, 1810, + /* 1710 */ 577, 1247, 1707, 56, 1699, 41, 227, 581, 510, 84, + /* 1720 */ 214, 1810, 1761, 16, 583, 232, 339, 15, 322, 584, + /* 1730 */ 219, 225, 515, 243, 1761, 1437, 583, 47, 78, 79, + /* 1740 */ 524, 23, 242, 229, 236, 234, 25, 1419, 1951, 1824, /* 1750 */ 1421, 238, 147, 294, 1793, 586, 1795, 1796, 582, 241, - /* 1760 */ 577, 1824, 1843, 242, 1782, 295, 1793, 586, 1795, 1796, - /* 1770 */ 582, 1792, 577, 24, 25, 252, 46, 1414, 83, 18, - /* 1780 */ 1781, 1792, 1394, 1393, 151, 1449, 1448, 333, 1453, 1454, - /* 1790 */ 1443, 1452, 334, 10, 1280, 1356, 1331, 45, 19, 1810, - /* 1800 */ 1827, 576, 1311, 1329, 341, 1328, 31, 584, 152, 1810, - /* 1810 */ 12, 20, 1761, 165, 583, 21, 589, 584, 585, 587, - /* 1820 */ 342, 1140, 1761, 1137, 583, 591, 594, 593, 1134, 596, - /* 1830 */ 597, 1792, 1128, 599, 600, 602, 1132, 1131, 1117, 1824, - /* 1840 */ 609, 1792, 1149, 295, 1793, 586, 1795, 1796, 582, 1824, - /* 1850 */ 577, 1792, 1126, 281, 1793, 586, 1795, 1796, 582, 1810, - /* 1860 */ 577, 603, 85, 86, 62, 263, 1145, 584, 1130, 1810, - /* 1870 */ 1129, 1042, 1761, 618, 583, 264, 1067, 584, 1086, 1810, - /* 1880 */ 621, 1065, 1761, 1064, 583, 1063, 1062, 584, 1061, 1060, - /* 1890 */ 1059, 1058, 1761, 1083, 583, 1081, 1055, 1054, 1053, 1824, - /* 1900 */ 1050, 1049, 1528, 282, 1793, 586, 1795, 1796, 582, 1824, - /* 1910 */ 577, 1048, 1047, 289, 1793, 586, 1795, 1796, 582, 1824, + /* 1760 */ 577, 1824, 1843, 1782, 17, 295, 1793, 586, 1795, 1796, + /* 1770 */ 582, 1792, 577, 24, 252, 1414, 83, 46, 1781, 1394, + /* 1780 */ 1449, 1792, 1393, 151, 18, 1448, 333, 1453, 1452, 10, + /* 1790 */ 1454, 45, 1443, 334, 1280, 1356, 1827, 1311, 19, 1810, + /* 1800 */ 589, 1331, 1329, 13, 341, 576, 31, 584, 1328, 1810, + /* 1810 */ 152, 12, 1761, 165, 583, 20, 21, 584, 585, 587, + /* 1820 */ 342, 1140, 1761, 591, 583, 1137, 593, 594, 596, 1134, + /* 1830 */ 1128, 1792, 597, 599, 600, 602, 1132, 1117, 1131, 1824, + /* 1840 */ 1149, 1792, 1126, 295, 1793, 586, 1795, 1796, 582, 1824, + /* 1850 */ 577, 1792, 263, 281, 1793, 586, 1795, 1796, 582, 1810, + /* 1860 */ 577, 603, 85, 609, 86, 62, 1130, 584, 1129, 1810, + /* 1870 */ 1145, 618, 1761, 1042, 583, 1086, 1067, 584, 621, 1810, + /* 1880 */ 264, 1065, 1761, 1062, 583, 1064, 1063, 584, 1061, 1060, + /* 1890 */ 1059, 1058, 1761, 1083, 583, 1081, 1048, 1055, 1054, 1824, + /* 1900 */ 1053, 1050, 1528, 282, 1793, 586, 1795, 1796, 582, 1824, + /* 1910 */ 577, 1049, 1047, 289, 1793, 586, 1795, 1796, 582, 1824, /* 1920 */ 577, 643, 644, 291, 1793, 586, 1795, 1796, 582, 645, /* 1930 */ 577, 1526, 1792, 647, 648, 649, 1524, 1522, 651, 652, /* 1940 */ 653, 655, 656, 657, 1510, 659, 1004, 1492, 267, 663, @@ -641,30 +641,30 @@ static const YYCODETYPE yy_lookahead[] = { /* 1630 */ 47, 259, 47, 35, 0, 39, 0, 0, 0, 327, /* 1640 */ 35, 94, 92, 331, 332, 333, 334, 335, 336, 327, /* 1650 */ 338, 22, 0, 331, 332, 333, 334, 335, 336, 287, - /* 1660 */ 338, 35, 43, 35, 43, 35, 35, 295, 35, 0, - /* 1670 */ 35, 22, 300, 22, 302, 35, 35, 0, 0, 22, - /* 1680 */ 35, 259, 0, 35, 0, 35, 0, 22, 49, 0, - /* 1690 */ 20, 369, 85, 259, 35, 0, 172, 22, 0, 327, + /* 1660 */ 338, 35, 43, 35, 43, 35, 35, 295, 35, 22, + /* 1670 */ 35, 0, 300, 22, 302, 35, 0, 22, 0, 22, + /* 1680 */ 35, 259, 0, 35, 0, 35, 0, 22, 49, 20, + /* 1690 */ 0, 369, 22, 259, 35, 0, 172, 0, 0, 327, /* 1700 */ 0, 0, 0, 331, 332, 333, 334, 335, 336, 287, - /* 1710 */ 338, 0, 0, 153, 0, 181, 149, 295, 153, 39, + /* 1710 */ 338, 181, 0, 153, 0, 43, 39, 295, 153, 95, /* 1720 */ 150, 287, 300, 230, 302, 46, 292, 85, 153, 295, - /* 1730 */ 86, 85, 155, 43, 300, 230, 302, 43, 85, 85, - /* 1740 */ 151, 86, 95, 85, 230, 46, 86, 86, 376, 327, + /* 1730 */ 86, 85, 155, 46, 300, 86, 302, 43, 85, 85, + /* 1740 */ 151, 85, 43, 149, 86, 85, 43, 86, 376, 327, /* 1750 */ 86, 85, 85, 331, 332, 333, 334, 335, 336, 85, - /* 1760 */ 338, 327, 340, 43, 46, 331, 332, 333, 334, 335, - /* 1770 */ 336, 259, 338, 85, 43, 46, 43, 86, 85, 43, - /* 1780 */ 46, 259, 86, 86, 46, 35, 35, 35, 35, 86, - /* 1790 */ 86, 35, 35, 2, 22, 193, 86, 224, 43, 287, - /* 1800 */ 85, 85, 22, 86, 292, 86, 85, 295, 46, 287, - /* 1810 */ 85, 85, 300, 46, 302, 85, 35, 295, 195, 96, - /* 1820 */ 35, 86, 300, 86, 302, 85, 85, 35, 86, 35, - /* 1830 */ 85, 259, 86, 35, 85, 35, 109, 109, 22, 327, - /* 1840 */ 97, 259, 35, 331, 332, 333, 334, 335, 336, 327, - /* 1850 */ 338, 259, 86, 331, 332, 333, 334, 335, 336, 287, - /* 1860 */ 338, 85, 85, 85, 85, 43, 22, 295, 109, 287, - /* 1870 */ 109, 62, 300, 61, 302, 43, 35, 295, 68, 287, - /* 1880 */ 83, 35, 300, 35, 302, 35, 35, 295, 35, 22, - /* 1890 */ 35, 35, 300, 68, 302, 35, 35, 35, 35, 327, + /* 1760 */ 338, 327, 340, 46, 230, 331, 332, 333, 334, 335, + /* 1770 */ 336, 259, 338, 85, 46, 86, 85, 43, 46, 86, + /* 1780 */ 35, 259, 86, 46, 43, 35, 35, 35, 35, 2, + /* 1790 */ 86, 224, 86, 35, 22, 193, 85, 22, 43, 287, + /* 1800 */ 35, 86, 86, 230, 292, 85, 85, 295, 86, 287, + /* 1810 */ 46, 85, 300, 46, 302, 85, 85, 295, 195, 96, + /* 1820 */ 35, 86, 300, 85, 302, 86, 35, 85, 35, 86, + /* 1830 */ 86, 259, 85, 35, 85, 35, 109, 22, 109, 327, + /* 1840 */ 35, 259, 86, 331, 332, 333, 334, 335, 336, 327, + /* 1850 */ 338, 259, 43, 331, 332, 333, 334, 335, 336, 287, + /* 1860 */ 338, 85, 85, 97, 85, 85, 109, 295, 109, 287, + /* 1870 */ 22, 61, 300, 62, 302, 68, 35, 295, 83, 287, + /* 1880 */ 43, 35, 300, 22, 302, 35, 35, 295, 35, 22, + /* 1890 */ 35, 35, 300, 68, 302, 35, 22, 35, 35, 327, /* 1900 */ 35, 35, 0, 331, 332, 333, 334, 335, 336, 327, /* 1910 */ 338, 35, 35, 331, 332, 333, 334, 335, 336, 327, /* 1920 */ 338, 35, 47, 331, 332, 333, 334, 335, 336, 39, @@ -783,23 +783,23 @@ static const unsigned short int yy_shift_ofst[] = { /* 450 */ 1603, 1606, 1608, 1553, 1611, 1613, 1581, 1571, 1580, 1620, /* 460 */ 1586, 1576, 1587, 1625, 1593, 1583, 1590, 1627, 1598, 1585, /* 470 */ 1596, 1634, 1636, 1637, 1638, 1547, 1550, 1605, 1629, 1652, - /* 480 */ 1626, 1628, 1630, 1631, 1619, 1621, 1633, 1635, 1640, 1641, - /* 490 */ 1669, 1649, 1677, 1651, 1639, 1678, 1657, 1645, 1682, 1648, - /* 500 */ 1684, 1650, 1686, 1665, 1670, 1689, 1560, 1659, 1695, 1524, - /* 510 */ 1675, 1565, 1570, 1698, 1700, 1575, 1577, 1701, 1702, 1711, - /* 520 */ 1607, 1644, 1534, 1712, 1642, 1589, 1646, 1714, 1680, 1567, - /* 530 */ 1653, 1647, 1679, 1690, 1493, 1654, 1655, 1658, 1660, 1661, - /* 540 */ 1666, 1694, 1664, 1667, 1674, 1688, 1691, 1720, 1699, 1718, - /* 550 */ 1693, 1731, 1505, 1696, 1697, 1729, 1573, 1733, 1734, 1738, - /* 560 */ 1703, 1736, 1514, 1704, 1750, 1751, 1752, 1753, 1756, 1757, - /* 570 */ 1704, 1791, 1772, 1602, 1755, 1715, 1710, 1716, 1717, 1721, - /* 580 */ 1719, 1762, 1725, 1726, 1767, 1780, 1623, 1730, 1723, 1735, - /* 590 */ 1781, 1785, 1740, 1737, 1792, 1741, 1742, 1794, 1745, 1746, - /* 600 */ 1798, 1749, 1766, 1800, 1776, 1727, 1728, 1759, 1761, 1816, - /* 610 */ 1743, 1777, 1778, 1807, 1779, 1822, 1822, 1844, 1809, 1812, - /* 620 */ 1841, 1810, 1797, 1832, 1846, 1848, 1850, 1851, 1853, 1867, - /* 630 */ 1855, 1856, 1825, 1619, 1860, 1621, 1861, 1862, 1863, 1865, - /* 640 */ 1866, 1876, 1877, 1902, 1886, 1875, 1890, 1931, 1898, 1887, + /* 480 */ 1626, 1628, 1630, 1631, 1619, 1621, 1633, 1635, 1647, 1640, + /* 490 */ 1671, 1651, 1676, 1655, 1639, 1678, 1657, 1645, 1682, 1648, + /* 500 */ 1684, 1650, 1686, 1665, 1669, 1690, 1560, 1659, 1695, 1524, + /* 510 */ 1670, 1565, 1570, 1697, 1698, 1575, 1577, 1700, 1701, 1702, + /* 520 */ 1642, 1644, 1530, 1712, 1646, 1589, 1653, 1714, 1677, 1594, + /* 530 */ 1654, 1624, 1679, 1672, 1493, 1656, 1649, 1660, 1658, 1661, + /* 540 */ 1666, 1694, 1664, 1667, 1674, 1688, 1689, 1699, 1687, 1717, + /* 550 */ 1691, 1703, 1534, 1693, 1696, 1728, 1567, 1734, 1732, 1737, + /* 560 */ 1704, 1741, 1573, 1706, 1745, 1750, 1751, 1752, 1753, 1758, + /* 570 */ 1706, 1787, 1772, 1602, 1755, 1711, 1715, 1720, 1716, 1721, + /* 580 */ 1722, 1764, 1726, 1730, 1767, 1775, 1623, 1731, 1723, 1735, + /* 590 */ 1765, 1785, 1738, 1739, 1791, 1742, 1743, 1793, 1747, 1744, + /* 600 */ 1798, 1749, 1756, 1800, 1776, 1727, 1729, 1757, 1759, 1815, + /* 610 */ 1766, 1777, 1779, 1805, 1780, 1809, 1809, 1848, 1811, 1810, + /* 620 */ 1841, 1807, 1795, 1837, 1846, 1850, 1851, 1861, 1853, 1867, + /* 630 */ 1855, 1856, 1825, 1619, 1860, 1621, 1862, 1863, 1865, 1866, + /* 640 */ 1876, 1874, 1877, 1902, 1886, 1875, 1890, 1931, 1898, 1887, /* 650 */ 1896, 1936, 1903, 1892, 1901, 1937, 1906, 1895, 1904, 1944, /* 660 */ 1910, 1911, 1947, 1926, 1928, 1930, 1932, 1929, 1933, }; @@ -987,8 +987,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* NOT => nothing */ 0, /* EXISTS => nothing */ 0, /* BUFFER => nothing */ - 0, /* CACHELAST => nothing */ - 0, /* CACHELASTSIZE => nothing */ + 0, /* CACHEMODEL => nothing */ + 0, /* CACHESIZE => nothing */ 0, /* COMP => nothing */ 0, /* DURATION => nothing */ 0, /* NK_VARIABLE => nothing */ @@ -1330,8 +1330,8 @@ static const char *const yyTokenName[] = { /* 61 */ "NOT", /* 62 */ "EXISTS", /* 63 */ "BUFFER", - /* 64 */ "CACHELAST", - /* 65 */ "CACHELASTSIZE", + /* 64 */ "CACHEMODEL", + /* 65 */ "CACHESIZE", /* 66 */ "COMP", /* 67 */ "DURATION", /* 68 */ "NK_VARIABLE", @@ -1726,8 +1726,8 @@ static const char *const yyRuleName[] = { /* 71 */ "exists_opt ::=", /* 72 */ "db_options ::=", /* 73 */ "db_options ::= db_options BUFFER NK_INTEGER", - /* 74 */ "db_options ::= db_options CACHELAST NK_INTEGER", - /* 75 */ "db_options ::= db_options CACHELASTSIZE NK_INTEGER", + /* 74 */ "db_options ::= db_options CACHEMODEL NK_STRING", + /* 75 */ "db_options ::= db_options CACHESIZE NK_INTEGER", /* 76 */ "db_options ::= db_options COMP NK_INTEGER", /* 77 */ "db_options ::= db_options DURATION NK_INTEGER", /* 78 */ "db_options ::= db_options DURATION NK_VARIABLE", @@ -1740,7 +1740,7 @@ static const char *const yyRuleName[] = { /* 85 */ "db_options ::= db_options PAGESIZE NK_INTEGER", /* 86 */ "db_options ::= db_options PRECISION NK_STRING", /* 87 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 88 */ "db_options ::= db_options STRICT NK_INTEGER", + /* 88 */ "db_options ::= db_options STRICT NK_STRING", /* 89 */ "db_options ::= db_options WAL NK_INTEGER", /* 90 */ "db_options ::= db_options VGROUPS NK_INTEGER", /* 91 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", @@ -1749,8 +1749,8 @@ static const char *const yyRuleName[] = { /* 94 */ "alter_db_options ::= alter_db_option", /* 95 */ "alter_db_options ::= alter_db_options alter_db_option", /* 96 */ "alter_db_option ::= BUFFER NK_INTEGER", - /* 97 */ "alter_db_option ::= CACHELAST NK_INTEGER", - /* 98 */ "alter_db_option ::= CACHELASTSIZE NK_INTEGER", + /* 97 */ "alter_db_option ::= CACHEMODEL NK_STRING", + /* 98 */ "alter_db_option ::= CACHESIZE NK_INTEGER", /* 99 */ "alter_db_option ::= FSYNC NK_INTEGER", /* 100 */ "alter_db_option ::= KEEP integer_list", /* 101 */ "alter_db_option ::= KEEP variable_list", @@ -2816,8 +2816,8 @@ static const struct { { 271, 0 }, /* (71) exists_opt ::= */ { 270, 0 }, /* (72) db_options ::= */ { 270, -3 }, /* (73) db_options ::= db_options BUFFER NK_INTEGER */ - { 270, -3 }, /* (74) db_options ::= db_options CACHELAST NK_INTEGER */ - { 270, -3 }, /* (75) db_options ::= db_options CACHELASTSIZE NK_INTEGER */ + { 270, -3 }, /* (74) db_options ::= db_options CACHEMODEL NK_STRING */ + { 270, -3 }, /* (75) db_options ::= db_options CACHESIZE NK_INTEGER */ { 270, -3 }, /* (76) db_options ::= db_options COMP NK_INTEGER */ { 270, -3 }, /* (77) db_options ::= db_options DURATION NK_INTEGER */ { 270, -3 }, /* (78) db_options ::= db_options DURATION NK_VARIABLE */ @@ -2830,7 +2830,7 @@ static const struct { { 270, -3 }, /* (85) db_options ::= db_options PAGESIZE NK_INTEGER */ { 270, -3 }, /* (86) db_options ::= db_options PRECISION NK_STRING */ { 270, -3 }, /* (87) db_options ::= db_options REPLICA NK_INTEGER */ - { 270, -3 }, /* (88) db_options ::= db_options STRICT NK_INTEGER */ + { 270, -3 }, /* (88) db_options ::= db_options STRICT NK_STRING */ { 270, -3 }, /* (89) db_options ::= db_options WAL NK_INTEGER */ { 270, -3 }, /* (90) db_options ::= db_options VGROUPS NK_INTEGER */ { 270, -3 }, /* (91) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ @@ -2839,8 +2839,8 @@ static const struct { { 272, -1 }, /* (94) alter_db_options ::= alter_db_option */ { 272, -2 }, /* (95) alter_db_options ::= alter_db_options alter_db_option */ { 276, -2 }, /* (96) alter_db_option ::= BUFFER NK_INTEGER */ - { 276, -2 }, /* (97) alter_db_option ::= CACHELAST NK_INTEGER */ - { 276, -2 }, /* (98) alter_db_option ::= CACHELASTSIZE NK_INTEGER */ + { 276, -2 }, /* (97) alter_db_option ::= CACHEMODEL NK_STRING */ + { 276, -2 }, /* (98) alter_db_option ::= CACHESIZE NK_INTEGER */ { 276, -2 }, /* (99) alter_db_option ::= FSYNC NK_INTEGER */ { 276, -2 }, /* (100) alter_db_option ::= KEEP integer_list */ { 276, -2 }, /* (101) alter_db_option ::= KEEP variable_list */ @@ -3543,12 +3543,12 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy616 = yylhsminor.yy616; break; - case 74: /* db_options ::= db_options CACHELAST NK_INTEGER */ -{ yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } + case 74: /* db_options ::= db_options CACHEMODEL NK_STRING */ +{ yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy616 = yylhsminor.yy616; break; - case 75: /* db_options ::= db_options CACHELASTSIZE NK_INTEGER */ -{ yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_CACHELASTSIZE, &yymsp[0].minor.yy0); } + case 75: /* db_options ::= db_options CACHESIZE NK_INTEGER */ +{ yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy616 = yylhsminor.yy616; break; case 76: /* db_options ::= db_options COMP NK_INTEGER */ @@ -3593,7 +3593,7 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy616 = yylhsminor.yy616; break; - case 88: /* db_options ::= db_options STRICT NK_INTEGER */ + case 88: /* db_options ::= db_options STRICT NK_STRING */ { yylhsminor.yy616 = setDatabaseOption(pCxt, yymsp[-2].minor.yy616, DB_OPTION_STRICT, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy616 = yylhsminor.yy616; break; @@ -3628,11 +3628,11 @@ static YYACTIONTYPE yy_reduce( case 96: /* alter_db_option ::= BUFFER NK_INTEGER */ { yymsp[-1].minor.yy409.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } break; - case 97: /* alter_db_option ::= CACHELAST NK_INTEGER */ -{ yymsp[-1].minor.yy409.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } + case 97: /* alter_db_option ::= CACHEMODEL NK_STRING */ +{ yymsp[-1].minor.yy409.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } break; - case 98: /* alter_db_option ::= CACHELASTSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy409.type = DB_OPTION_CACHELASTSIZE; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } + case 98: /* alter_db_option ::= CACHESIZE NK_INTEGER */ +{ yymsp[-1].minor.yy409.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } break; case 99: /* alter_db_option ::= FSYNC NK_INTEGER */ { yymsp[-1].minor.yy409.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy409.val = yymsp[0].minor.yy0; } diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp index f4d0ba1cc8..079a9540c3 100644 --- a/source/libs/parser/test/parInitialATest.cpp +++ b/source/libs/parser/test/parInitialATest.cpp @@ -38,7 +38,7 @@ TEST_F(ParserInitialATest, alterDnode) { TEST_F(ParserInitialATest, alterDatabase) { useDb("root", "test"); - run("ALTER DATABASE test CACHELAST 1 FSYNC 200 WAL 1"); + run("ALTER DATABASE test CACHEMODEL 'last_row' FSYNC 200 WAL 1"); run("ALTER DATABASE test KEEP 2400"); } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index e9c8fb5326..7ea4fab470 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -43,7 +43,8 @@ TEST_F(ParserInitialCTest, createBnode) { * * database_option: { * BUFFER value - * | CACHELAST value + * | CACHEMODEL {'none' | 'last_row' | 'last_value' | 'both'} + * | CACHESIZE value * | COMP {0 | 1 | 2} * | DURATION value * | FSYNC value @@ -55,7 +56,7 @@ TEST_F(ParserInitialCTest, createBnode) { * | PRECISION {'ms' | 'us' | 'ns'} * | REPLICA value * | RETENTIONS ingestion_duration:keep_duration ... - * | STRICT value + * | STRICT {'off' | 'on'} * | WAL value * | VGROUPS value * | SINGLE_STABLE {0 | 1} @@ -76,8 +77,8 @@ TEST_F(ParserInitialCTest, createDatabase) { expect.db[len] = '\0'; expect.ignoreExist = igExists; expect.buffer = TSDB_DEFAULT_BUFFER_PER_VNODE; - expect.cacheLast = TSDB_DEFAULT_CACHE_LAST; - expect.cacheLastSize = TSDB_DEFAULT_CACHE_LAST_SIZE; + expect.cacheLast = TSDB_DEFAULT_CACHE_MODEL; + expect.cacheLastSize = TSDB_DEFAULT_CACHE_SIZE; expect.compression = TSDB_DEFAULT_COMP_LEVEL; expect.daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; expect.fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; @@ -203,8 +204,8 @@ TEST_F(ParserInitialCTest, createDatabase) { setDbSchemalessFunc(1); run("CREATE DATABASE IF NOT EXISTS wxy_db " "BUFFER 64 " - "CACHELAST 2 " - "CACHELASTSIZE 20 " + "CACHEMODEL 'last_value' " + "CACHESIZE 20 " "COMP 1 " "DURATION 100 " "FSYNC 100 " @@ -216,7 +217,7 @@ TEST_F(ParserInitialCTest, createDatabase) { "PRECISION 'ns' " "REPLICA 3 " "RETENTIONS 15s:7d,1m:21d,15m:500d " - "STRICT 1 " + "STRICT 'on' " "WAL 2 " "VGROUPS 100 " "SINGLE_STABLE 1 " diff --git a/tests/pytest/alter/alter_cacheLastRow.py b/tests/pytest/alter/alter_cacheLastRow.py index 36a2864d0f..3152e46af1 100644 --- a/tests/pytest/alter/alter_cacheLastRow.py +++ b/tests/pytest/alter/alter_cacheLastRow.py @@ -67,7 +67,7 @@ class TDTestCase: slow = 0 #count time where lastRow on is slower for i in range(5): #switch lastRow to off and check - tdSql.execute('alter database db cachelast 0') + tdSql.execute('alter database db cachemodel 'none'') tdSql.query('show databases') tdSql.checkData(0,15,0) @@ -79,7 +79,7 @@ class TDTestCase: tdLog.debug(f'time used:{lastRow_Off_end-lastRow_Off_start}') #switch lastRow to on and check - tdSql.execute('alter database db cachelast 1') + tdSql.execute('alter database db cachemodel 'last_row'') tdSql.query('show databases') tdSql.checkData(0,15,1) diff --git a/tests/pytest/query/last_cache.py b/tests/pytest/query/last_cache.py index c31d9821e2..9ee5be8c72 100644 --- a/tests/pytest/query/last_cache.py +++ b/tests/pytest/query/last_cache.py @@ -89,36 +89,36 @@ class TDTestCase: tdSql.prepare() # last_cache_0.sim - tdSql.execute("create database test1 cachelast 0") + tdSql.execute("create database test1 cachemodel 'none'") tdSql.execute("use test1") self.insertData() self.executeQueries() - tdSql.execute("alter database test1 cachelast 1") + tdSql.execute("alter database test1 cachemodel 'last_row'") self.executeQueries() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries() - tdSql.execute("alter database test1 cachelast 0") + tdSql.execute("alter database test1 cachemodel 'none'") self.executeQueries() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries() # last_cache_1.sim - tdSql.execute("create database test2 cachelast 1") + tdSql.execute("create database test2 cachemodel 'last_row'") tdSql.execute("use test2") self.insertData() self.executeQueries() - tdSql.execute("alter database test2 cachelast 0") + tdSql.execute("alter database test2 cachemodel 'none'") self.executeQueries() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries() - tdSql.execute("alter database test2 cachelast 1") + tdSql.execute("alter database test2 cachemodel 'last_row'") self.executeQueries() tdDnodes.stop(1) tdDnodes.start(1) diff --git a/tests/pytest/query/last_row_cache.py b/tests/pytest/query/last_row_cache.py index 0e11e3d60c..de9f08b277 100644 --- a/tests/pytest/query/last_row_cache.py +++ b/tests/pytest/query/last_row_cache.py @@ -142,56 +142,56 @@ class TDTestCase: tdSql.prepare() print("============== Step1: last_row_cache_0.sim") - tdSql.execute("create database test1 cachelast 0") + tdSql.execute("create database test1 cachemodel 'none'") tdSql.execute("use test1") self.insertData() self.executeQueries() self.insertData2() self.executeQueries2() - print("============== Step2: alter database test1 cachelast 1") - tdSql.execute("alter database test1 cachelast 1") + print("============== Step2: alter database test1 cachemodel 'last_row'") + tdSql.execute("alter database test1 cachemodel 'last_row'") self.executeQueries2() - print("============== Step3: alter database test1 cachelast 2") - tdSql.execute("alter database test1 cachelast 2") + print("============== Step3: alter database test1 cachemodel 'last_value'") + tdSql.execute("alter database test1 cachemodel 'last_value'") self.executeQueries2() - print("============== Step4: alter database test1 cachelast 3") - tdSql.execute("alter database test1 cachelast 3") + print("============== Step4: alter database test1 cachemodel 'both'") + tdSql.execute("alter database test1 cachemodel 'both'") self.executeQueries2() - print("============== Step5: alter database test1 cachelast 0 and restart taosd") - tdSql.execute("alter database test1 cachelast 0") + print("============== Step5: alter database test1 cachemodel 'none' and restart taosd") + tdSql.execute("alter database test1 cachemodel 'none'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step6: alter database test1 cachelast 1 and restart taosd") - tdSql.execute("alter database test1 cachelast 1") + print("============== Step6: alter database test1 cachemodel 'last_row' and restart taosd") + tdSql.execute("alter database test1 cachemodel 'last_row'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step7: alter database test1 cachelast 2 and restart taosd") - tdSql.execute("alter database test1 cachelast 2") + print("============== Step7: alter database test1 cachemodel 'last_value' and restart taosd") + tdSql.execute("alter database test1 cachemodel 'last_value'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step8: alter database test1 cachelast 3 and restart taosd") - tdSql.execute("alter database test1 cachelast 3") + print("============== Step8: alter database test1 cachemodel 'both' and restart taosd") + tdSql.execute("alter database test1 cachemodel 'both'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step9: create database test2 cachelast 1") - tdSql.execute("create database test2 cachelast 1") + print("============== Step9: create database test2 cachemodel 'last_row'") + tdSql.execute("create database test2 cachemodel 'last_row'") tdSql.execute("use test2") self.insertData() self.executeQueries() @@ -201,45 +201,45 @@ class TDTestCase: tdDnodes.start(1) self.executeQueries2() - print("============== Step8: alter database test2 cachelast 0") - tdSql.execute("alter database test2 cachelast 0") + print("============== Step8: alter database test2 cachemodel 'none'") + tdSql.execute("alter database test2 cachemodel 'none'") self.executeQueries2() - print("============== Step9: alter database test2 cachelast 1") - tdSql.execute("alter database test2 cachelast 1") + print("============== Step9: alter database test2 cachemodel 'last_row'") + tdSql.execute("alter database test2 cachemodel 'last_row'") self.executeQueries2() - print("============== Step10: alter database test2 cachelast 2") - tdSql.execute("alter database test2 cachelast 2") + print("============== Step10: alter database test2 cachemodel 'last_value'") + tdSql.execute("alter database test2 cachemodel 'last_value'") self.executeQueries2() - print("============== Step11: alter database test2 cachelast 3") - tdSql.execute("alter database test2 cachelast 3") + print("============== Step11: alter database test2 cachemodel 'both'") + tdSql.execute("alter database test2 cachemodel 'both'") self.executeQueries2() - print("============== Step12: alter database test2 cachelast 0 and restart taosd") - tdSql.execute("alter database test2 cachelast 0") + print("============== Step12: alter database test2 cachemodel 'none' and restart taosd") + tdSql.execute("alter database test2 cachemodel 'none'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step13: alter database test2 cachelast 1 and restart taosd") - tdSql.execute("alter database test2 cachelast 1") + print("============== Step13: alter database test2 cachemodel 'last_row' and restart taosd") + tdSql.execute("alter database test2 cachemodel 'last_row'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step14: alter database test2 cachelast 2 and restart taosd") - tdSql.execute("alter database test2 cachelast 2") + print("============== Step14: alter database test2 cachemodel 'last_value' and restart taosd") + tdSql.execute("alter database test2 cachemodel 'last_value'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - print("============== Step15: alter database test2 cachelast 3 and restart taosd") - tdSql.execute("alter database test2 cachelast 3") + print("============== Step15: alter database test2 cachemodel 'both' and restart taosd") + tdSql.execute("alter database test2 cachemodel 'both'") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) diff --git a/tests/pytest/util/boundary.py b/tests/pytest/util/boundary.py index 4cef926f2e..086821e7cf 100644 --- a/tests/pytest/util/boundary.py +++ b/tests/pytest/util/boundary.py @@ -39,6 +39,6 @@ class DataBoundary: self.DB_PARAM_PRECISION_CONFIG = {"create_name": "precision", "query_name": "precision", "vnode_json_key": "", "boundary": ['ms', 'us', 'ns'], "default": "ms"} self.DB_PARAM_REPLICA_CONFIG = {"create_name": "replica", "query_name": "replica", "vnode_json_key": "", "boundary": [1], "default": 1} self.DB_PARAM_SINGLE_STABLE_CONFIG = {"create_name": "single_stable", "query_name": "single_stable_model", "vnode_json_key": "", "boundary": [0, 1], "default": 0} - self.DB_PARAM_STRICT_CONFIG = {"create_name": "strict", "query_name": "strict", "vnode_json_key": "", "boundary": {"no_strict": 0, "strict": 1}, "default": "no_strict"} + self.DB_PARAM_STRICT_CONFIG = {"create_name": "strict", "query_name": "strict", "vnode_json_key": "", "boundary": {"off": 0, "strict": 1}, "default": "off"} self.DB_PARAM_VGROUPS_CONFIG = {"create_name": "vgroups", "query_name": "vgroups", "vnode_json_key": "", "boundary": [1, 32], "default": 2} self.DB_PARAM_WAL_CONFIG = {"create_name": "wal", "query_name": "wal", "vnode_json_key": "", "boundary": [1, 2], "default": 1} \ No newline at end of file diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 1eb4e36da6..cda1656888 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -69,7 +69,7 @@ endi if $data4_db != 3 then # replica return -1 endi -if $data5_db != no_strict then # strict +if $data5_db != off then # strict return -1 endi if $data6_db != 345600m then # duration @@ -333,31 +333,31 @@ sql_error alter database db comp 5 sql_error alter database db comp -1 print ============== modify cachelast [0, 1, 2, 3] -sql alter database db cachelast 2 +sql alter database db cachemodel 'last_value' sql show databases print cachelast $data16_db if $data16_db != 2 then return -1 endi -sql alter database db cachelast 1 +sql alter database db cachemodel 'last_row' sql show databases print cachelast $data16_db if $data16_db != 1 then return -1 endi -sql alter database db cachelast 0 +sql alter database db cachemodel 'none' sql show databases print cachelast $data16_db if $data16_db != 0 then return -1 endi -sql alter database db cachelast 2 +sql alter database db cachemodel 'last_value' sql show databases print cachelast $data16_db if $data16_db != 2 then return -1 endi -sql alter database db cachelast 3 +sql alter database db cachemodel 'both' sql show databases print cachelast $data16_db if $data16_db != 3 then diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 7525fe2087..1daccb03bc 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -15,7 +15,7 @@ $tb = $tbPrefix . $i print =============== step1 # quorum presicion -sql create database $db vgroups 8 replica 1 duration 2 keep 10 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us' +sql create database $db vgroups 8 replica 1 duration 2 keep 10 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachemodel 'last_value' precision 'us' sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index bdfc556e18..ee65229be1 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -89,7 +89,7 @@ if $data4_db != 1 then # replica print expect 1, actual: $data4_db return -1 endi -if $data5_db != no_strict then # strict +if $data5_db != off then # strict return -1 endi if $data6_db != 14400m then # duration diff --git a/tests/script/tsim/parser/last_cache.sim b/tests/script/tsim/parser/last_cache.sim index 5f45b64796..7ffb3749aa 100644 --- a/tests/script/tsim/parser/last_cache.sim +++ b/tests/script/tsim/parser/last_cache.sim @@ -7,7 +7,7 @@ print ======================== dnode1 start $db = testdb sql drop database if exists $db -sql create database $db cachelast 2 +sql create database $db cachemodel 'last_value' sql use $db sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) diff --git a/tests/script/tsim/parser/like.sim b/tests/script/tsim/parser/like.sim index 3d1bff6abf..e7c191ed92 100644 --- a/tests/script/tsim/parser/like.sim +++ b/tests/script/tsim/parser/like.sim @@ -8,7 +8,7 @@ print ======================== dnode1 start $db = testdb sql drop database if exists $db -sql create database $db cachelast 2 +sql create database $db cachemodel 'last_value' sql use $db $table1 = table_name diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 449f5c7806..fce3cc7043 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -25,7 +25,7 @@ class TDTestCase: def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ): tdLog.info(" prepare datas for auto check abs function ") - tdSql.execute(" create database test cachelast 1 ") + tdSql.execute(" create database test cachemodel 'last_row' ") tdSql.execute(" use test ") tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)") @@ -63,7 +63,7 @@ class TDTestCase: def prepare_datas(self): - tdSql.execute("create database if not exists db keep 3650 duration 1000 cachelast 1") + tdSql.execute("create database if not exists db keep 3650 duration 1000 cachemodel 'last_row'") tdSql.execute("use db") tdSql.execute( '''create table stb1 @@ -124,7 +124,7 @@ class TDTestCase: def prepare_tag_datas(self): # prepare datas tdSql.execute( - "create database if not exists testdb keep 3650 duration 1000 cachelast 1") + "create database if not exists testdb keep 3650 duration 1000 cachemodel 'last_row'") tdSql.execute(" use testdb ") tdSql.execute(f" create stable stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp , uc1 int unsigned,\ @@ -528,7 +528,7 @@ class TDTestCase: def check_boundary_values(self): tdSql.execute("drop database if exists bound_test") - tdSql.execute("create database if not exists bound_test cachelast 2") + tdSql.execute("create database if not exists bound_test cachemodel 'last_value'") time.sleep(3) tdSql.execute("use bound_test") tdSql.execute( diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 4611726c14..12b14b0661 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -122,9 +122,9 @@ class TDTestCase: tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') tdSql.query('show databases;') - tdSql.checkData(2,5,'no_strict') - tdSql.error('alter database db strict 0') - # tdSql.execute('alter database db strict 1') + tdSql.checkData(2,5,'off') + tdSql.error('alter database db strict 'off'') + # tdSql.execute('alter database db strict 'on'') # tdSql.query('show databases;') # tdSql.checkData(2,5,'strict') From d730bc3b0006fb1758c77f14a20bde9f57c40299 Mon Sep 17 00:00:00 2001 From: tomchon Date: Sat, 16 Jul 2022 12:15:52 +0800 Subject: [PATCH 084/124] test:modify testcase --- .../tsim/sync/3Replica5VgElect3mnodedrop.sim | 204 +++++++++--------- 1 file changed, 102 insertions(+), 102 deletions(-) diff --git a/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim index ae02a23c9b..507d01595d 100644 --- a/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim +++ b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim @@ -110,21 +110,21 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] == FOLLOWER then - if $data[0][8] == FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][8] == FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][6] == FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -132,21 +132,21 @@ else goto check_vg_ready endi -if $data[1][4] == LEADER then - if $data[1][6] == FOLLOWER then - if $data[1][8] == FOLLOWER then +if $data[1][4] == leader then + if $data[1][6] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] endi endi -elif $data[1][6] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][8] == FOLLOWER then +elif $data[1][6] == leader then + if $data[1][4] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] endi endi -elif $data[1][8] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][6] == FOLLOWER then +elif $data[1][8] == leader then + if $data[1][4] == follower then + if $data[1][6] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] endi endi @@ -154,21 +154,21 @@ else goto check_vg_ready endi -if $data[2][4] == LEADER then - if $data[2][6] == FOLLOWER then - if $data[2][8] == FOLLOWER then +if $data[2][4] == leader then + if $data[2][6] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] endi endi -elif $data[2][6] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][8] == FOLLOWER then +elif $data[2][6] == leader then + if $data[2][4] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] endi endi -elif $data[2][8] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][6] == FOLLOWER then +elif $data[2][8] == leader then + if $data[2][4] == follower then + if $data[2][6] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] endi endi @@ -176,21 +176,21 @@ else goto check_vg_ready endi -if $data[3][4] == LEADER then - if $data[3][6] == FOLLOWER then - if $data[3][8] == FOLLOWER then +if $data[3][4] == leader then + if $data[3][6] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] endi endi -elif $data[3][6] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][8] == FOLLOWER then +elif $data[3][6] == leader then + if $data[3][4] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] endi endi -elif $data[3][8] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][6] == FOLLOWER then +elif $data[3][8] == leader then + if $data[3][4] == follower then + if $data[3][6] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] endi endi @@ -198,21 +198,21 @@ else goto check_vg_ready endi -if $data[4][4] == LEADER then - if $data[4][6] == FOLLOWER then - if $data[4][8] == FOLLOWER then +if $data[4][4] == leader then + if $data[4][6] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] endi endi -elif $data[4][6] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][8] == FOLLOWER then +elif $data[4][6] == leader then + if $data[4][4] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] endi endi -elif $data[4][8] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][6] == FOLLOWER then +elif $data[4][8] == leader then + if $data[4][4] == follower then + if $data[4][6] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] endi endi @@ -305,13 +305,13 @@ if $data[0][0] != 1 then return -1 endi -if $data[0][2] != LEADER then +if $data[0][2] != leader then goto check_mnode_ready_2 endi -if $data[1][2] != FOLLOWER then +if $data[1][2] != follower then goto check_mnode_ready_2 endi -if $data[2][2] != FOLLOWER then +if $data[2][2] != follower then goto check_mnode_ready_2 endi @@ -337,21 +337,21 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] == FOLLOWER then - if $data[0][8] == FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][8] == FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][6] == FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -359,21 +359,21 @@ else goto check_vg_ready1 endi -if $data[1][4] == LEADER then - if $data[1][6] == FOLLOWER then - if $data[1][8] == FOLLOWER then +if $data[1][4] == leader then + if $data[1][6] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] endi endi -elif $data[1][6] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][8] == FOLLOWER then +elif $data[1][6] == leader then + if $data[1][4] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] endi endi -elif $data[1][8] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][6] == FOLLOWER then +elif $data[1][8] == leader then + if $data[1][4] == follower then + if $data[1][6] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] endi endi @@ -381,21 +381,21 @@ else goto check_vg_ready1 endi -if $data[2][4] == LEADER then - if $data[2][6] == FOLLOWER then - if $data[2][8] == FOLLOWER then +if $data[2][4] == leader then + if $data[2][6] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] endi endi -elif $data[2][6] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][8] == FOLLOWER then +elif $data[2][6] == leader then + if $data[2][4] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] endi endi -elif $data[2][8] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][6] == FOLLOWER then +elif $data[2][8] == leader then + if $data[2][4] == follower then + if $data[2][6] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] endi endi @@ -403,21 +403,21 @@ else goto check_vg_ready1 endi -if $data[3][4] == LEADER then - if $data[3][6] == FOLLOWER then - if $data[3][8] == FOLLOWER then +if $data[3][4] == leader then + if $data[3][6] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] endi endi -elif $data[3][6] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][8] == FOLLOWER then +elif $data[3][6] == leader then + if $data[3][4] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] endi endi -elif $data[3][8] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][6] == FOLLOWER then +elif $data[3][8] == leader then + if $data[3][4] == follower then + if $data[3][6] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] endi endi @@ -425,21 +425,21 @@ else goto check_vg_ready1 endi -if $data[4][4] == LEADER then - if $data[4][6] == FOLLOWER then - if $data[4][8] == FOLLOWER then +if $data[4][4] == leader then + if $data[4][6] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] endi endi -elif $data[4][6] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][8] == FOLLOWER then +elif $data[4][6] == leader then + if $data[4][4] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] endi endi -elif $data[4][8] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][6] == FOLLOWER then +elif $data[4][8] == leader then + if $data[4][4] == follower then + if $data[4][6] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] endi endi @@ -558,27 +558,27 @@ if $data[0][0] != 1 then return -1 endi -if $data[0][2] == LEADER then - if $data[1][2] != FOLLOWER then +if $data[0][2] == leader then + if $data[1][2] != follower then goto check_mnode_ready_3 endi - if $data[2][2] != FOLLOWER then + if $data[2][2] != follower then goto check_mnode_ready_3 endi endi -if $data[1][2] == LEADER then - if $data[0][2] != FOLLOWER then +if $data[1][2] == leader then + if $data[0][2] != follower then goto check_mnode_ready_3 endi - if $data[2][2] != FOLLOWER then + if $data[2][2] != follower then goto check_mnode_ready_3 endi endi -if $data[2][2] == LEADER then - if $data[1][2] != FOLLOWER then +if $data[2][2] == leader then + if $data[1][2] != follower then goto check_mnode_ready_3 endi - if $data[0][2] != FOLLOWER then + if $data[0][2] != follower then goto check_mnode_ready_3 endi endi From 8caa4bcad0483ee70275ce8178f6e167da5fcead Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 13:34:24 +0800 Subject: [PATCH 085/124] feat(query): add twa function scalar version --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 582fe5a09b..a3ec033773 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -109,6 +109,7 @@ int32_t apercentileScalarFunction(SScalarParam *pInput, int32_t inputNum, SScala int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index b9e2b6feaf..fd50b12515 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2320,6 +2320,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTwaFuncEnv, .initFunc = twaFunctionSetup, .processFunc = twaFunction, + .sprocessFunc = twaScalarFunction, .finalizeFunc = twaFinalize }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 13b29f8ae3..c7bb8b3d80 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2413,3 +2413,7 @@ int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return nonCalcScalarFunction(pInput, inputNum, pOutput); } + +int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return avgScalarFunction(pInput, inputNum, pOutput); +} From 59839f0d0aeb3ba31364ebfebb2ca206c154cea3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 13:41:42 +0800 Subject: [PATCH 086/124] feat(query): add mavg scalar version --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index a3ec033773..5cbe380234 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -110,6 +110,7 @@ int32_t spreadScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index fd50b12515..31776610b6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2447,6 +2447,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getMavgFuncEnv, .initFunc = mavgFunctionSetup, .processFunc = mavgFunction, + .sprocessFunc = mavgScalarFunction, .finalizeFunc = NULL }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index c7bb8b3d80..9687528136 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2417,3 +2417,7 @@ int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return avgScalarFunction(pInput, inputNum, pOutput); } + +int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return avgScalarFunction(pInput, inputNum, pOutput); +} From a760fb8071b96f59a6b535112b6be1371d29a819 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 16 Jul 2022 05:48:09 +0000 Subject: [PATCH 087/124] fix coredump --- source/dnode/vnode/src/meta/metaSnapshot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 2143a3f085..85261d302e 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -180,7 +180,7 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) SMetaEntry metaEntry = {0}; SDecoder* pDecoder = &(SDecoder){0}; - tDecoderInit(pDecoder, pData, nData); + tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); metaDecodeEntry(pDecoder, &metaEntry); code = metaHandleEntry(pMeta, &metaEntry); From a57d2417c350d59ae9b14874e69964aa1afe3be7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 14:01:49 +0800 Subject: [PATCH 088/124] feat(query): add hyperloglog scalar version TD-17344 --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 5 +++-- source/libs/scalar/src/sclfunc.c | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 5cbe380234..34b455379e 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -111,6 +111,7 @@ int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 31776610b6..6f31448360 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2369,6 +2369,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getHLLFuncEnv, .initFunc = functionSetup, .processFunc = hllFunction, + .sprocessFunc = hllScalarFunction, .finalizeFunc = hllFinalize, .invertFunc = NULL, .combineFunc = hllCombine, @@ -2463,7 +2464,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "tail", .type = FUNCTION_TYPE_TAIL, - .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | + .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateTail, .getEnvFunc = getTailFuncEnv, @@ -2474,7 +2475,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "unique", .type = FUNCTION_TYPE_UNIQUE, - .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | + .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateUnique, .getEnvFunc = getUniqueFuncEnv, diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 9687528136..1442b5b5b5 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2421,3 +2421,7 @@ int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return avgScalarFunction(pInput, inputNum, pOutput); } + +int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return countScalarFunction(pInput, inputNum, pOutput); +} From eeb870924189d1e38617a36b238fc11bec3e1b3c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 16 Jul 2022 14:05:26 +0800 Subject: [PATCH 089/124] fix(query): add returned code check. --- source/libs/executor/src/scanoperator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6aacb0ee01..001ddf9d5b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -520,6 +520,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + return NULL; + } } SSDataBlock* result = doTableScanGroup(pOperator); From 6d63d1b5b811b38b2748bdef9f5e86504e6c4450 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 14:07:32 +0800 Subject: [PATCH 090/124] fix count scalar function behavior --- source/libs/scalar/src/sclfunc.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 1442b5b5b5..339ea101a6 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1746,20 +1746,14 @@ int32_t countScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam SColumnInfoData *pOutputData = pOutput->columnData; int64_t *out = (int64_t *)pOutputData->pData; - bool hasNull = false; *out = 0; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - hasNull = true; - break; + continue; } (*out)++; } - if (hasNull) { - colDataAppendNULL(pOutputData, 0); - } - pOutput->numOfRows = 1; return TSDB_CODE_SUCCESS; } From 00d7dba673f10ca6d8c373eec9831df694a05e65 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jul 2022 13:37:46 +0800 Subject: [PATCH 091/124] test: valgrind case --- tests/script/jenkins/basic.txt | 2 +- tests/script/tsim/parser/auto_create_tb.sim | 34 +++--- tests/script/tsim/valgrind/basic1.sim | 113 +++++++------------- tests/script/tsim/valgrind/basic2.sim | 79 +++++++------- 4 files changed, 100 insertions(+), 128 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 6c1938d9fc..01896ec161 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -87,7 +87,7 @@ ./test.sh -f tsim/parser/alter__for_community_version.sim ./test.sh -f tsim/parser/alter_column.sim ./test.sh -f tsim/parser/alter_stable.sim -# jira ./test.sh -f tsim/parser/auto_create_tb.sim +# nojira ./test.sh -f tsim/parser/auto_create_tb.sim ./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim ./test.sh -f tsim/parser/between_and.sim ./test.sh -f tsim/parser/binary_escapeCharacter.sim diff --git a/tests/script/tsim/parser/auto_create_tb.sim b/tests/script/tsim/parser/auto_create_tb.sim index 673a472cf7..485f4f480c 100644 --- a/tests/script/tsim/parser/auto_create_tb.sim +++ b/tests/script/tsim/parser/auto_create_tb.sim @@ -184,23 +184,26 @@ if $data(3)[8] != 涛思数据3 then return -1 endi -sql select count(*), first(c9) from $stb group by t1 order by t1 asc slimit 2 soffset 1 -if $rows != 2 then +sql select t1, count(*), first(c9) from $stb partition by t1 order by t1 asc slimit 3 +if $rows != 3 then return -1 endi -if $data00 != 1 then +if $data(1)[1] != 1 then return -1 endi -if $data01 != 涛思数据2 then +if $data(1)[2] != 涛思数据1 then return -1 endi -if $data02 != 2 then +if $data(2)[1] != 1 then return -1 endi -if $data11 != 涛思数据3 then +if $data(2)[2] != 涛思数据2 then return -1 endi -if $data12 != 3 then +if $data(3)[1] != 1 then + return -1 +endi +if $data(3)[2] != 涛思数据3 then return -1 endi @@ -248,23 +251,26 @@ if $data(3)[8] != 涛思数据3 then return -1 endi -sql select count(*), first(c9) from $stb group by t1 order by t1 asc slimit 2 soffset 1 -if $rows != 2 then +sql select t1, count(*), first(c9) from $stb partition by t1 order by t1 asc slimit 3 +if $rows != 3 then return -1 endi -if $data00 != 1 then +if $data(1)[1] != 1 then return -1 endi -if $data01 != 涛思数据2 then +if $data(1)[2] != 涛思数据1 then return -1 endi -if $data02 != 2 then +if $data(2)[1] != 1 then return -1 endi -if $data11 != 涛思数据3 then +if $data(2)[2] != 涛思数据2 then return -1 endi -if $data12 != 3 then +if $data(3)[1] != 1 then + return -1 +endi +if $data(3)[2] != 涛思数据3 then return -1 endi diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 86926509bb..3e39f35fa7 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -1,87 +1,50 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c debugflag -v 131 -system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode1 -s start -v sql connect -print =============== step1: create drop show dnodes -$x = 0 -step1: - $x = $x + 1 - sleep 1000 - if $x == 10 then - print ---> dnode not ready! - return -1 - endi -sql show dnodes -print ---> $data00 $data01 $data02 $data03 $data04 $data05 -if $rows != 1 then +print ======== step1 +sql drop database if exists db1; +sql create database db1 vgroups 3; +sql use db1; +sql create stable st1 (ts timestamp, f1 int, f2 binary(200)) tags(t1 int); +sql create table tb1 using st1 tags(1); +sql insert into tb1 values ('2022-07-07 10:01:01', 11, "aaa"); +sql insert into tb1 values ('2022-07-07 11:01:02', 12, "bbb"); +sql create table tb2 using st1 tags(2); +sql insert into tb2 values ('2022-07-07 10:02:01', 21, "aaa"); +sql insert into tb2 values ('2022-07-07 11:02:02', 22, "bbb"); +sql create table tb3 using st1 tags(3); +sql insert into tb3 values ('2022-07-07 10:03:01', 31, "aaa"); +sql insert into tb3 values ('2022-07-07 11:03:02', 32, "bbb"); +sql create table tb4 using st1 tags(4); + +sql insert into tb4 select * from tb1; + +goto _OVER + +sql select * from tb4; +if $rows != 2 then return -1 endi -if $data(1)[4] != ready then - goto step1 -endi - -print =============== step2: create db -sql create database d1 vgroups 3 buffer 3 -sql show databases -sql use d1 -sql show vgroups - -print =============== step3: create show stable, include all type -sql create table if not exists stb (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(16), c9 nchar(16), c10 timestamp, c11 tinyint unsigned, c12 smallint unsigned, c13 int unsigned, c14 bigint unsigned) tags (t1 bool, t2 tinyint, t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 binary(16), t9 nchar(16), t10 timestamp, t11 tinyint unsigned, t12 smallint unsigned, t13 int unsigned, t14 bigint unsigned) -sql create stable if not exists stb_1 (ts timestamp, c1 int) tags (j int) -sql create table stb_2 (ts timestamp, c1 int) tags (t1 int) -sql create stable stb_3 (ts timestamp, c1 int) tags (t1 int) -sql show stables -if $rows != 4 then +sql insert into tb4 select ts,f1,f2 from st1; +sql select * from tb4; +if $rows != 6 then return -1 endi - -print =============== step4: ccreate child table -sql create table c1 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql create table c2 using stb tags(false, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 2', 'child tbl 2', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql show tables -if $rows != 2 then +sql create table tba (ts timestamp, f1 binary(10), f2 bigint, f3 double); +sql_error insert into tba select * from tb1; +sql insert into tba (ts,f2,f1) select * from tb1; +sql select * from tba; +if $rows != 2 then + return -1 +endi +sql create table tbb (ts timestamp, f1 binary(10), f2 bigint, f3 double); +sql insert into tbb (f2,f1,ts) select f1+1,f2,ts+3 from tb2; +sql select * from tbb; +if $rows != 2 then return -1 endi - -print =============== step5: insert data -sql insert into c1 values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c1 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+2s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c2 values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c2 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+2s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) - -print =============== step6: alter insert -sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) - -print =============== restart -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode1 -s start -v - -print =============== stepa: query data -sql select * from c1 -sql select * from stb -sql select * from stb_1 -sql select ts, c1, c2, c3 from c1 -sql select ts, c1, c2, c3 from stb -sql select ts, c1 from stb_2 -sql select ts, c1, t1 from c1 -sql select ts, c1, t1 from stb -sql select ts, c1, t1 from stb_2 - -print =============== stepb: count -sql select count(*) from c1; -sql select count(*) from stb; -sql select count(ts), count(c1), count(c2), count(c3) from c1 -sql select count(ts), count(c1), count(c2), count(c3) from stb - -print =============== stepc: func -sql select first(ts), first(c1), first(c2), first(c3) from c1 -sql select min(c2), min(c3), min(c4) from c1 -sql select max(c2), max(c3), max(c4) from c1 -sql select sum(c2), sum(c3), sum(c4) from c1 _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -96,4 +59,4 @@ endi if $system_content == $null then return -1 -endi +endi \ No newline at end of file diff --git a/tests/script/tsim/valgrind/basic2.sim b/tests/script/tsim/valgrind/basic2.sim index 15e092da35..d3c72d1e5c 100644 --- a/tests/script/tsim/valgrind/basic2.sim +++ b/tests/script/tsim/valgrind/basic2.sim @@ -23,62 +23,65 @@ if $data(1)[4] != ready then endi print =============== step2: create db -sql create database d1 vgroups 3 buffer 3 +sql create database d1 vgroups 2 buffer 3 sql show databases sql use d1 sql show vgroups -print =============== step3: create show stable, include all type -sql create table if not exists stb (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(16), c9 nchar(16), c10 timestamp, c11 tinyint unsigned, c12 smallint unsigned, c13 int unsigned, c14 bigint unsigned) tags (t1 bool, t2 tinyint, t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 binary(16), t9 nchar(16), t10 timestamp, t11 tinyint unsigned, t12 smallint unsigned, t13 int unsigned, t14 bigint unsigned) -sql create stable if not exists stb_1 (ts timestamp, c1 int) tags (j int) -sql create table stb_2 (ts timestamp, c1 int) tags (t1 int) -sql create stable stb_3 (ts timestamp, c1 int) tags (t1 int) +print =============== step3: create show stable +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) sql show stables -if $rows != 4 then +if $rows != 1 then return -1 endi -print =============== step4: ccreate child table -sql create table c1 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql create table c2 using stb tags(false, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 2', 'child tbl 2', '2022-02-25 18:00:00.000', 10, 20, 30, 40) +print =============== step4: create show table +sql create table ct1 using stb tags(1000) +sql create table ct2 using stb tags(2000) +sql create table ct3 using stb tags(3000) sql show tables -if $rows != 2 then +if $rows != 3 then return -1 endi print =============== step5: insert data -sql insert into c1 values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c1 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+2s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c2 values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c2 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+2s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) +sql insert into ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) -print =============== step6: alter insert -sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) -sql insert into c3 using stb tags(true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) - -goto _OVER -print =============== stepa: query data -sql select * from c1 +print =============== step6: query data +sql select * from ct1 sql select * from stb -sql select * from stb_1 -sql select ts, c1, c2, c3 from c1 +sql select c1, c2, c3 from ct1 sql select ts, c1, c2, c3 from stb -sql select ts, c1 from stb_2 -sql select ts, c1, t1 from c1 -sql select ts, c1, t1 from stb -sql select ts, c1, t1 from stb_2 -print =============== stepb: count -#sql select count(*) from c1; -#sql select count(*) from stb; -#sql select count(ts), count(c1), count(c2), count(c3) from c1 -#sql select count(ts), count(c1), count(c2), count(c3) from stb +print =============== step7: count +sql select count(*) from ct1; +sql select count(*) from stb; +sql select count(ts), count(c1), count(c2), count(c3) from ct1 +sql select count(ts), count(c1), count(c2), count(c3) from stb -print =============== stepc: func -#sql select first(ts), first(c1), first(c2), first(c3) from c1 -#sql select min(c1), min(c2), min(c3) from c1 -#sql select max(c1), max(c2), max(c3) from c1 -#sql select sum(c1), sum(c2), sum(c3) from c1 +print =============== step8: func +sql select first(ts), first(c1), first(c2), first(c3) from ct1 +sql select min(c1), min(c2), min(c3) from ct1 +sql select max(c1), max(c2), max(c3) from ct1 +sql select sum(c1), sum(c2), sum(c3) from ct1 + +print =============== step9: insert select +sql create table ct4 using stb tags(4000); +sql insert into ct4 select * from ct1; +sql select * from ct4; +sql insert into ct4 select ts,c1,c2,c3 from stb; + +sql create table tb1 (ts timestamp, c1 int, c2 float, c3 double); +sql insert into tb1 (ts, c1, c2, c3) select * from ct1; +sql select * from tb1; + +sql create table tb2 (ts timestamp, f1 binary(10), c1 int, c2 double); +sql insert into tb2 (c2, c1, ts) select c2+1, c1, ts+3 from ct2; +sql select * from tb2; _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT From 2d1428eadd2465abfd275479986defe0a37420d7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 14:13:45 +0800 Subject: [PATCH 092/124] feat(query): add csum function scalar version TD-17344 --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 34b455379e..a6ed944632 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -112,6 +112,7 @@ int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6f31448360..185f529963 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2438,6 +2438,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getCsumFuncEnv, .initFunc = functionSetup, .processFunc = csumFunction, + .sprocessFunc = csumScalarFunction, .finalizeFunc = NULL }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 339ea101a6..b94379c7b5 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2419,3 +2419,7 @@ int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return countScalarFunction(pInput, inputNum, pOutput); } + +int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return sumScalarFunction(pInput, inputNum, pOutput); +} From 299e117d382379a1e49abe03ac05f907c27e22ff Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 16 Jul 2022 14:22:30 +0800 Subject: [PATCH 093/124] fix: database options 'strict' and 'cachelast' syntax adjustments --- source/libs/command/src/command.c | 16 ++++++------ tests/pytest/util/constant.py | 2 +- tests/script/tsim/db/alter_option.sim | 18 ++++++------- tests/script/tsim/db/create_all_options.sim | 22 ++++++++-------- tests/system-test/0-others/cachelast.py | 28 +++++++++++++-------- tests/system-test/6-cluster/5dnode1mnode.py | 2 +- 6 files changed, 47 insertions(+), 41 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index d22b3d88b4..3cb2553f2e 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -15,10 +15,10 @@ #include "command.h" #include "catalog.h" -#include "tdatablock.h" -#include "tglobal.h" #include "commandInt.h" #include "scheduler.h" +#include "tdatablock.h" +#include "tglobal.h" extern SConfig* tsCfg; @@ -222,7 +222,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S char* retentions = buildRetension(pCfg->pRetensions); len += sprintf(buf2 + VARSTR_HEADER_SIZE, - "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm " + "CREATE DATABASE `%s` BUFFER %d CACHEMODEL %d COMP %d DURATION %dm " "FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " "STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d", dbFName, pCfg->buffer, pCfg->cacheLast, pCfg->compression, pCfg->daysPerFile, pCfg->fsyncPeriod, @@ -483,7 +483,7 @@ static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableR static int32_t execAlterCmd(char* cmd, char* value, bool* processed) { int32_t code = 0; - + if (0 == strcasecmp(cmd, COMMAND_RESET_LOG)) { taosResetLog(); cfgDumpCfg(tsCfg, 0, false); @@ -502,13 +502,13 @@ _return: if (code) { terrno = code; } - - return code; + + return code; } static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { bool processed = false; - + if (execAlterCmd(pStmt->config, pStmt->value, &processed)) { return terrno; } @@ -516,7 +516,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { if (processed) { goto _return; } - + if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CMD)) { return terrno; } diff --git a/tests/pytest/util/constant.py b/tests/pytest/util/constant.py index 509d87e489..c4541386b4 100644 --- a/tests/pytest/util/constant.py +++ b/tests/pytest/util/constant.py @@ -54,7 +54,7 @@ TAOS_KEYWORDS = [ "BOOL", "EQ", "LINEAR", "RESET", "TSERIES", "BY", "EXISTS", "LOCAL", "RESTRICT", "UMINUS", "CACHE", "EXPLAIN", "LP", "ROW", "UNION", - "CACHELAST", "FAIL", "LSHIFT", "RP", "UNSIGNED", + "CACHEMODEL", "FAIL", "LSHIFT", "RP", "UNSIGNED", "CASCADE", "FILE", "LT", "RSHIFT", "UPDATE", "CHANGE", "FILL", "MATCH", "SCORES", "UPLUS", "CLUSTER", "FLOAT", "MAXROWS", "SELECT", "USE", diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index cda1656888..1648eb56fa 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -40,13 +40,13 @@ print ============= create database #database_option: { # | BUFFER value [3~16384, default: 96] # | PAGES value [64~16384, default: 256] -# | CACHELAST value [0, 1, 2, 3] +# | CACHEMODEL value ['node', 'last_row', 'last_value', 'both'] # | FSYNC value [0 ~ 180000 ms] # | KEEP value [duration, 365000] # | REPLICA value [1 | 3] # | WAL value [1 | 2] -sql create database db CACHELAST 3 COMP 0 DURATION 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 +sql create database db CACHEMODEL 'both' COMP 0 DURATION 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 sql show databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -102,7 +102,7 @@ endi if $data15_db != 0 then # comp return -1 endi -if $data16_db != 3 then # cachelast +if $data16_db != both then # cachelast return -1 endi if $data17_db != ns then # precision @@ -336,37 +336,37 @@ print ============== modify cachelast [0, 1, 2, 3] sql alter database db cachemodel 'last_value' sql show databases print cachelast $data16_db -if $data16_db != 2 then +if $data16_db != last_value then return -1 endi sql alter database db cachemodel 'last_row' sql show databases print cachelast $data16_db -if $data16_db != 1 then +if $data16_db != last_row then return -1 endi sql alter database db cachemodel 'none' sql show databases print cachelast $data16_db -if $data16_db != 0 then +if $data16_db != none then return -1 endi sql alter database db cachemodel 'last_value' sql show databases print cachelast $data16_db -if $data16_db != 2 then +if $data16_db != last_value then return -1 endi sql alter database db cachemodel 'both' sql show databases print cachelast $data16_db -if $data16_db != 3 then +if $data16_db != both then return -1 endi sql_error alter database db cachelast 4 sql_error alter database db cachelast 10 -sql_error alter database db cachelast -1 +sql_error alter database db cachelast 'other' print ============== modify precision sql_error alter database db precision 'ms' diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index ee65229be1..9de44ca335 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -40,7 +40,7 @@ print ============= create database with all options # | BUFFER value [3~16384, default: 96] # | PAGES value [64~16384, default: 256] # | PAGESIZE value [1~16384, default: 4] -# | CACHELAST value [0, 1, 2, 3, default: 0] +# | CACHEMODEL value ['node', 'last_row', 'last_value', 'both', default: 'node'] # | COMP [0 | 1 | 2, default: 2] # | DURATION value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] # | FSYNC value [0 ~ 180000 ms, default: 3000] @@ -122,7 +122,7 @@ endi if $data15_db != 2 then # comp return -1 endi -if $data16_db != 0 then # cachelast +if $data16_db != none then # cachelast return -1 endi if $data17_db != ms then # precision @@ -167,32 +167,32 @@ sql drop database db #endi #sql drop database db -print ====> CACHELAST value [0, 1, 2, 3, default: 0] -sql create database db CACHELAST 1 +print ====> CACHEMODEL value [0, 1, 2, 3, default: 0] +sql create database db CACHEMODEL 'last_row' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data16_db != 1 then +if $data16_db != last_row then return -1 endi sql drop database db -sql create database db CACHELAST 2 +sql create database db CACHEMODEL 'last_value' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data16_db != 2 then +if $data16_db != last_value then return -1 endi sql drop database db -sql create database db CACHELAST 3 +sql create database db CACHEMODEL 'both' sql show databases print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db -if $data16_db != 3 then +if $data16_db != both then return -1 endi sql drop database db -sql_error create database db CACHELAST 4 -sql_error create database db CACHELAST -1 +sql_error create database db CACHEMODEL 'other' +sql_error create database db CACHEMODEL '-1' print ====> COMP [0 | 1 | 2, default: 2] sql create database db COMP 1 diff --git a/tests/system-test/0-others/cachelast.py b/tests/system-test/0-others/cachelast.py index b7b4148179..2df6b8c9cc 100644 --- a/tests/system-test/0-others/cachelast.py +++ b/tests/system-test/0-others/cachelast.py @@ -36,22 +36,31 @@ class TDTestCase: def illegal_params(self): - illegal_params = ["1","0","NULL","None","False","True" ,"keep","now" ,"*" , "," ,"_" , "abc" ,"keep"] + illegal_params = ["1","0","NULL","False","True" ,"keep","now" ,"*" , "," ,"_" , "abc" ,"keep"] for value in illegal_params: - tdSql.error("create database testdb replica 1 cachelast '%s' " %value) + tdSql.error("create database testdb replica 1 cachemodel '%s' " %value) unexpected_numbers = [-1 , 0.0 , 3.0 , 4, 10 , 100] for number in unexpected_numbers: - tdSql.error("create database testdb replica 1 cachelast %s " %number) + tdSql.error("create database testdb replica 1 cachemodel %s " %number) + def getCacheModelStr(self, value): + numbers = { + 0 : "none", + 1 : "last_row", + 2 : "last_value", + 3 : "both" + } + return numbers.get(value, 'other') def prepare_datas(self): for i in range(4): - tdSql.execute("create database test_db_%d replica 1 cachelast %d " %(i,i)) - tdSql.execute("use test_db_%d"%i) + str = self.getCacheModelStr(i) + tdSql.execute("create database testdb_%s replica 1 cachemodel '%s' " %(str, str)) + tdSql.execute("use testdb_%s"%str) tdSql.execute("create stable st(ts timestamp , c1 int ,c2 float ) tags(ind int) ") tdSql.execute("create table tb1 using st tags(1) ") tdSql.execute("create table tb2 using st tags(2) ") @@ -81,10 +90,10 @@ class TDTestCase: # cache_last_set value for k , v in cache_lasts.items(): - if k.split("_")[-1]==str(v): - tdLog.info(" database %s cache_last value check pass, value is %d "%(k,v) ) + if k=="testdb_"+str(v): + tdLog.info(" database %s cache_last value check pass, value is %s "%(k,v) ) else: - tdLog.exit(" database %s cache_last value check fail, value is %d "%(k,v) ) + tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,v) ) # # check storage layer implementation @@ -132,13 +141,10 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - - self.illegal_params() self.prepare_datas() self.check_cache_last_sets() self.restart_check_cache_last_sets() - def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 12b14b0661..391cf3396d 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -123,7 +123,7 @@ class TDTestCase: tdSql.query('show databases;') tdSql.checkData(2,5,'off') - tdSql.error('alter database db strict 'off'') + tdSql.error("alter database db strict 'off'") # tdSql.execute('alter database db strict 'on'') # tdSql.query('show databases;') # tdSql.checkData(2,5,'strict') From 55407c50e559f9d6e5dc040bdfd28c4b2894e81c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 16 Jul 2022 14:44:31 +0800 Subject: [PATCH 094/124] fix(query): disable the optimized implementation of first/last/lastrow function. --- source/libs/function/src/builtinsimpl.c | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 12b796c5ca..a9a569480d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2723,6 +2723,9 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + // please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function. + // we will use this opt implementation in an new version that is only available in scan subplan +#if 0 if (blockDataOrder == TSDB_ORDER_ASC) { // filter according to current result firstly if (pResInfo->numOfRes > 0) { @@ -2770,6 +2773,22 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { } } } +#else + for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } + + numOfElems++; + + char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); + if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { + doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); + pResInfo->numOfRes = 1; + } + } +#endif SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; @@ -2801,6 +2820,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + // please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function. +#if 0 if (blockDataOrder == TSDB_ORDER_ASC) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { @@ -2833,6 +2854,22 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { break; } } +#else + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } + + numOfElems++; + + char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } + } +#endif SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; @@ -2988,6 +3025,9 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; +#if 0 + // the optimized version only function if all tuples in one block are monotonious increasing or descreasing. + // this is NOT always works if project operator exists in downstream. if (blockDataOrder == TSDB_ORDER_ASC) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { char* data = colDataGetData(pInputCol, i); @@ -2997,6 +3037,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { doSaveLastrow(pCtx, data, i, cts, pInfo); } + break; } } else { // descending order @@ -3011,7 +3052,19 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { break; } } +#else + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); + numOfElems++; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveLastrow(pCtx, data, i, cts, pInfo); + pResInfo->numOfRes = 1; + } + } + +#endif SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -5926,6 +5979,7 @@ int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) { TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { doSaveLastrow(pCtx, data, i, cts, pInfo); + pResInfo->numOfRes = 1; } } From 9ea66e08af630173c2613da473b4b62012287b90 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 16 Jul 2022 14:58:55 +0800 Subject: [PATCH 095/124] enh: change redirect to network_unavail --- source/client/src/clientMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 62052457fd..b4c2615f70 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -142,6 +142,7 @@ void taos_close(TAOS *taos) { int taos_errno(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { + if (terrno == TSDB_CODE_RPC_REDIRECT) errno = TSDB_CODE_RPC_NETWORK_UNAVAIL; return terrno; } @@ -149,11 +150,13 @@ int taos_errno(TAOS_RES *res) { return 0; } + if (pRequest->code == TSDB_CODE_RPC_REDIRECT) pRequest->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; return ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { + if (terrno == TSDB_CODE_RPC_REDIRECT) errno = TSDB_CODE_RPC_NETWORK_UNAVAIL; return (const char *)tstrerror(terrno); } @@ -165,6 +168,7 @@ const char *taos_errstr(TAOS_RES *res) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { + if (pRequest->code == TSDB_CODE_RPC_REDIRECT) pRequest->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; return (const char *)tstrerror(pRequest->code); } } From 3b094cd21a451662d66660d2c96993470190f400 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jul 2022 15:02:04 +0800 Subject: [PATCH 096/124] test: valgrind case --- tests/script/tsim/valgrind/basic3.sim | 139 +++++---------------- tests/script/tsim/valgrind/checkError2.sim | 30 ++++- 2 files changed, 56 insertions(+), 113 deletions(-) diff --git a/tests/script/tsim/valgrind/basic3.sim b/tests/script/tsim/valgrind/basic3.sim index 7fba66c468..6a42a8eb7f 100644 --- a/tests/script/tsim/valgrind/basic3.sim +++ b/tests/script/tsim/valgrind/basic3.sim @@ -1,13 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start -v -system sh/exec.sh -n dnode2 -s start -v sql connect -print =============== add dnode2 into cluster -sql create dnode $hostname port 7200 - +print =============== step1: create drop show dnodes $x = 0 step1: $x = $x + 1 @@ -18,135 +15,59 @@ step1: endi sql show dnodes print ---> $data00 $data01 $data02 $data03 $data04 $data05 -print ---> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 2 then +if $rows != 1 then return -1 endi if $data(1)[4] != ready then goto step1 endi -if $data(2)[4] != ready then - goto step1 -endi - -print =============== create database, stable, table -sql create database db vgroups 3 -sql use db -sql create table stb (ts timestamp, c int) tags (t int) -sql create table t0 using stb tags (0) -sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10)); - -print =============== run show xxxx -sql show dnodes -if $rows != 2 then - return -1 -endi - -sql show mnodes -if $rows != 1 then - return -1 -endi +print =============== step2: create db +sql create database d1 vgroups 2 buffer 3 sql show databases -if $rows != 3 then - return -1 -endi +sql use d1 +sql show vgroups +print =============== step3: create show stable +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) sql show stables if $rows != 1 then return -1 endi +print =============== step4: create show table +sql create table ct1 using stb tags(1000) +sql create table ct2 using stb tags(2000) +sql create table ct3 using stb tags(3000) sql show tables -if $rows != 2 then - return -1 -endi - -sql show users -if $rows != 1 then - return -1 -endi - -sql show vgroups if $rows != 3 then return -1 endi -print =============== run select * from information_schema.xxxx -sql select * from information_schema.`dnodes` -if $rows != 2 then - return -1 -endi +print =============== step5: insert data +sql insert into ct1 values(now+0d, 10, 2.0, 3.0) +sql insert into ct1 values(now+1d, 11, 2.1, 3.1)(now+2d, -12, -2.2, -3.2)(now+3d, -13, -2.3, -3.3) +sql insert into ct2 values(now+0d, 10, 2.0, 3.0) +sql insert into ct2 values(now+1d, 11, 2.1, 3.1)(now+2d, -12, -2.2, -3.2)(now+3d, -13, -2.3, -3.3) +sql insert into ct3 values('2022-01-01 00:00:00.000', 10, 2.0, 3.0) -sql select * from information_schema.`mnodes` -if $rows != 1 then - return -1 -endi +print =============== step6: query data +sql select * from ct1 where ts < now -1d and ts > now +1d +sql select * from stb where ts < now -1d and ts > now +1d +sql select * from ct1 where ts < now -1d and ts > now +1d order by ts desc +sql select * from stb where ts < now -1d and ts > now +1d order by ts desc -sql select * from information_schema.user_databases -if $rows != 3 then - return -1 -endi - -sql select * from information_schema.user_stables -if $rows != 1 then - return -1 -endi - -sql select * from information_schema.user_tables -if $rows != 31 then - return -1 -endi - -sql select * from information_schema.user_users -if $rows != 1 then - return -1 -endi - -sql select * from information_schema.`vgroups` -if $rows != 3 then - return -1 -endi - -sql show variables; -if $rows != 4 then - return -1 -endi - -sql show dnode 1 variables; -if $rows <= 0 then - return -1 -endi - -sql show local variables; -if $rows <= 0 then - return -1 -endi - -print ==== stop dnode1 and dnode2, and restart dnodes +_OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT +print =============== check +$null= -print =============== check dnode1 system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content <= 0 then - return 0 +if $system_content > 1 then + return -1 endi -$null= if $system_content == $null then - return 0 -endi - -print =============== check dnode2 -system_content sh/checkValgrind.sh -n dnode2 -print cmd return result ----> [ $system_content ] -if $system_content <= 0 then - return 0 -endi - -$null= -if $system_content == $null then - return 0 + return -1 endi diff --git a/tests/script/tsim/valgrind/checkError2.sim b/tests/script/tsim/valgrind/checkError2.sim index 3a82218015..9746fe9ecf 100644 --- a/tests/script/tsim/valgrind/checkError2.sim +++ b/tests/script/tsim/valgrind/checkError2.sim @@ -23,7 +23,7 @@ if $data(1)[4] != ready then endi print =============== step2: create db -sql create database d1 vgroups 1 buffer 3 +sql create database d1 vgroups 2 buffer 3 sql show databases sql use d1 sql show vgroups @@ -44,24 +44,32 @@ if $rows != 3 then return -1 endi -print =============== step5: insert data +print =============== step5: insert data (null / update) sql insert into ct1 values(now+0s, 10, 2.0, 3.0) -sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into ct1 values(now+1s, 11, 2.1, NULL)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) sql insert into ct2 values(now+0s, 10, 2.0, 3.0) sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) -sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) +sql insert into ct3 values('2021-01-01 00:00:00.000', NULL, NULL, 3.0) +sql insert into ct3 values('2022-03-02 16:59:00.010', 3 , 4, 5), ('2022-03-02 16:59:00.010', 33 , 4, 5), ('2022-04-01 16:59:00.011', 4, 4, 5), ('2022-04-01 16:59:00.011', 6, 4, 5), ('2022-03-06 16:59:00.013', 8, 4, 5); +sql insert into ct3 values('2022-03-02 16:59:00.010', 103, 1, 2), ('2022-03-02 16:59:00.010', 303, 3, 4), ('2022-04-01 16:59:00.011', 40, 5, 6), ('2022-04-01 16:59:00.011', 60, 4, 5), ('2022-03-06 16:59:00.013', 80, 4, 5); print =============== step6: query data sql select * from ct1 sql select * from stb sql select c1, c2, c3 from ct1 sql select ts, c1, c2, c3 from stb +sql select * from ct1 where ts < now -1d and ts > now +1d +sql select * from stb where ts < now -1d and ts > now +1d +#sql select * from ct1 where ts < now -1d and ts > now +1d order by ts desc +#sql select * from stb where ts < now -1d and ts > now +1d order by ts desc print =============== step7: count sql select count(*) from ct1; sql select count(*) from stb; sql select count(ts), count(c1), count(c2), count(c3) from ct1 sql select count(ts), count(c1), count(c2), count(c3) from stb +sql select count(*) from ct1 where ts < now -1d and ts > now +1d +sql select count(*) from stb where ts < now -1d and ts > now +1d print =============== step8: func sql select first(ts), first(c1), first(c2), first(c3) from ct1 @@ -69,6 +77,20 @@ sql select min(c1), min(c2), min(c3) from ct1 sql select max(c1), max(c2), max(c3) from ct1 sql select sum(c1), sum(c2), sum(c3) from ct1 +print =============== step9: insert select +#sql create table ct4 using stb tags(4000); +#sql insert into ct4 select * from ct1; +#sql select * from ct4; +#sql insert into ct4 select ts,c1,c2,c3 from stb; + +#sql create table tb1 (ts timestamp, c1 int, c2 float, c3 double); +#sql insert into tb1 (ts, c1, c2, c3) select * from ct1; +#sql select * from tb1; + +#sql create table tb2 (ts timestamp, f1 binary(10), c1 int, c2 double); +#sql insert into tb2 (c2, c1, ts) select c2+1, c1, ts+3 from ct2; +#sql select * from tb2; + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check From 790cecf12306965f9d8a28a207b8d08f04e61333 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 16 Jul 2022 15:04:45 +0800 Subject: [PATCH 097/124] feat(query): add diff function scalar version --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index a6ed944632..dfdb69ee3c 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -113,6 +113,7 @@ int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 185f529963..10e0808c4d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2408,6 +2408,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getDiffFuncEnv, .initFunc = diffFunctionSetup, .processFunc = diffFunction, + .sprocessFunc = diffScalarFunction, .finalizeFunc = functionFinalize }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index b94379c7b5..3f26cd46f8 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2408,6 +2408,10 @@ int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam return nonCalcScalarFunction(pInput, inputNum, pOutput); } +int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return nonCalcScalarFunction(pInput, inputNum, pOutput); +} + int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return avgScalarFunction(pInput, inputNum, pOutput); } From 6f29fd4020f1ecabbd78140f84b3353fdb124d66 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 16 Jul 2022 15:34:22 +0800 Subject: [PATCH 098/124] fix(sync): snapshot begin index --- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/src/syncAppendEntries.c | 5 +- source/libs/sync/src/syncAppendEntriesReply.c | 16 +++- source/libs/sync/src/syncMain.c | 11 +++ tests/script/tsim/sync/vnodesnapshot-test.sim | 81 ------------------- 6 files changed, 32 insertions(+), 84 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 0914827950..1645aa37f9 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -119,7 +119,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { taosRealPath(tdir, NULL, sizeof(tdir)); // for test tsdb snapshot -#if 0 +#if 1 pVnode->config.walCfg.segSize = 200; pVnode->config.walCfg.retentionSize = 2000; #endif diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 1f26033f63..0cbd7d36b2 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -223,6 +223,7 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode); // snapshot -------------- bool syncNodeHasSnapshot(SSyncNode* pSyncNode); +void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode); SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode); SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index dca80f6826..9678b335fd 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -711,6 +711,9 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc syncNodeEventLog(ths, logBuf); } while (0); + // maybe update commit index by snapshot + syncNodeMaybeUpdateCommitBySnapshot(ths); + // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; @@ -718,7 +721,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc pReply->term = ths->pRaftStore->currentTerm; pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; pReply->success = false; - pReply->matchIndex = SYNC_INDEX_INVALID; + pReply->matchIndex = ths->commitIndex; // msg event log do { diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index c18c2b4d38..bb55e3d9d6 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -147,6 +147,15 @@ static void syncNodeStartSnapshotOnce(SSyncNode* ths, SyncIndex beginIndex, Sync int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; + // print log + do { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, term:%lu, match:%ld, success:%d", pMsg->term, + pMsg->matchIndex, pMsg->success); + syncNodeEventLog(ths, logBuf); + + } while (0); + // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { syncNodeEventLog(ths, "recv sync-append-entries-reply, maybe replica already dropped"); @@ -238,7 +247,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie SSnapshot oldSnapshot; ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &oldSnapshot); SyncTerm newSnapshotTerm = oldSnapshot.lastApplyTerm; - syncNodeStartSnapshotOnce(ths, SYNC_INDEX_BEGIN, nextIndex, newSnapshotTerm, pMsg); + syncNodeStartSnapshotOnce(ths, pMsg->matchIndex + 1, nextIndex, newSnapshotTerm, pMsg); // get sender SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); @@ -256,6 +265,11 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie } syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + SyncIndex oldMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId)); + if (pMsg->matchIndex > oldMatchIndex) { + syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); + } + // event log, update next-index do { char host[64]; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b6c01f2923..bb7454ea6f 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1083,6 +1083,17 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { return pSyncNode; } +void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode) { + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + SSnapshot snapshot; + int32_t code = pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + ASSERT(code == 0); + if (snapshot.lastApplyIndex > pSyncNode->commitIndex) { + pSyncNode->commitIndex = snapshot.lastApplyIndex; + } + } +} + void syncNodeStart(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { diff --git a/tests/script/tsim/sync/vnodesnapshot-test.sim b/tests/script/tsim/sync/vnodesnapshot-test.sim index 8589078b50..2f0eccf02e 100644 --- a/tests/script/tsim/sync/vnodesnapshot-test.sim +++ b/tests/script/tsim/sync/vnodesnapshot-test.sim @@ -170,84 +170,3 @@ if $rows != 100 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -######################################################## - - -######################################################## -print ===> start dnode1 dnode3 dnode4 -system sh/exec.sh -n dnode1 -s start -#system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start -system sh/exec.sh -n dnode4 -s start - -sleep 7000 - -print =============== query data -sql connect -sql use db -sql select * from ct1 -print rows: $rows -print $data00 $data01 $data02 -if $rows != 100 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -#system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -######################################################## - - -######################################################## -print ===> start dnode1 dnode2 dnode4 -system sh/exec.sh -n dnode1 -s start -system sh/exec.sh -n dnode2 -s start -#system sh/exec.sh -n dnode3 -s start -system sh/exec.sh -n dnode4 -s start - -sleep 3000 - -print =============== query data -sql select * from ct1 -print rows: $rows -print $data00 $data01 $data02 -if $rows != 100 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -#system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -######################################################## - - -######################################################## -print ===> start dnode1 dnode2 dnode3 -system sh/exec.sh -n dnode1 -s start -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start -#system sh/exec.sh -n dnode4 -s start - -sleep 3000 - -print =============== query data -sql select * from ct1 -print rows: $rows -print $data00 $data01 $data02 -if $rows != 100 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -#system sh/exec.sh -n dnode4 -s stop -x SIGINT -######################################################## - - From 92ec6ccea0a711b8abe0a70773f94044918dba40 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 16 Jul 2022 15:37:48 +0800 Subject: [PATCH 099/124] enh: change redirect to network_unavail --- source/client/src/clientMain.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b4c2615f70..77b31011a3 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -142,7 +142,7 @@ void taos_close(TAOS *taos) { int taos_errno(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) errno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; return terrno; } @@ -150,13 +150,13 @@ int taos_errno(TAOS_RES *res) { return 0; } - if (pRequest->code == TSDB_CODE_RPC_REDIRECT) pRequest->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - return ((SRequestObj *)res)->code; + return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL + : ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) errno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; return (const char *)tstrerror(terrno); } @@ -168,8 +168,8 @@ const char *taos_errstr(TAOS_RES *res) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { - if (pRequest->code == TSDB_CODE_RPC_REDIRECT) pRequest->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - return (const char *)tstrerror(pRequest->code); + return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL) + : (const char *)tstrerror(pRequest->code); } } From e0b56ea593a0542fa50de65923576797b091e03c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 16 Jul 2022 15:38:46 +0800 Subject: [PATCH 100/124] fix: fix query retry issue --- source/dnode/mgmt/mgmt_qnode/src/qmWorker.c | 1 + source/libs/catalog/src/ctgAsync.c | 6 +-- source/libs/executor/src/executorimpl.c | 1 + source/libs/qworker/inc/qwInt.h | 6 ++- source/libs/qworker/inc/qwMsg.h | 1 + source/libs/qworker/src/qwDbg.c | 58 ++++++++++++++------- source/libs/qworker/src/qwMsg.c | 1 - source/libs/qworker/src/qworker.c | 23 ++++---- source/libs/scheduler/inc/schInt.h | 17 +++--- source/libs/scheduler/src/schJob.c | 46 +++++++++++----- source/libs/scheduler/src/schRemote.c | 12 ++--- source/libs/scheduler/src/schTask.c | 36 ++++++++----- 12 files changed, 132 insertions(+), 76 deletions(-) diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 6814643b59..1c7edbe6be 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -81,6 +81,7 @@ int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { taosWriteQitem(pMgmt->queryWorker.queue, pMsg); return 0; case READ_QUEUE: + case FETCH_QUEUE: dTrace("msg:%p, is created and will put into qnode-fetch queue", pMsg); taosWriteQitem(pMgmt->fetchWorker.queue, pMsg); return 0; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index e77df8f7f2..920acbac2e 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -163,7 +163,7 @@ int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -178,7 +178,7 @@ int32_t ctgInitGetDnodeTask(SCtgJob *pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -264,7 +264,7 @@ int32_t ctgInitGetSvrVerTask(SCtgJob *pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " [%dth] task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 89542571ea..72279d63b2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1978,6 +1978,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { qDebug("%s fetch rsp received, index:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfRows); } else { pSourceDataInfo->code = code; + qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 56af7ee1e0..c8e5204e91 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -77,7 +77,7 @@ typedef struct SQWDebug { bool dumpEnable; bool sleepSimulate; bool deadSimulate; - bool tmp; + bool redirectSimulate; } SQWDebug; extern SQWDebug gQWDebug; @@ -380,7 +380,9 @@ void qwDbgDumpMgmtInfo(SQWorker *mgmt); int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore); int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SEpSet *pEpSet); int32_t qwAddTaskCtx(QW_FPARAMS_DEF); -int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped); +void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped); +void qwDbgSimulateSleep(void); +void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped); #ifdef __cplusplus diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index 16fe6b21c2..5378934343 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -46,6 +46,7 @@ int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *rsp, int32_t code); int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SRpcHandleInfo *pConn); +int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); #ifdef __cplusplus } diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 4e174018f4..98d7825b2c 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -9,7 +9,7 @@ #include "tmsg.h" #include "tname.h" -SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true, .tmp = false}; +SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = false, .redirectSimulate = false, .deadSimulate = false, .sleepSimulate = false}; int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { if (!gQWDebug.statusEnable) { @@ -147,8 +147,17 @@ int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int return TSDB_CODE_SUCCESS; } -int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) { - if (gQWDebug.tmp) { +void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) { + static int32_t ignoreTime = 0; + if (*rsped) { + return; + } + + if (gQWDebug.redirectSimulate) { + if (++ignoreTime <= 10) { + return; + } + if (TDMT_SCH_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { SEpSet epSet = {0}; epSet.inUse = 1; @@ -163,44 +172,55 @@ int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) { ctx->phase = QW_PHASE_POST_QUERY; qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, &epSet); *rsped = true; - return TSDB_CODE_SUCCESS; + return; } if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { QW_SET_PHASE(ctx, QW_PHASE_POST_QUERY); qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); *rsped = true; - return TSDB_CODE_SUCCESS; + return; } - if ((TDMT_SCH_FETCH == qwMsg->msgType) && (0 == taosRand() % 3)) { + if ((TDMT_SCH_FETCH == qwMsg->msgType) && (0 == taosRand() % 9)) { qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); *rsped = true; - return TSDB_CODE_SUCCESS; + return; } } - - *rsped = false; - return TSDB_CODE_SUCCESS; } -void qwDbgSimulateSleep() { +void qwDbgSimulateSleep(void) { if (!gQWDebug.sleepSimulate) { return; } - taosSsleep(taosRand() % 10); + static int32_t ignoreTime = 0; + if (++ignoreTime > 10) { + taosSsleep(taosRand() % 20); + } } -void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t msgType) { +void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) { if (!gQWDebug.deadSimulate) { return; } - SRpcHandleInfo *pConn = ((msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) ? &ctx->dataConnInfo : &ctx->ctrlConnInfo); - qwBuildAndSendErrorRsp(msgType + 1, pConn, TSDB_CODE_RPC_BROKEN_LINK); + if (*rsped) { + return; + } - qwDropTask(QW_FPARAMS()); + static int32_t ignoreTime = 0; + + if (++ignoreTime > 10 && 0 == taosRand() % 9) { + SRpcHandleInfo *pConn = ((ctx->msgType == TDMT_SCH_FETCH || ctx->msgType == TDMT_SCH_MERGE_FETCH) ? &ctx->dataConnInfo : &ctx->ctrlConnInfo); + qwBuildAndSendErrorRsp(ctx->msgType + 1, pConn, TSDB_CODE_RPC_BROKEN_LINK); + + qwBuildAndSendDropMsg(QW_FPARAMS(), pConn); + *rsped = true; + + return; + } } @@ -236,9 +256,9 @@ int32_t qwDbgEnableDebug(char *option) { return TSDB_CODE_SUCCESS; } - if (0 == strcasecmp(option, "tmp")) { - gQWDebug.tmp = true; - qError("qw tmp debug enabled"); + if (0 == strcasecmp(option, "redirect")) { + gQWDebug.redirectSimulate = true; + qError("qw redirect debug enabled"); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 8bbd03f735..b8d9957c30 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -198,7 +198,6 @@ int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { int32_t code = tmsgPutToQueue(&mgmt->msgCb, FETCH_QUEUE, &pNewMsg); if (TSDB_CODE_SUCCESS != code) { QW_SCH_TASK_ELOG("put drop task msg to queue failed, vgId:%d, code:%s", mgmt->nodeId, tstrerror(code)); - rpcFreeCont(req); QW_ERR_RET(code); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 7e12a3a424..e99695e962 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -83,6 +83,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { // if *taskHandle is NULL, it's killed right now if (taskHandle) { + qwDbgSimulateSleep(); code = qExecTask(taskHandle, &pRes, &useconds); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { @@ -431,12 +432,12 @@ _return: bool rsped = false; SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; - qwDbgResponseRedirect(&qwMsg, ctx, &rsped); + qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); + qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); if (!rsped) { qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); - } - - QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); + } } if (ctx) { @@ -656,13 +657,12 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); ctx->msgType = qwMsg->msgType; + ctx->dataConnInfo = qwMsg->connInfo; SOutputData sOutput = {0}; QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); if (NULL == rsp) { - ctx->dataConnInfo = qwMsg->connInfo; - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); } else { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); @@ -708,12 +708,15 @@ _return: if (code || rsp) { bool rsped = false; - qwDbgResponseRedirect(qwMsg, ctx, &rsped); + if (ctx) { + qwDbgSimulateRedirect(qwMsg, ctx, &rsped); + qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); + } if (!rsped) { qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); + QW_TASK_DLOG("%s send, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1), qwMsg->connInfo.handle, code, tstrerror(code), + dataLen); } - QW_TASK_DLOG("%s send, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1), qwMsg->connInfo.handle, code, tstrerror(code), - dataLen); } QW_RET(TSDB_CODE_SUCCESS); @@ -745,8 +748,6 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } if (!dropped) { - ctx->ctrlConnInfo = qwMsg->connInfo; - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); } diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 4979a41f17..65b45cc612 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -55,13 +55,11 @@ typedef enum { #define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 #define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 200 // unit is TSDB_TABLE_NUM_UNIT #define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ +#define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 #define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000 #define SCH_MAX_TASK_TIMEOUT_USEC 60000000 -#define SCH_MAX_CANDIDATE_EP_NUM (TSDB_MAX_REPLICA + 100) - - - +#define SCH_DEFAULT_MAX_RETRY_NUM 6 typedef struct SSchDebug { bool lockEnable; @@ -275,7 +273,8 @@ typedef struct SSchJob { int32_t errCode; SRWLatch resLock; SExecResult execRes; - void *resData; //TODO free it or not + void *fetchRes; //TODO free it or not + bool fetched; int32_t resNumOfRows; SSchResInfo userRes; const char *sql; @@ -327,7 +326,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) #define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) #define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)))) -#define SCH_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) +#define SCH_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_LINK_BROKEN || (_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) #define SCH_TASK_NEED_REDIRECT(_task, _msgType, _code, _rspLen) (SCH_REDIRECT_MSGTYPE(_msgType) && (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_MERGE_TASK_NETWORK_ERR((_task), (_code), (_rspLen)))) #define SCH_NEED_RETRY(_msgType, _code) ((SCH_NETWORK_ERR(_code) && SCH_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) @@ -369,6 +368,8 @@ extern SSchedulerMgmt schMgmt; qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) #define SCH_TASK_DLOG(param, ...) \ qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) +#define SCH_TASK_TLOG(param, ...) \ + qTrace("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) #define SCH_TASK_DLOGL(param, ...) \ qDebugL("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) #define SCH_TASK_WLOG(param, ...) \ @@ -442,7 +443,7 @@ void schFreeRpcCtx(SRpcCtx *pCtx); int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp); bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus); int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask); -int32_t schSaveJobQueryRes(SSchJob *pJob, SQueryTableRsp *rsp); +int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp); int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp); void schProcessOnDataFetched(SSchJob *job); int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask); @@ -493,7 +494,7 @@ int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode); int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode); int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode); -bool schChkCurrentOp(SSchJob *pJob, int32_t op, bool sync); +bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync); extern SSchDebug gSCHDebug; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 1b1268baf1..13a369fac9 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -110,7 +110,7 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_PART_SUCC: if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC && - newStatus != JOB_TASK_STATUS_DROP) { + newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -389,13 +389,18 @@ int32_t schDumpJobExecRes(SSchJob* pJob, SExecResult* pRes) { int32_t schDumpJobFetchRes(SSchJob* pJob, void** pData) { int32_t code = 0; - if (pJob->resData && ((SRetrieveTableRsp *)pJob->resData)->completed) { - SCH_ERR_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_SUCC, NULL)); + + SCH_LOCK(SCH_WRITE, &pJob->resLock); + + pJob->fetched = true; + + if (pJob->fetchRes && ((SRetrieveTableRsp *)pJob->fetchRes)->completed) { + SCH_ERR_JRET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_SUCC, NULL)); } while (true) { - *pData = atomic_load_ptr(&pJob->resData); - if (*pData != atomic_val_compare_exchange_ptr(&pJob->resData, *pData, NULL)) { + *pData = atomic_load_ptr(&pJob->fetchRes); + if (*pData != atomic_val_compare_exchange_ptr(&pJob->fetchRes, *pData, NULL)) { continue; } @@ -414,7 +419,11 @@ int32_t schDumpJobFetchRes(SSchJob* pJob, void** pData) { SCH_JOB_DLOG("fetch done, totalRows:%d", pJob->resNumOfRows); - return TSDB_CODE_SUCCESS; +_return: + + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + return code; } int32_t schNotifyUserExecRes(SSchJob* pJob) { @@ -512,8 +521,12 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) { } -int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { - schPostJobRes(pJob, SCH_OP_EXEC); +int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { + if (schChkCurrentOp(pJob, SCH_OP_FETCH, -1)) { + SCH_ERR_RET(schLaunchFetchTask(pJob)); + } else { + schPostJobRes(pJob, 0); + } return TSDB_CODE_SUCCESS; } @@ -526,7 +539,7 @@ int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRs SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); atomic_store_32(&pJob->resNumOfRows, htonl(pRsp->numOfRows)); - atomic_store_ptr(&pJob->resData, pRsp); + atomic_store_ptr(&pJob->fetchRes, pRsp); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); @@ -561,7 +574,7 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t schSaveJobQueryRes(SSchJob *pJob, SQueryTableRsp *rsp) { +int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp) { if (rsp->tbFName[0]) { SCH_LOCK(SCH_WRITE, &pJob->resLock); @@ -600,7 +613,7 @@ int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask) { int32_t schLaunchJob(SSchJob *pJob) { if (EXPLAIN_MODE_STATIC == pJob->attr.explainMode) { - SCH_ERR_RET(qExecStaticExplain(pJob->pDag, (SRetrieveTableRsp **)&pJob->resData)); + SCH_ERR_RET(qExecStaticExplain(pJob->pDag, (SRetrieveTableRsp **)&pJob->fetchRes)); SCH_ERR_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL)); } else { SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); @@ -661,7 +674,7 @@ void schFreeJobImpl(void *job) { qDestroyQueryPlan(pJob->pDag); taosMemoryFreeClear(pJob->userRes.execRes); - taosMemoryFreeClear(pJob->resData); + taosMemoryFreeClear(pJob->fetchRes); taosMemoryFree(pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); @@ -795,9 +808,14 @@ void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode) { } } -bool schChkCurrentOp(SSchJob *pJob, int32_t op, bool sync) { +bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync) { + bool r = false; SCH_LOCK(SCH_READ, &pJob->opStatus.lock); - bool r = (pJob->opStatus.op == op) && (pJob->opStatus.syncReq == sync); + if (sync >= 0) { + r = (pJob->opStatus.op == op) && (pJob->opStatus.syncReq == sync); + } else { + r = (pJob->opStatus.op == op); + } SCH_UNLOCK(SCH_READ, &pJob->opStatus.lock); return r; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 41d9f46a87..6983bbf013 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -256,7 +256,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SCH_ERR_JRET(rsp->code); - SCH_ERR_JRET(schSaveJobQueryRes(pJob, rsp)); + SCH_ERR_JRET(schSaveJobExecRes(pJob, rsp)); atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); @@ -277,8 +277,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - if (pJob->resData) { - SCH_TASK_ELOG("explain result is already generated, res:%p", pJob->resData); + if (pJob->fetchRes) { + SCH_TASK_ELOG("explain result is already generated, res:%p", pJob->fetchRes); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -325,13 +325,13 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa return TSDB_CODE_SUCCESS; } - if (pJob->resData) { - SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->resData); + if (pJob->fetchRes) { + SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->fetchRes); taosMemoryFreeClear(rsp); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } - atomic_store_ptr(&pJob->resData, rsp); + atomic_store_ptr(&pJob->fetchRes, rsp); atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); if (rsp->completed) { diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index bfba81b118..282e81bb5d 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -47,10 +47,10 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask) { void schInitTaskRetryTimes(SSchJob *pJob, SSchTask *pTask, SSchLevel *pLevel) { if (SCH_IS_DATA_BIND_TASK(pTask) || (!SCH_IS_QUERY_JOB(pJob)) || (SCH_ALL != schMgmt.cfg.schPolicy)) { - pTask->maxRetryTimes = SCH_MAX_CANDIDATE_EP_NUM; + pTask->maxRetryTimes = SCH_DEFAULT_MAX_RETRY_NUM; } else { int32_t nodeNum = taosArrayGetSize(pJob->nodeList); - pTask->maxRetryTimes = TMAX(nodeNum, SCH_MAX_CANDIDATE_EP_NUM); + pTask->maxRetryTimes = TMAX(nodeNum, SCH_DEFAULT_MAX_RETRY_NUM); } pTask->maxExecTimes = pTask->maxRetryTimes * (pLevel->level + 1); @@ -64,11 +64,11 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * pTask->execId = -1; pTask->timeoutUsec = SCH_DEFAULT_TASK_TIMEOUT_USEC; pTask->taskId = schGenTaskId(); - pTask->execNodes = - taosHashInit(SCH_MAX_CANDIDATE_EP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); schInitTaskRetryTimes(pJob, pTask, pLevel); + pTask->execNodes = + taosHashInit(pTask->maxExecTimes, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pTask->profile.execTime = taosArrayInit(pTask->maxExecTimes, sizeof(int64_t)); if (NULL == pTask->execNodes || NULL == pTask->profile.execTime) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -405,6 +405,18 @@ _return: int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { int32_t code = 0; + if (JOB_TASK_STATUS_PART_SUCC == pJob->status) { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->fetched) { + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + SCH_TASK_ELOG("already fetched while got error %s", tstrerror(rspCode)); + SCH_ERR_RET(rspCode); + } + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC); + } + if (SCH_IS_DATA_BIND_TASK(pTask)) { if (NULL == pData->pEpSet) { SCH_TASK_ELOG("no epset updated while got error %s", tstrerror(rspCode)); @@ -602,7 +614,7 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { if (pJob->nodeList) { nodeNum = taosArrayGetSize(pJob->nodeList); - for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { + for (int32_t i = 0; i < nodeNum; ++i) { SQueryNodeLoad *nload = taosArrayGet(pJob->nodeList, i); SQueryNodeAddr *naddr = &nload->addr; @@ -611,8 +623,8 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("set %dth candidate addr, id %d, fqdn:%s, port:%d", i, naddr->nodeId, SCH_GET_CUR_EP(naddr)->fqdn, - SCH_GET_CUR_EP(naddr)->port); + SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId, naddr->epSet.inUse, naddr->epSet.numOfEps, + SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port); ++addNum; } @@ -632,9 +644,9 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } pTask->candidateIdx = 0; - pTask->candidateAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); + pTask->candidateAddrs = taosArrayInit(SCHEDULE_DEFAULT_MAX_NODE_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->candidateAddrs) { - SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCH_MAX_CANDIDATE_EP_NUM); + SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCHEDULE_DEFAULT_MAX_NODE_NUM); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -897,9 +909,9 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { int32_t schLaunchFetchTask(SSchJob *pJob) { int32_t code = 0; - void *resData = atomic_load_ptr(&pJob->resData); - if (resData) { - SCH_JOB_DLOG("res already fetched, res:%p", resData); + void *fetchRes = atomic_load_ptr(&pJob->fetchRes); + if (fetchRes) { + SCH_JOB_DLOG("res already fetched, res:%p", fetchRes); return TSDB_CODE_SUCCESS; } From fddecc07cf2120e52bf24ddc8a74deab66aa2d7f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jul 2022 15:11:36 +0800 Subject: [PATCH 101/124] test: valgrind case --- tests/script/jenkins/basic.txt | 1 + tests/script/tsim/valgrind/checkError1.sim | 67 +++++++++------ tests/script/tsim/valgrind/checkError2.sim | 2 +- tests/script/tsim/valgrind/checkError3.sim | 2 +- tests/script/tsim/valgrind/checkError4.sim | 95 ++++++++++++++++++++++ 5 files changed, 141 insertions(+), 26 deletions(-) create mode 100644 tests/script/tsim/valgrind/checkError4.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 01896ec161..4f64908bd6 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -309,6 +309,7 @@ ./test.sh -f tsim/valgrind/checkError1.sim ./test.sh -f tsim/valgrind/checkError2.sim ./test.sh -f tsim/valgrind/checkError3.sim +# jira ./test.sh -f tsim/valgrind/checkError4.sim # --- vnode # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index fe4f4654b1..bd764c4064 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -1,28 +1,13 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode1 -c debugflag -v 131 +system sh/cfg.sh -n dnode2 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start -v +system sh/exec.sh -n dnode2 -s start -v sql connect -print =============== step1: show dnodes -$x = 0 -step1: - $x = $x + 1 - sleep 1000 - if $x == 10 then - print ---> dnode not ready! - return -1 - endi -sql show dnodes -print ---> $data00 $data01 $data02 $data03 $data04 $data05 -if $rows != 1 then - return -1 -endi -if $data(1)[4] != ready then - goto step1 -endi - -print =============== step2: create alter drop show user +print =============== step1: create alter drop show user sql create user u1 pass 'taosdata' sql show users sql alter user u1 sysinfo 1 @@ -31,11 +16,34 @@ sql alter user u1 pass 'taosdata' sql drop user u1 sql_error alter user u2 sysinfo 0 -print =============== step3: create drop dnode +print =============== step2 create drop dnode sql create dnode $hostname port 7200 -sql drop dnode 2 +sql create dnode $hostname port 7300 +sql drop dnode 3 sql alter dnode 1 'debugflag 131' +print =============== step3: show dnodes +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step3 +endi +if $data(2)[4] != ready then + goto step3 +endi + print =============== create database, stable, table sql create database db vgroups 3 sql use db @@ -45,7 +53,7 @@ sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10)); print =============== run show xxxx sql show dnodes -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -76,7 +84,7 @@ endi print =============== run select * from information_schema.xxxx sql select * from information_schema.`dnodes` -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -127,16 +135,27 @@ endi print =============== stop system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT print =============== check $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 1 then +if $system_content > 0 then return -1 endi if $system_content == $null then return -1 endi + +system_content sh/checkValgrind.sh -n dnode2 +print cmd return result ----> [ $system_content ] +if $system_content > 2 then + return -1 +endi + +if $system_content == $null then + return -1 +endi \ No newline at end of file diff --git a/tests/script/tsim/valgrind/checkError2.sim b/tests/script/tsim/valgrind/checkError2.sim index 9746fe9ecf..b14f817f0d 100644 --- a/tests/script/tsim/valgrind/checkError2.sim +++ b/tests/script/tsim/valgrind/checkError2.sim @@ -98,7 +98,7 @@ $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 1 then +if $system_content > 0 then return -1 endi diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 86926509bb..52ef01785e 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -90,7 +90,7 @@ $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 1 then +if $system_content > 0 then return -1 endi diff --git a/tests/script/tsim/valgrind/checkError4.sim b/tests/script/tsim/valgrind/checkError4.sim new file mode 100644 index 0000000000..be1dc003ad --- /dev/null +++ b/tests/script/tsim/valgrind/checkError4.sim @@ -0,0 +1,95 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 +system sh/exec.sh -n dnode1 -s start -v +sql connect + +print =============== step1: create drop show dnodes +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not ready! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +if $rows != 1 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 2 buffer 3 +sql create database d2 vgroups 2 buffer 3 +sql show databases; +sql show d1.vgroups; + +print =============== step3: create show stable +sql create table if not exists d1.stb1 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d1.stb2 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d2.stb1 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql show d1.stables +sql show d2.stables + +print =============== step4: create show table +sql create table d1.ct1 using d1.stb1 tags(1000) +sql create table d1.ct2 using d1.stb1 tags(2000) +sql create table d1.ct3 using d1.stb2 tags(3000) +sql create table d2.ct1 using d2.stb1 tags(1000) +sql create table d2.ct2 using d2.stb1 tags(2000) +sql show d1.tables +sql show d2.tables + +print =============== step5: insert data +sql insert into d1.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into d1.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into d1.ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) + +print =============== step6: create db +sql drop table d1.ct1 +sql drop table d1.stb2 +sql drop database d1 +sql drop database d2 + +goto _OVER +print =============== step7: repeat +sql create database d1 vgroups 2 buffer 3 +sql create database d2 vgroups 2 buffer 3 +sql create table if not exists d1.stb1 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d1.stb2 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d2.stb1 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table d1.ct1 using d1.stb1 tags(1000) +sql create table d1.ct2 using d1.stb1 tags(2000) +sql create table d1.ct3 using d1.stb2 tags(3000) +sql create table d2.ct1 using d2.stb1 tags(1000) +sql create table d2.ct2 using d2.stb1 tags(2000) +sql insert into d1.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into d1.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into d1.ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) +sql drop table d1.ct1 +sql drop table d1.stb2 +sql drop database d1 +sql drop database d2 + +_OVER: +system sh/exec.sh -n dnode1 -s stop -x SIGINT +print =============== check +$null= + +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content > 0 then + return -1 +endi + +if $system_content == $null then + return -1 +endi From 5cefb68ddefc875316d5deca96c42dbaa541d1e1 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 16 Jul 2022 16:00:16 +0800 Subject: [PATCH 102/124] fix(sync): snapshot end index --- source/libs/sync/src/syncAppendEntriesReply.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index bb55e3d9d6..5ad2cb8e94 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -247,7 +247,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie SSnapshot oldSnapshot; ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &oldSnapshot); SyncTerm newSnapshotTerm = oldSnapshot.lastApplyTerm; - syncNodeStartSnapshotOnce(ths, pMsg->matchIndex + 1, nextIndex, newSnapshotTerm, pMsg); + syncNodeStartSnapshotOnce(ths, pMsg->matchIndex + 1, oldSnapshot.lastApplyIndex, newSnapshotTerm, pMsg); // get sender SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); From 1a9f21c724e049272ef188a5bdb8692129e8a39f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 16 Jul 2022 08:02:39 +0000 Subject: [PATCH 103/124] fix coredump --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 04368ddfb7..f318c69c6f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1151,7 +1151,7 @@ int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFr code = tBlockDataAddColData(pBlockData, iColData, &pColData); if (code) goto _exit; - tColDataInit(pColData, pColDataFrom->cid, pColData->type, pColData->smaOn); + tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn); for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); if (code) goto _exit; From 49c953fa0cb2b07e18818e2b2b7b726f214540bb Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 16 Jul 2022 16:10:50 +0800 Subject: [PATCH 104/124] refactor(sync): disable wal cfg --- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 1645aa37f9..0914827950 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -119,7 +119,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { taosRealPath(tdir, NULL, sizeof(tdir)); // for test tsdb snapshot -#if 1 +#if 0 pVnode->config.walCfg.segSize = 200; pVnode->config.walCfg.retentionSize = 2000; #endif From c978bd10ec38e73803d88c1abb96b134056e6ee4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 16 Jul 2022 16:23:56 +0800 Subject: [PATCH 105/124] fix(sync): do not sender snapshot when there's wal --- source/libs/sync/src/syncAppendEntriesReply.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 5ad2cb8e94..5137922522 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -247,7 +247,14 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie SSnapshot oldSnapshot; ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &oldSnapshot); SyncTerm newSnapshotTerm = oldSnapshot.lastApplyTerm; - syncNodeStartSnapshotOnce(ths, pMsg->matchIndex + 1, oldSnapshot.lastApplyIndex, newSnapshotTerm, pMsg); + + SyncIndex endIndex; + if (ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex + 1)) { + endIndex = nextIndex; + } else { + endIndex = oldSnapshot.lastApplyIndex; + } + syncNodeStartSnapshotOnce(ths, pMsg->matchIndex + 1, endIndex, newSnapshotTerm, pMsg); // get sender SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); From 83b44e2ffadb0fc98a6d7e179a0b6ac99e1b4d20 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 16 Jul 2022 16:30:09 +0800 Subject: [PATCH 106/124] fix(query):support multigroup in groupby operator. --- source/libs/executor/src/executil.c | 3 -- source/libs/executor/src/executorimpl.c | 8 +-- source/libs/executor/src/groupoperator.c | 69 ++++++++++-------------- 3 files changed, 31 insertions(+), 49 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a8ff3188c8..061ac23905 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -115,9 +115,6 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int p->groupId = *(uint64_t*)key; p->pos = *(SResultRowPosition*)pData; memcpy(p->key, (char*)key + sizeof(uint64_t), keyLen - sizeof(uint64_t)); -#ifdef BUF_PAGE_DEBUG - qDebug("page_groupRes, groupId:%" PRIu64 ",pageId:%d,offset:%d\n", p->groupId, p->pos.pageId, p->pos.offset); -#endif taosArrayPush(pGroupResInfo->pRows, &p); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c8f2083456..28faa473db 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1506,15 +1506,11 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI int32_t numOfExprs) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t start = pGroupResInfo->index; -#ifdef BUF_PAGE_DEBUG - qDebug("\npage_copytoblock rows:%d", numOfRows); -#endif + for (int32_t i = start; i < numOfRows; i += 1) { SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); -#ifdef BUF_PAGE_DEBUG - qDebug("page_copytoblock pos pageId:%d, offset:%d", pPos->pos.pageId, pPos->pos.offset); -#endif + SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index ee20bc7ba6..c266e8be23 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -29,7 +29,7 @@ static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len); static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity); static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes, - int32_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup); + uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup); static void destroyGroupOperatorInfo(void* param, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*)param; @@ -264,7 +264,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); - int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, 0, pInfo->aggSup.pResultBuf, &pInfo->aggSup); + int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -282,7 +282,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, - 0, pInfo->aggSup.pResultBuf, &pInfo->aggSup); + pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -293,6 +293,29 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } } +static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { + SGroupbyOperatorInfo* pInfo = pOperator->info; + + SSDataBlock* pRes = pInfo->binfo.pRes; + while(1) { + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + doFilter(pInfo->pCondition, pRes); + + bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); + if (!hasRemain) { + doSetOperatorCompleted(pOperator); + break; + } + + if (pRes->info.rows > 0) { + break; + } + } + + pOperator->resultInfo.totalRows += pRes->info.rows; + return (pRes->info.rows == 0)? NULL:pRes; +} + static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -304,22 +327,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - while(1) { - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes); - - bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); - if (!hasRemain) { - doSetOperatorCompleted(pOperator); - break; - } - - if (pRes->info.rows > 0) { - break; - } - } - pOperator->resultInfo.totalRows += pRes->info.rows; - return (pRes->info.rows == 0)? NULL:pRes; + return buildGroupResultDataBlock(pOperator); } int32_t order = TSDB_ORDER_ASC; @@ -373,26 +381,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, 0); pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - - while(1) { - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes); - - bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); - if (!hasRemain) { - doSetOperatorCompleted(pOperator); - break; - } - - if (pRes->info.rows > 0) { - break; - } - } - - size_t rows = pRes->info.rows; - pOperator->resultInfo.totalRows += rows; - - return (rows == 0)? NULL:pRes; + return buildGroupResultDataBlock(pOperator); } SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, @@ -800,7 +789,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition } int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes, - int32_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) { + uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; From 402a65fd73e5039ed2758c86de2f8624d86dcebd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jul 2022 16:43:40 +0800 Subject: [PATCH 107/124] fix: definite lost while drop db --- source/dnode/mnode/impl/src/mndVgroup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0b24b570a5..3eb3a6cd1f 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -639,6 +639,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p if (pShow->pIter == NULL) break; if (pDb != NULL && pVgroup->dbUid != pDb->uid) { + sdbRelease(pSdb, pVgroup); continue; } From bb58ec1e023a773947e3fb11aeb7dc4c22b69d97 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 16 Jul 2022 16:44:07 +0800 Subject: [PATCH 108/124] feat(tmq): support background heartbeat --- include/common/tmsg.h | 142 ++--- include/common/tmsgdef.h | 1 + source/client/src/clientHb.c | 96 ++-- source/client/src/tmq.c | 563 +++++++++++--------- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mnode/impl/src/mndConsumer.c | 61 ++- source/dnode/mnode/impl/src/mndProfile.c | 64 +-- 7 files changed, 448 insertions(+), 480 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 49feba4d3c..592ba6c3bb 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -75,13 +75,18 @@ typedef uint16_t tmsg_t; #define TSDB_IE_TYPE_DNODE_EXT 6 #define TSDB_IE_TYPE_DNODE_STATE 7 -enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__UDFD, CONN_TYPE__MAX }; +enum { + CONN_TYPE__QUERY = 1, + CONN_TYPE__TMQ, + CONN_TYPE__UDFD, + CONN_TYPE__MAX, +}; enum { HEARTBEAT_KEY_USER_AUTHINFO = 1, HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_STBINFO, - HEARTBEAT_KEY_MQ_TMP, + HEARTBEAT_KEY_TMQ, }; typedef enum _mgmt_table { @@ -2145,6 +2150,15 @@ typedef struct { char cgroup[TSDB_CGROUP_LEN]; } SMqAskEpReq; +typedef struct { + int64_t consumerId; + int32_t epoch; +} SMqHbReq; + +typedef struct { + int8_t reserved; +} SMqHbRsp; + typedef struct { int32_t key; int32_t valueLen; @@ -2333,29 +2347,30 @@ static FORCE_INLINE int32_t tDecodeSClientHbKey(SDecoder* pDecoder, SClientHbKey return 0; } -typedef struct SMqHbVgInfo { +typedef struct { int32_t vgId; -} SMqHbVgInfo; + // TODO stas +} SMqReportVgInfo; -static FORCE_INLINE int32_t taosEncodeSMqVgInfo(void** buf, const SMqHbVgInfo* pVgInfo) { +static FORCE_INLINE int32_t taosEncodeSMqVgInfo(void** buf, const SMqReportVgInfo* pVgInfo) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgInfo->vgId); return tlen; } -static FORCE_INLINE void* taosDecodeSMqVgInfo(void* buf, SMqHbVgInfo* pVgInfo) { +static FORCE_INLINE void* taosDecodeSMqVgInfo(void* buf, SMqReportVgInfo* pVgInfo) { buf = taosDecodeFixedI32(buf, &pVgInfo->vgId); return buf; } -typedef struct SMqHbTopicInfo { +typedef struct { int32_t epoch; int64_t topicUid; char name[TSDB_TOPIC_FNAME_LEN]; SArray* pVgInfo; // SArray -} SMqHbTopicInfo; +} SMqTopicInfo; -static FORCE_INLINE int32_t taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) { +static FORCE_INLINE int32_t taosEncodeSMqTopicInfoMsg(void** buf, const SMqTopicInfo* pTopicInfo) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pTopicInfo->epoch); tlen += taosEncodeFixedI64(buf, pTopicInfo->topicUid); @@ -2363,35 +2378,35 @@ static FORCE_INLINE int32_t taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbT int32_t sz = taosArrayGetSize(pTopicInfo->pVgInfo); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { - SMqHbVgInfo* pVgInfo = (SMqHbVgInfo*)taosArrayGet(pTopicInfo->pVgInfo, i); + SMqReportVgInfo* pVgInfo = (SMqReportVgInfo*)taosArrayGet(pTopicInfo->pVgInfo, i); tlen += taosEncodeSMqVgInfo(buf, pVgInfo); } return tlen; } -static FORCE_INLINE void* taosDecodeSMqHbTopicInfoMsg(void* buf, SMqHbTopicInfo* pTopicInfo) { +static FORCE_INLINE void* taosDecodeSMqTopicInfoMsg(void* buf, SMqTopicInfo* pTopicInfo) { buf = taosDecodeFixedI32(buf, &pTopicInfo->epoch); buf = taosDecodeFixedI64(buf, &pTopicInfo->topicUid); buf = taosDecodeStringTo(buf, pTopicInfo->name); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); - pTopicInfo->pVgInfo = taosArrayInit(sz, sizeof(SMqHbVgInfo)); + pTopicInfo->pVgInfo = taosArrayInit(sz, sizeof(SMqReportVgInfo)); for (int32_t i = 0; i < sz; i++) { - SMqHbVgInfo vgInfo; + SMqReportVgInfo vgInfo; buf = taosDecodeSMqVgInfo(buf, &vgInfo); taosArrayPush(pTopicInfo->pVgInfo, &vgInfo); } return buf; } -typedef struct SMqHbMsg { +typedef struct { int32_t status; // ask hb endpoint int32_t epoch; int64_t consumerId; SArray* pTopics; // SArray -} SMqHbMsg; +} SMqReportReq; -static FORCE_INLINE int32_t taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) { +static FORCE_INLINE int32_t taosEncodeSMqReportMsg(void** buf, const SMqReportReq* pMsg) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pMsg->status); tlen += taosEncodeFixedI32(buf, pMsg->epoch); @@ -2399,22 +2414,22 @@ static FORCE_INLINE int32_t taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) { int32_t sz = taosArrayGetSize(pMsg->pTopics); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { - SMqHbTopicInfo* topicInfo = (SMqHbTopicInfo*)taosArrayGet(pMsg->pTopics, i); - tlen += taosEncodeSMqHbTopicInfoMsg(buf, topicInfo); + SMqTopicInfo* topicInfo = (SMqTopicInfo*)taosArrayGet(pMsg->pTopics, i); + tlen += taosEncodeSMqTopicInfoMsg(buf, topicInfo); } return tlen; } -static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { +static FORCE_INLINE void* taosDecodeSMqReportMsg(void* buf, SMqReportReq* pMsg) { buf = taosDecodeFixedI32(buf, &pMsg->status); buf = taosDecodeFixedI32(buf, &pMsg->epoch); buf = taosDecodeFixedI64(buf, &pMsg->consumerId); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); - pMsg->pTopics = taosArrayInit(sz, sizeof(SMqHbTopicInfo)); + pMsg->pTopics = taosArrayInit(sz, sizeof(SMqTopicInfo)); for (int32_t i = 0; i < sz; i++) { - SMqHbTopicInfo topicInfo; - buf = taosDecodeSMqHbTopicInfoMsg(buf, &topicInfo); + SMqTopicInfo topicInfo; + buf = taosDecodeSMqTopicInfoMsg(buf, &topicInfo); taosArrayPush(pMsg->pTopics, &topicInfo); } return buf; @@ -2919,89 +2934,6 @@ typedef struct { int32_t tEncodeSMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); int32_t tDecodeSMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); -#if 0 -typedef struct { - SMqRspHead head; - int64_t reqOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t blockNum; - int8_t withTbName; - int8_t withSchema; - SArray* blockDataLen; // SArray - SArray* blockData; // SArray - SArray* blockTbName; // SArray - SArray* blockSchema; // SArray -} SMqDataBlkRsp; - -static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); - tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); - tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); - tlen += taosEncodeFixedI32(buf, pRsp->blockNum); - if (pRsp->blockNum != 0) { - tlen += taosEncodeFixedI8(buf, pRsp->withTbName); - tlen += taosEncodeFixedI8(buf, pRsp->withSchema); - - for (int32_t i = 0; i < pRsp->blockNum; i++) { - int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); - void* data = taosArrayGetP(pRsp->blockData, i); - tlen += taosEncodeFixedI32(buf, bLen); - tlen += taosEncodeBinary(buf, data, bLen); - if (pRsp->withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRsp->blockSchema, i); - tlen += taosEncodeSSchemaWrapper(buf, pSW); - } - if (pRsp->withTbName) { - char* tbName = (char*)taosArrayGetP(pRsp->blockTbName, i); - tlen += taosEncodeString(buf, tbName); - } - } - } - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); - buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); - buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); - buf = taosDecodeFixedI32(buf, &pRsp->blockNum); - if (pRsp->blockNum != 0) { - pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); - buf = taosDecodeFixedI8(buf, &pRsp->withTbName); - buf = taosDecodeFixedI8(buf, &pRsp->withSchema); - if (pRsp->withTbName) { - pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); - } - if (pRsp->withSchema) { - pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); - } - - for (int32_t i = 0; i < pRsp->blockNum; i++) { - int32_t bLen = 0; - void* data = NULL; - buf = taosDecodeFixedI32(buf, &bLen); - buf = taosDecodeBinary(buf, &data, bLen); - taosArrayPush(pRsp->blockDataLen, &bLen); - taosArrayPush(pRsp->blockData, &data); - if (pRsp->withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); - buf = taosDecodeSSchemaWrapper(buf, pSW); - taosArrayPush(pRsp->blockSchema, &pSW); - } - if (pRsp->withTbName) { - char* name = NULL; - buf = taosDecodeString(buf, &name); - taosArrayPush(pRsp->blockTbName, &name); - } - } - } - return (void*)buf; -} -#endif - typedef struct { SMqRspHead head; char cgroup[TSDB_CGROUP_LEN]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index a24eda33bb..81a12d9b2f 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -144,6 +144,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "ask-ep", SMqAskEpReq, SMqAskEpRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "consumer-lost", SMqConsumerLostMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "consumer-recover", SMqConsumerRecoverMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_HB, "consumer-hb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "do-rebalance", SMqDoRebalanceMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DROP_CGROUP, "drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 2a9d113108..71f839d24c 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -24,9 +24,15 @@ static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); static void hbStopThread(); -static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; } +static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { + // + return 0; +} -static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } +static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { + // + return 0; +} static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; @@ -70,7 +76,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (NULL == vgInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - + vgInfo->vgVersion = rsp->vgVersion; vgInfo->hashMethod = rsp->hashMethod; vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); @@ -156,18 +162,18 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { STscObj *pTscObj = (STscObj *)acquireTscObj(pRsp->connKey.tscRid); if (NULL == pTscObj) { tscDebug("tscObj rid %" PRIx64 " not exist", pRsp->connKey.tscRid); - } else { + } else { if (pRsp->query->totalDnodes > 1 && !isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pRsp->query->epSet)) { - SEpSet* pOrig = &pTscObj->pAppInfo->mgmtEp.epSet; - SEp* pOrigEp = &pOrig->eps[pOrig->inUse]; - SEp* pNewEp = &pRsp->query->epSet.eps[pRsp->query->epSet.inUse]; - tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in hb", - pOrig->inUse, pOrig->numOfEps, pOrigEp->fqdn, pOrigEp->port, - pRsp->query->epSet.inUse, pRsp->query->epSet.numOfEps, pNewEp->fqdn, pNewEp->port); - + SEpSet *pOrig = &pTscObj->pAppInfo->mgmtEp.epSet; + SEp *pOrigEp = &pOrig->eps[pOrig->inUse]; + SEp *pNewEp = &pRsp->query->epSet.eps[pRsp->query->epSet.inUse]; + tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in hb", pOrig->inUse, pOrig->numOfEps, + pOrigEp->fqdn, pOrigEp->port, pRsp->query->epSet.inUse, pRsp->query->epSet.numOfEps, pNewEp->fqdn, + pNewEp->port); + updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pRsp->query->epSet); } - + pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; @@ -263,13 +269,13 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { } static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { - static int32_t emptyRspNum = 0; + static int32_t emptyRspNum = 0; char *key = (char *)param; SClientHbBatchRsp pRsp = {0}; if (TSDB_CODE_SUCCESS == code) { tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); } - + int32_t rspNum = taosArrayGetSize(pRsp.rsps); taosThreadMutexLock(&appInfo.mutex); @@ -373,7 +379,7 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { releaseTscObj(connKey->tscRid); return TSDB_CODE_QRY_OUT_OF_MEMORY; } - + hbBasic->connId = pTscObj->connId; int32_t numOfQueries = pTscObj->pRequests ? taosHashGetSize(pTscObj->pRequests) : 0; @@ -392,7 +398,6 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - int32_t code = hbBuildQueryDesc(hbBasic, pTscObj); if (code) { releaseTscObj(connKey->tscRid); @@ -436,13 +441,12 @@ int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S if (NULL == req->info) { req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); } - + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); return TSDB_CODE_SUCCESS; } - int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SDbVgVersion *dbs = NULL; uint32_t dbNum = 0; @@ -483,8 +487,8 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SSTableVersion *stbs = NULL; - uint32_t stbNum = 0; - int32_t code = 0; + uint32_t stbNum = 0; + int32_t code = 0; code = catalogGetExpiredSTables(pCatalog, &stbs, &stbNum); if (TSDB_CODE_SUCCESS != code) { @@ -521,20 +525,19 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC } int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { - SAppHbReq* pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); + SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL != pApp) { memcpy(&req->app, pApp, sizeof(*pApp)); } else { memset(&req->app.summary, 0, sizeof(req->app.summary)); req->app.pid = taosGetPId(); req->app.appId = clientHbMgr.appId; - taosGetAppName(req->app.name, NULL); + taosGetAppName(req->app.name, NULL); } return TSDB_CODE_SUCCESS; } - int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { int64_t *clusterId = (int64_t *)param; struct SCatalog *pCatalog = NULL; @@ -567,7 +570,8 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req return TSDB_CODE_SUCCESS; } -void hbMgrInitMqHbHandle() { +static FORCE_INLINE void hbMgrInitHandle() { + // init all handle clientHbMgr.reqHandle[CONN_TYPE__QUERY] = hbQueryHbReqHandle; clientHbMgr.reqHandle[CONN_TYPE__TMQ] = hbMqHbReqHandle; @@ -575,11 +579,6 @@ void hbMgrInitMqHbHandle() { clientHbMgr.rspHandle[CONN_TYPE__TMQ] = hbMqHbRspHandle; } -static FORCE_INLINE void hbMgrInitHandle() { - // init all handle - hbMgrInitMqHbHandle(); -} - SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { SClientHbBatchReq *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { @@ -602,7 +601,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { continue; } - //hbClearClientHbReq(pOneReq); + // hbClearClientHbReq(pOneReq); pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); } @@ -615,11 +614,9 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { return pBatchReq; } -void hbThreadFuncUnexpectedStopped(void) { - atomic_store_8(&clientHbMgr.threadStop, 2); -} +void hbThreadFuncUnexpectedStopped(void) { atomic_store_8(&clientHbMgr.threadStop, 2); } -void hbMergeSummary(SAppClusterSummary* dst, SAppClusterSummary* src) { +void hbMergeSummary(SAppClusterSummary *dst, SAppClusterSummary *src) { dst->numOfInsertsReq += src->numOfInsertsReq; dst->numOfInsertRows += src->numOfInsertRows; dst->insertElapsedTime += src->insertElapsedTime; @@ -633,7 +630,7 @@ void hbMergeSummary(SAppClusterSummary* dst, SAppClusterSummary* src) { int32_t hbGatherAppInfo(void) { SAppHbReq req = {0}; - int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); + int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); if (sz > 0) { req.pid = taosGetPId(); req.appId = clientHbMgr.appId; @@ -641,11 +638,11 @@ int32_t hbGatherAppInfo(void) { } taosHashClear(clientHbMgr.appSummary); - + for (int32_t i = 0; i < sz; ++i) { SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); - uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId; - SAppHbReq* pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); + uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId; + SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL == pApp) { memcpy(&req.summary, &pAppHbMgr->pAppInstInfo->summary, sizeof(req.summary)); req.startTime = pAppHbMgr->startTime; @@ -654,7 +651,7 @@ int32_t hbGatherAppInfo(void) { if (pAppHbMgr->startTime < pApp->startTime) { pApp->startTime = pAppHbMgr->startTime; } - + hbMergeSummary(&pApp->summary, &pAppHbMgr->pAppInstInfo->summary); } } @@ -662,7 +659,6 @@ int32_t hbGatherAppInfo(void) { return TSDB_CODE_SUCCESS; } - static void *hbThreadFunc(void *param) { setThreadName("hb"); #ifdef WINDOWS @@ -681,7 +677,7 @@ static void *hbThreadFunc(void *param) { if (sz > 0) { hbGatherAppInfo(); } - + for (int i = 0; i < sz; i++) { SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); @@ -698,7 +694,6 @@ static void *hbThreadFunc(void *param) { if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); - //hbClearReqInfo(pAppHbMgr); break; } @@ -708,7 +703,6 @@ static void *hbThreadFunc(void *param) { if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); - //hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); break; } @@ -725,7 +719,6 @@ static void *hbThreadFunc(void *param) { SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); tFreeClientHbBatchReq(pReq); - //hbClearReqInfo(pAppHbMgr); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } @@ -759,7 +752,7 @@ static void hbStopThread() { return; } - taosThreadJoin(clientHbMgr.thread, NULL); + taosThreadJoin(clientHbMgr.thread, NULL); tscDebug("hb thread stopped"); } @@ -808,7 +801,7 @@ void hbFreeAppHbMgr(SAppHbMgr *pTarget) { } taosHashCleanup(pTarget->activeInfo); pTarget->activeInfo = NULL; - + taosMemoryFree(pTarget->key); taosMemoryFree(pTarget); } @@ -843,7 +836,7 @@ int hbMgrInit() { clientHbMgr.appId = tGenIdPI64(); tscDebug("app %" PRIx64 " initialized", clientHbMgr.appId); - + clientHbMgr.appSummary = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *)); taosThreadMutexInit(&clientHbMgr.lock, NULL); @@ -881,7 +874,7 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, int64_t clust SClientHbReq hbReq = {0}; hbReq.connKey = connKey; hbReq.clusterId = clusterId; - //hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + // hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq)); @@ -896,12 +889,9 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in }; switch (connType) { - case CONN_TYPE__QUERY: { + case CONN_TYPE__QUERY: + case CONN_TYPE__TMQ: return hbRegisterConnImpl(pAppHbMgr, connKey, clusterId); - } - case CONN_TYPE__TMQ: { - return 0; - } default: return 0; } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 6ad6413159..ed0ec516b2 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -50,13 +50,16 @@ struct tmq_list_t { }; struct tmq_conf_t { - char clientId[256]; - char groupId[TSDB_CGROUP_LEN]; - int8_t autoCommit; - int8_t resetOffset; - int8_t withTbName; - int8_t spEnable; - int32_t spBatchSize; + char clientId[256]; + char groupId[TSDB_CGROUP_LEN]; + int8_t autoCommit; + int8_t resetOffset; + int8_t withTbName; + int8_t ssEnable; + int32_t ssBatchSize; + + bool hbBgEnable; + uint16_t port; int32_t autoCommitInterval; char* ip; @@ -68,14 +71,17 @@ struct tmq_conf_t { struct tmq_t { // conf - char groupId[TSDB_CGROUP_LEN]; - char clientId[256]; - int8_t withTbName; - int8_t useSnapshot; - int8_t autoCommit; - int32_t autoCommitInterval; - int32_t resetOffsetCfg; - int64_t consumerId; + char groupId[TSDB_CGROUP_LEN]; + char clientId[256]; + int8_t withTbName; + int8_t useSnapshot; + int8_t autoCommit; + int32_t autoCommitInterval; + int32_t resetOffsetCfg; + int64_t consumerId; + + bool hbBgEnable; + tmq_commit_cb* commitCb; void* commitCbUserParam; @@ -89,7 +95,8 @@ struct tmq_t { int64_t pollCnt; // timer - tmr_h hbTimer; + tmr_h hbLiveTimer; + tmr_h epTimer; tmr_h reportTimer; tmr_h commitTimer; @@ -124,7 +131,7 @@ enum { }; enum { - TMQ_DELAYED_TASK__HB = 1, + TMQ_DELAYED_TASK__ASK_EP = 1, TMQ_DELAYED_TASK__REPORT, TMQ_DELAYED_TASK__COMMIT, }; @@ -297,18 +304,31 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value if (strcmp(key, "experimental.snapshot.enable") == 0) { if (strcmp(value, "true") == 0) { - conf->spEnable = true; + conf->ssEnable = true; return TMQ_CONF_OK; } else if (strcmp(value, "false") == 0) { - conf->spEnable = false; + conf->ssEnable = false; return TMQ_CONF_OK; } else { return TMQ_CONF_INVALID; } } + if (strcmp(key, "enable.heartbeat.background") == 0) { + if (strcmp(value, "true") == 0) { + conf->hbBgEnable = true; + return TMQ_CONF_OK; + } else if (strcmp(value, "false") == 0) { + conf->hbBgEnable = false; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + return TMQ_CONF_OK; + } + if (strcmp(key, "experimental.snapshot.batch.size") == 0) { - conf->spBatchSize = atoi(value); + conf->ssBatchSize = atoi(value); return TMQ_CONF_OK; } @@ -757,10 +777,10 @@ END: } #endif -void tmqAssignDelayedHbTask(void* param, void* tmrId) { +void tmqAssignAskEpTask(void* param, void* tmrId) { tmq_t* tmq = (tmq_t*)param; int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); - *pTaskType = TMQ_DELAYED_TASK__HB; + *pTaskType = TMQ_DELAYED_TASK__ASK_EP; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); } @@ -781,6 +801,46 @@ void tmqAssignDelayedReportTask(void* param, void* tmrId) { tsem_post(&tmq->rspSem); } +int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { + if (pMsg && pMsg->pData) taosMemoryFree(pMsg->pData); + return 0; +} + +void tmqSendHbReq(void* param, void* tmrId) { + // TODO replace with ref + tmq_t* tmq = (tmq_t*)param; + int64_t consumerId = tmq->consumerId; + int32_t epoch = tmq->epoch; + SMqHbReq* pReq = taosMemoryMalloc(sizeof(SMqHbReq)); + if (pReq == NULL) goto OVER; + pReq->consumerId = consumerId; + pReq->epoch = epoch; + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(pReq); + } + sendInfo->msgInfo = (SDataBuf){ + .pData = pReq, + .len = sizeof(SMqHbReq), + .handle = NULL, + }; + + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = NULL; + sendInfo->fp = tmqHbCb; + sendInfo->msgType = TDMT_MND_MQ_HB; + + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + +OVER: + taosTmrReset(tmqSendHbReq, 1000, tmq, tmqMgmt.timer, &tmq->hbLiveTimer); +} + int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { STaosQall* qall = taosAllocateQall(); taosReadAllQitems(tmq->delayedTask, qall); @@ -789,9 +849,9 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { taosGetQitem(qall, (void**)&pTaskType); if (pTaskType == NULL) break; - if (*pTaskType == TMQ_DELAYED_TASK__HB) { + if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) { tmqAskEp(tmq, true); - taosTmrReset(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer, &tmq->hbTimer); + taosTmrReset(tmqAssignAskEpTask, 1000, tmq, tmqMgmt.timer, &tmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { tmqCommitInner2(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer); @@ -935,13 +995,15 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); pTmq->withTbName = conf->withTbName; - pTmq->useSnapshot = conf->spEnable; + pTmq->useSnapshot = conf->ssEnable; pTmq->autoCommit = conf->autoCommit; pTmq->autoCommitInterval = conf->autoCommitInterval; pTmq->commitCb = conf->commitCb; pTmq->commitCbUserParam = conf->commitCbUserParam; pTmq->resetOffsetCfg = conf->resetOffset; + pTmq->hbBgEnable = conf->hbBgEnable; + // assign consumerId pTmq->consumerId = tGenIdPI64(); @@ -959,6 +1021,10 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { goto FAIL; } + if (pTmq->hbBgEnable) { + pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pTmq, tmqMgmt.timer); + } + tscInfo("consumer %ld is setup, consumer group %s", pTmq->consumerId, pTmq->groupId); return pTmq; @@ -1055,9 +1121,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { taosMsleep(500); } - // init hb timer - if (tmq->hbTimer == NULL) { - tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer); + // init ep timer + if (tmq->epTimer == NULL) { + tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, 1000, tmq, tmqMgmt.timer); } // init auto commit timer @@ -1858,9 +1924,9 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -tmq_raw_data *tmq_get_raw_meta(TAOS_RES* res) { +tmq_raw_data* tmq_get_raw_meta(TAOS_RES* res) { if (TD_RES_TMQ_META(res)) { - tmq_raw_data *raw = taosMemoryCalloc(1, sizeof(tmq_raw_data)); + tmq_raw_data* raw = taosMemoryCalloc(1, sizeof(tmq_raw_data)); SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; raw->raw_meta = pMetaRspObj->metaRsp.metaRsp; raw->raw_meta_len = pMetaRspObj->metaRsp.metaRspLen; @@ -1870,7 +1936,8 @@ tmq_raw_data *tmq_get_raw_meta(TAOS_RES* res) { return NULL; } -static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t){ +static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, + int8_t t) { char* string = NULL; cJSON* json = cJSON_CreateObject(); if (json == NULL) { @@ -1879,32 +1946,32 @@ static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* sch cJSON* type = cJSON_CreateString("create"); cJSON_AddItemToObject(json, "type", type); -// char uid[32] = {0}; -// sprintf(uid, "%"PRIi64, id); -// cJSON* id_ = cJSON_CreateString(uid); -// cJSON_AddItemToObject(json, "id", id_); + // char uid[32] = {0}; + // sprintf(uid, "%"PRIi64, id); + // cJSON* id_ = cJSON_CreateString(uid); + // cJSON_AddItemToObject(json, "id", id_); cJSON* tableName = cJSON_CreateString(name); cJSON_AddItemToObject(json, "tableName", tableName); cJSON* tableType = cJSON_CreateString(t == TSDB_NORMAL_TABLE ? "normal" : "super"); cJSON_AddItemToObject(json, "tableType", tableType); -// cJSON* version = cJSON_CreateNumber(1); -// cJSON_AddItemToObject(json, "version", version); + // cJSON* version = cJSON_CreateNumber(1); + // cJSON_AddItemToObject(json, "version", version); cJSON* columns = cJSON_CreateArray(); - for(int i = 0; i < schemaRow->nCols; i++){ - cJSON* column = cJSON_CreateObject(); - SSchema *s = schemaRow->pSchema + i; - cJSON* cname = cJSON_CreateString(s->name); + for (int i = 0; i < schemaRow->nCols; i++) { + cJSON* column = cJSON_CreateObject(); + SSchema* s = schemaRow->pSchema + i; + cJSON* cname = cJSON_CreateString(s->name); cJSON_AddItemToObject(column, "name", cname); cJSON* ctype = cJSON_CreateNumber(s->type); cJSON_AddItemToObject(column, "type", ctype); - if(s->type == TSDB_DATA_TYPE_BINARY){ + if (s->type == TSDB_DATA_TYPE_BINARY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(column, "length", cbytes); - }else if (s->type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (s->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (s->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(column, "length", cbytes); } cJSON_AddItemToArray(columns, column); @@ -1912,20 +1979,20 @@ static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* sch cJSON_AddItemToObject(json, "columns", columns); cJSON* tags = cJSON_CreateArray(); - for(int i = 0; schemaTag && i < schemaTag->nCols; i++){ - cJSON* tag = cJSON_CreateObject(); - SSchema *s = schemaTag->pSchema + i; - cJSON* tname = cJSON_CreateString(s->name); + for (int i = 0; schemaTag && i < schemaTag->nCols; i++) { + cJSON* tag = cJSON_CreateObject(); + SSchema* s = schemaTag->pSchema + i; + cJSON* tname = cJSON_CreateString(s->name); cJSON_AddItemToObject(tag, "name", tname); cJSON* ttype = cJSON_CreateNumber(s->type); cJSON_AddItemToObject(tag, "type", ttype); - if(s->type == TSDB_DATA_TYPE_BINARY){ + if (s->type == TSDB_DATA_TYPE_BINARY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(tag, "length", cbytes); - }else if (s->type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (s->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (s->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(tag, "length", cbytes); } cJSON_AddItemToArray(tags, tag); @@ -1937,10 +2004,10 @@ static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* sch return string; } -static char *buildAlterSTableJson(void* alterData, int32_t alterDataLen){ - SMAlterStbReq req = {0}; - cJSON* json = NULL; - char* string = NULL; +static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { + SMAlterStbReq req = {0}; + cJSON* json = NULL; + char* string = NULL; if (tDeserializeSMAlterStbReq(alterData, alterDataLen, &req) != 0) { goto end; @@ -1952,8 +2019,8 @@ static char *buildAlterSTableJson(void* alterData, int32_t alterDataLen){ } cJSON* type = cJSON_CreateString("alter"); cJSON_AddItemToObject(json, "type", type); -// cJSON* uid = cJSON_CreateNumber(id); -// cJSON_AddItemToObject(json, "uid", uid); + // cJSON* uid = cJSON_CreateNumber(id); + // cJSON_AddItemToObject(json, "uid", uid); SName name = {0}; tNameFromString(&name, req.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); cJSON* tableName = cJSON_CreateString(name.tname); @@ -1966,53 +2033,53 @@ static char *buildAlterSTableJson(void* alterData, int32_t alterDataLen){ switch (req.alterType) { case TSDB_ALTER_TABLE_ADD_TAG: case TSDB_ALTER_TABLE_ADD_COLUMN: { - TAOS_FIELD *field = taosArrayGet(req.pFields, 0); - cJSON* colName = cJSON_CreateString(field->name); + TAOS_FIELD* field = taosArrayGet(req.pFields, 0); + cJSON* colName = cJSON_CreateString(field->name); cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(field->type); cJSON_AddItemToObject(json, "colType", colType); - if(field->type == TSDB_DATA_TYPE_BINARY){ + if (field->type == TSDB_DATA_TYPE_BINARY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); - }else if (field->type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (field->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (field->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); } break; } case TSDB_ALTER_TABLE_DROP_TAG: - case TSDB_ALTER_TABLE_DROP_COLUMN:{ - TAOS_FIELD *field = taosArrayGet(req.pFields, 0); - cJSON* colName = cJSON_CreateString(field->name); + case TSDB_ALTER_TABLE_DROP_COLUMN: { + TAOS_FIELD* field = taosArrayGet(req.pFields, 0); + cJSON* colName = cJSON_CreateString(field->name); cJSON_AddItemToObject(json, "colName", colName); break; } case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:{ - TAOS_FIELD *field = taosArrayGet(req.pFields, 0); - cJSON* colName = cJSON_CreateString(field->name); + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { + TAOS_FIELD* field = taosArrayGet(req.pFields, 0); + cJSON* colName = cJSON_CreateString(field->name); cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(field->type); cJSON_AddItemToObject(json, "colType", colType); - if(field->type == TSDB_DATA_TYPE_BINARY){ + if (field->type == TSDB_DATA_TYPE_BINARY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); - }else if (field->type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (field->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (field->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); } break; } case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:{ - TAOS_FIELD *oldField = taosArrayGet(req.pFields, 0); - TAOS_FIELD *newField = taosArrayGet(req.pFields, 1); - cJSON* colName = cJSON_CreateString(oldField->name); + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { + TAOS_FIELD* oldField = taosArrayGet(req.pFields, 0); + TAOS_FIELD* newField = taosArrayGet(req.pFields, 1); + cJSON* colName = cJSON_CreateString(oldField->name); cJSON_AddItemToObject(json, "colName", colName); cJSON* colNewName = cJSON_CreateString(newField->name); cJSON_AddItemToObject(json, "colNewName", colNewName); @@ -2023,19 +2090,19 @@ static char *buildAlterSTableJson(void* alterData, int32_t alterDataLen){ } string = cJSON_PrintUnformatted(json); - end: +end: cJSON_Delete(json); tFreeSMAltertbReq(&req); return string; } -static char *processCreateStb(SMqMetaRsp *metaRsp){ +static char* processCreateStb(SMqMetaRsp* metaRsp) { SVCreateStbReq req = {0}; SDecoder coder; - char* string = NULL; + char* string = NULL; // decode and process req - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); @@ -2046,18 +2113,18 @@ static char *processCreateStb(SMqMetaRsp *metaRsp){ tDecoderClear(&coder); return string; - _err: +_err: tDecoderClear(&coder); return string; } -static char *processAlterStb(SMqMetaRsp *metaRsp){ +static char* processAlterStb(SMqMetaRsp* metaRsp) { SVCreateStbReq req = {0}; SDecoder coder; - char* string = NULL; + char* string = NULL; // decode and process req - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); @@ -2068,24 +2135,24 @@ static char *processAlterStb(SMqMetaRsp *metaRsp){ tDecoderClear(&coder); return string; - _err: +_err: tDecoderClear(&coder); return string; } -static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* tagName, int64_t id){ - char* string = NULL; +static char* buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* tagName, int64_t id) { + char* string = NULL; SArray* pTagVals = NULL; - cJSON* json = cJSON_CreateObject(); + cJSON* json = cJSON_CreateObject(); if (json == NULL) { return string; } cJSON* type = cJSON_CreateString("create"); cJSON_AddItemToObject(json, "type", type); -// char cid[32] = {0}; -// sprintf(cid, "%"PRIi64, id); -// cJSON* cid_ = cJSON_CreateString(cid); -// cJSON_AddItemToObject(json, "id", cid_); + // char cid[32] = {0}; + // sprintf(cid, "%"PRIi64, id); + // cJSON* cid_ = cJSON_CreateString(cid); + // cJSON_AddItemToObject(json, "id", cid_); cJSON* tableName = cJSON_CreateString(name); cJSON_AddItemToObject(json, "tableName", tableName); @@ -2093,10 +2160,10 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* cJSON_AddItemToObject(json, "tableType", tableType); cJSON* using = cJSON_CreateString(sname); cJSON_AddItemToObject(json, "using", using); -// cJSON* version = cJSON_CreateNumber(1); -// cJSON_AddItemToObject(json, "version", version); + // cJSON* version = cJSON_CreateNumber(1); + // cJSON_AddItemToObject(json, "version", version); - cJSON* tags = cJSON_CreateArray(); + cJSON* tags = cJSON_CreateArray(); int32_t code = tTagToValArray(pTag, &pTagVals); if (code) { goto end; @@ -2104,18 +2171,18 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* if (tTagIsJson(pTag)) { STag* p = (STag*)pTag; - if(p->nTag == 0){ + if (p->nTag == 0) { goto end; } - char* pJson = parseTagDatatoJson(pTag); - cJSON* tag = cJSON_CreateObject(); + char* pJson = parseTagDatatoJson(pTag); + cJSON* tag = cJSON_CreateObject(); STagVal* pTagVal = taosArrayGet(pTagVals, 0); char* ptname = taosArrayGet(tagName, 0); cJSON* tname = cJSON_CreateString(ptname); cJSON_AddItemToObject(tag, "name", tname); -// cJSON* cid_ = cJSON_CreateString(""); -// cJSON_AddItemToObject(tag, "cid", cid_); + // cJSON* cid_ = cJSON_CreateString(""); + // cJSON_AddItemToObject(tag, "cid", cid_); cJSON* ttype = cJSON_CreateNumber(TSDB_DATA_TYPE_JSON); cJSON_AddItemToObject(tag, "type", ttype); cJSON* tvalue = cJSON_CreateString(pJson); @@ -2125,7 +2192,7 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* goto end; } - for(int i = 0; i < taosArrayGetSize(pTagVals); i++){ + for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i); cJSON* tag = cJSON_CreateObject(); @@ -2133,8 +2200,8 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* char* ptname = taosArrayGet(tagName, i); cJSON* tname = cJSON_CreateString(ptname); cJSON_AddItemToObject(tag, "name", tname); -// cJSON* cid = cJSON_CreateNumber(pTagVal->cid); -// cJSON_AddItemToObject(tag, "cid", cid); + // cJSON* cid = cJSON_CreateNumber(pTagVal->cid); + // cJSON_AddItemToObject(tag, "cid", cid); cJSON* ttype = cJSON_CreateNumber(pTagVal->type); cJSON_AddItemToObject(tag, "type", ttype); @@ -2153,7 +2220,7 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* cJSON_AddItemToArray(tags, tag); } - end: +end: cJSON_AddItemToObject(json, "tags", tags); string = cJSON_PrintUnformatted(json); cJSON_Delete(json); @@ -2161,13 +2228,13 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* return string; } -static char *processCreateTable(SMqMetaRsp *metaRsp){ +static char* processCreateTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVCreateTbBatchReq req = {0}; - SVCreateTbReq *pCreateReq; - char *string = NULL; + SVCreateTbReq* pCreateReq; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVCreateTbBatchReq(&decoder, &req) < 0) { @@ -2177,27 +2244,29 @@ static char *processCreateTable(SMqMetaRsp *metaRsp){ // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; - if(pCreateReq->type == TSDB_CHILD_TABLE){ - string = buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name, pCreateReq->ctb.tagName, pCreateReq->uid); - }else if(pCreateReq->type == TSDB_NORMAL_TABLE){ - string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE); + if (pCreateReq->type == TSDB_CHILD_TABLE) { + string = buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name, + pCreateReq->ctb.tagName, pCreateReq->uid); + } else if (pCreateReq->type == TSDB_NORMAL_TABLE) { + string = + buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE); } } tDecoderClear(&decoder); - _exit: +_exit: tDecoderClear(&decoder); return string; } -static char *processAlterTable(SMqMetaRsp *metaRsp){ - SDecoder decoder = {0}; - SVAlterTbReq vAlterTbReq = {0}; - char *string = NULL; +static char* processAlterTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVAlterTbReq vAlterTbReq = {0}; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVAlterTbReq(&decoder, &vAlterTbReq) < 0) { @@ -2210,8 +2279,8 @@ static char *processAlterTable(SMqMetaRsp *metaRsp){ } cJSON* type = cJSON_CreateString("alter"); cJSON_AddItemToObject(json, "type", type); -// cJSON* uid = cJSON_CreateNumber(id); -// cJSON_AddItemToObject(json, "uid", uid); + // cJSON* uid = cJSON_CreateNumber(id); + // cJSON_AddItemToObject(json, "uid", uid); cJSON* tableName = cJSON_CreateString(vAlterTbReq.tbName); cJSON_AddItemToObject(json, "tableName", tableName); cJSON* tableType = cJSON_CreateString(vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL ? "child" : "normal"); @@ -2226,55 +2295,55 @@ static char *processAlterTable(SMqMetaRsp *metaRsp){ cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); cJSON_AddItemToObject(json, "colType", colType); - if(vAlterTbReq.type == TSDB_DATA_TYPE_BINARY){ + if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY) { int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); - }else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); } break; } - case TSDB_ALTER_TABLE_DROP_COLUMN:{ + case TSDB_ALTER_TABLE_DROP_COLUMN: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); cJSON_AddItemToObject(json, "colName", colName); break; } - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:{ + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.colModType); cJSON_AddItemToObject(json, "colType", colType); - if(vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY){ + if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY) { int32_t length = vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); - }else if (vAlterTbReq.colModType == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (vAlterTbReq.colModType == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); } break; } - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:{ + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); cJSON_AddItemToObject(json, "colName", colName); cJSON* colNewName = cJSON_CreateString(vAlterTbReq.colNewName); cJSON_AddItemToObject(json, "colNewName", colNewName); break; } - case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:{ + case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: { cJSON* tagName = cJSON_CreateString(vAlterTbReq.tagName); cJSON_AddItemToObject(json, "colName", tagName); bool isNull = vAlterTbReq.isNull; - if(vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON){ - STag *jsonTag = (STag *)vAlterTbReq.pTagVal; - if(jsonTag->nTag == 0) isNull = true; + if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { + STag* jsonTag = (STag*)vAlterTbReq.pTagVal; + if (jsonTag->nTag == 0) isNull = true; } - if (!isNull){ + if (!isNull) { char* buf = NULL; if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { @@ -2299,18 +2368,18 @@ static char *processAlterTable(SMqMetaRsp *metaRsp){ } string = cJSON_PrintUnformatted(json); - _exit: +_exit: tDecoderClear(&decoder); return string; } -static char *processDropSTable(SMqMetaRsp *metaRsp){ - SDecoder decoder = {0}; - SVDropStbReq req = {0}; - char *string = NULL; +static char* processDropSTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVDropStbReq req = {0}; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVDropStbReq(&decoder, &req) < 0) { @@ -2330,18 +2399,18 @@ static char *processDropSTable(SMqMetaRsp *metaRsp){ string = cJSON_PrintUnformatted(json); - _exit: +_exit: tDecoderClear(&decoder); return string; } -static char *processDropTable(SMqMetaRsp *metaRsp){ - SDecoder decoder = {0}; - SVDropTbBatchReq req = {0}; - char *string = NULL; +static char* processDropTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVDropTbBatchReq req = {0}; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVDropTbBatchReq(&decoder, &req) < 0) { @@ -2354,10 +2423,10 @@ static char *processDropTable(SMqMetaRsp *metaRsp){ } cJSON* type = cJSON_CreateString("drop"); cJSON_AddItemToObject(json, "type", type); -// cJSON* uid = cJSON_CreateNumber(id); -// cJSON_AddItemToObject(json, "uid", uid); -// cJSON* tableType = cJSON_CreateString("normal"); -// cJSON_AddItemToObject(json, "tableType", tableType); + // cJSON* uid = cJSON_CreateNumber(id); + // cJSON_AddItemToObject(json, "uid", uid); + // cJSON* tableType = cJSON_CreateString("normal"); + // cJSON_AddItemToObject(json, "tableType", tableType); cJSON* tableNameList = cJSON_CreateArray(); for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { @@ -2370,55 +2439,53 @@ static char *processDropTable(SMqMetaRsp *metaRsp){ string = cJSON_PrintUnformatted(json); - _exit: +_exit: tDecoderClear(&decoder); return string; } -char *tmq_get_json_meta(TAOS_RES *res){ +char* tmq_get_json_meta(TAOS_RES* res) { if (!TD_RES_TMQ_META(res)) { return NULL; } SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB){ + if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) { return processCreateStb(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB) { return processAlterStb(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB) { return processDropSTable(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE) { return processCreateTable(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE) { return processAlterTable(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) { return processDropTable(&pMetaRspObj->metaRsp); } return NULL; } -void tmq_free_json_meta(char* jsonMeta){ - taosMemoryFreeClear(jsonMeta); -} +void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } -static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ +static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { SVCreateStbReq req = {0}; SDecoder coder; SMCreateStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj* pRequest = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); if (code != TSDB_CODE_SUCCESS) { goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVCreateStbReq(&coder, &req) < 0) { @@ -2427,16 +2494,16 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ } // build create stable pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SField)); - for(int32_t i = 0; i < req.schemaRow.nCols; i++){ + for (int32_t i = 0; i < req.schemaRow.nCols; i++) { SSchema* pSchema = req.schemaRow.pSchema + i; - SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; + SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; strcpy(field.name, pSchema->name); taosArrayPush(pReq.pColumns, &field); } pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField)); - for(int32_t i = 0; i < req.schemaTag.nCols; i++){ + for (int32_t i = 0; i < req.schemaTag.nCols; i++) { SSchema* pSchema = req.schemaTag.pSchema + i; - SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; + SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; strcpy(field.name, pSchema->name); taosArrayPush(pReq.pTags, &field); } @@ -2451,7 +2518,7 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ pReq.igExists = true; STscObj* pTscObj = pRequest->pTscObj; - SName tableName; + SName tableName; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); SCmdMsgInfo pCmdMsg = {0}; @@ -2465,7 +2532,7 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ } tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery = {0}; + SQuery pQuery = {0}; pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; @@ -2475,18 +2542,18 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); - end: +end: destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); tDecoderClear(&coder); return code; } -static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ +static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { SVDropStbReq req = {0}; SDecoder coder; SMDropStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); @@ -2494,12 +2561,12 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVDropStbReq(&coder, &req) < 0) { @@ -2513,7 +2580,7 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ pReq.suid = req.suid; STscObj* pTscObj = pRequest->pTscObj; - SName tableName; + SName tableName; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); SCmdMsgInfo pCmdMsg = {0}; @@ -2527,7 +2594,7 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ } tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery = {0}; + SQuery pQuery = {0}; pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; @@ -2537,7 +2604,7 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); - end: +end: destroyRequest(pRequest); tDecoderClear(&coder); return code; @@ -2550,7 +2617,7 @@ typedef struct SVgroupCreateTableBatch { } SVgroupCreateTableBatch; static void destroyCreateTbReqBatch(void* data) { - SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*) data; + SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*)data; taosArrayDestroy(pTbBatch->req.pArray); } @@ -2567,13 +2634,12 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } - - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) { @@ -2598,15 +2664,15 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; SVgroupInfo pInfo = {0}; - SName pName; + SName pName; toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2638,7 +2704,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_CREATE_TABLE; pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT); + pQuery->pRoot = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT); code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); if (code != TSDB_CODE_SUCCESS) { @@ -2646,10 +2712,10 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { } launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end - code = pRequest->code; + pQuery = NULL; // no need to free in the end + code = pRequest->code; - end: +end: taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); tDecoderClear(&coder); @@ -2681,12 +2747,12 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVDropTbBatchReq(&coder, &req) < 0) { @@ -2711,16 +2777,16 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pDropReq = req.pReqs + iReq; pDropReq->igNotExists = true; SVgroupInfo pInfo = {0}; - SName pName; + SName pName; toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2750,7 +2816,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_DROP_TABLE; pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_DROP_TABLE_STMT); + pQuery->pRoot = nodesMakeNode(QUERY_NODE_DROP_TABLE_STMT); code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); if (code != TSDB_CODE_SUCCESS) { @@ -2758,10 +2824,10 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { } launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end - code = pRequest->code; + pQuery = NULL; // no need to free in the end + code = pRequest->code; - end: +end: taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); tDecoderClear(&coder); @@ -2769,27 +2835,27 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { return code; } -static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ - SVAlterTbReq req = {0}; - SDecoder coder = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj *pRequest = NULL; - SQuery *pQuery = NULL; - SArray *pArray = NULL; - SVgDataBlocks *pVgData = NULL; +static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { + SVAlterTbReq req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + SQuery* pQuery = NULL; + SArray* pArray = NULL; + SVgDataBlocks* pVgData = NULL; - code = buildRequest(*(int64_t*) taos, "", 0, NULL, false, &pRequest); + code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); if (code != TSDB_CODE_SUCCESS) { goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVAlterTbReq(&coder, &req) < 0) { @@ -2798,24 +2864,24 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ } // do not deal TSDB_ALTER_TABLE_UPDATE_OPTIONS - if(req.action == TSDB_ALTER_TABLE_UPDATE_OPTIONS){ + if (req.action == TSDB_ALTER_TABLE_UPDATE_OPTIONS) { goto end; } STscObj* pTscObj = pRequest->pTscObj; - SCatalog *pCatalog = NULL; + SCatalog* pCatalog = NULL; code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { goto end; } SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; SVgroupInfo pInfo = {0}; - SName pName = {0}; + SName pName = {0}; toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2849,7 +2915,7 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_ALTER_TABLE; pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + pQuery->pRoot = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); code = rewriteToVnodeModifyOpStmt(pQuery, pArray); if (code != TSDB_CODE_SUCCESS) { @@ -2857,17 +2923,17 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ } launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end + pQuery = NULL; // no need to free in the end pVgData = NULL; - pArray = NULL; - code = pRequest->code; - if (code == TSDB_CODE_VND_TABLE_NOT_EXIST){ + pArray = NULL; + code = pRequest->code; + if (code == TSDB_CODE_VND_TABLE_NOT_EXIST) { code = 0; } - end: +end: taosArrayDestroy(pArray); - if(pVgData) taosMemoryFreeClear(pVgData->pData); + if (pVgData) taosMemoryFreeClear(pVgData->pData); taosMemoryFreeClear(pVgData); destroyRequest(pRequest); tDecoderClear(&coder); @@ -2875,33 +2941,38 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ return code; } -int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta){ +int32_t taos_write_raw_meta(TAOS* taos, tmq_raw_data* raw_meta) { if (!taos || !raw_meta) { return TSDB_CODE_INVALID_PARA; } - if(raw_meta->raw_meta_type == TDMT_VND_CREATE_STB) { + if (raw_meta->raw_meta_type == TDMT_VND_CREATE_STB) { return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_STB){ + } else if (raw_meta->raw_meta_type == TDMT_VND_ALTER_STB) { return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_STB){ + } else if (raw_meta->raw_meta_type == TDMT_VND_DROP_STB) { return taosDropStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_CREATE_TABLE){ + } else if (raw_meta->raw_meta_type == TDMT_VND_CREATE_TABLE) { return taosCreateTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_TABLE){ + } else if (raw_meta->raw_meta_type == TDMT_VND_ALTER_TABLE) { return taosAlterTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_TABLE){ + } else if (raw_meta->raw_meta_type == TDMT_VND_DROP_TABLE) { return taosDropTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); } return TSDB_CODE_INVALID_PARA; } -void tmq_free_raw_meta(tmq_raw_data *rawMeta){ +void tmq_free_raw_meta(tmq_raw_data* rawMeta) { + // taosMemoryFreeClear(rawMeta); } void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { + // tmqCommitInner2(tmq, msg, 0, 1, cb, param); } -int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) { return tmqCommitInner2(tmq, msg, 0, 0, NULL, NULL); } \ No newline at end of file +int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) { + // + return tmqCommitInner2(tmq, msg, 0, 0, NULL, NULL); +} diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index b9a02728fc..81b1b7f081 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -196,6 +196,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_HB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 27b785f4d2..d84822d7e3 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -48,6 +48,7 @@ static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg); static int32_t mndProcessAskEpReq(SRpcMsg *pMsg); +static int32_t mndProcessMqHbReq(SRpcMsg *pMsg); static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg); static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg); static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg); @@ -62,6 +63,7 @@ int32_t mndInitConsumer(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_HB, mndProcessMqHbReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_ASK_EP, mndProcessAskEpReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg); @@ -255,24 +257,15 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { return 0; } -static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - SMqAskEpReq *pReq = (SMqAskEpReq *)pMsg->pCont; - SMqAskEpRsp rsp = {0}; - int64_t consumerId = be64toh(pReq->consumerId); - int32_t epoch = ntohl(pReq->epoch); +static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { + SMnode *pMnode = pMsg->info.node; + SMqHbReq *pReq = (SMqHbReq *)pMsg->pCont; + int64_t consumerId = be64toh(pReq->consumerId); - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->info.node, consumerId); - if (pConsumer == NULL) { - terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; - return -1; - } + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); - ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); - /*int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus);*/ atomic_store_32(&pConsumer->hbStatus, 0); - // 1. check consumer status int32_t status = atomic_load_32(&pConsumer->status); if (status == MQ_CONSUMER_STATUS__LOST_REBD) { @@ -286,6 +279,46 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg); } + mndReleaseConsumer(pMnode, pConsumer); + + return 0; +} + +static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { + SMnode *pMnode = pMsg->info.node; + SMqAskEpReq *pReq = (SMqAskEpReq *)pMsg->pCont; + SMqAskEpRsp rsp = {0}; + int64_t consumerId = be64toh(pReq->consumerId); + int32_t epoch = ntohl(pReq->epoch); + + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); + if (pConsumer == NULL) { + terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; + return -1; + } + + ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); + +#if 1 + atomic_store_32(&pConsumer->hbStatus, 0); +#endif + + // 1. check consumer status + int32_t status = atomic_load_32(&pConsumer->status); + +#if 0 + if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); + + pRecoverMsg->consumerId = consumerId; + SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg)); + pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_RECOVER; + pRpcMsg->pCont = pRecoverMsg; + pRpcMsg->contLen = sizeof(SMqConsumerRecoverMsg); + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg); + } +#endif + if (status != MQ_CONSUMER_STATUS__READY) { terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 38368b4ece..b76b371583 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -15,10 +15,10 @@ #define _DEFAULT_SOURCE #include "mndProfile.h" -#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndPrivilege.h" #include "mndQnode.h" #include "mndShow.h" #include "mndStb.h" @@ -387,67 +387,7 @@ static void mndCancelGetNextApp(SMnode *pMnode, void *pIter) { } static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { -#if 0 - SClientHbRsp* pRsp = taosMemoryMalloc(sizeof(SClientHbRsp)); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - pRsp->connKey = pReq->connKey; - SMqHbBatchRsp batchRsp; - batchRsp.batchRsps = taosArrayInit(0, sizeof(SMqHbRsp)); - if (batchRsp.batchRsps == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - SClientHbKey connKey = pReq->connKey; - SHashObj* pObj = pReq->info; - SKv* pKv = taosHashGet(pObj, "mq-tmp", strlen("mq-tmp") + 1); - if (pKv == NULL) { - taosMemoryFree(pRsp); - return NULL; - } - SMqHbMsg mqHb; - taosDecodeSMqMsg(pKv->value, &mqHb); - /*int64_t clientUid = htonl(pKv->value);*/ - /*if (mqHb.epoch )*/ - int sz = taosArrayGetSize(mqHb.pTopics); - SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, mqHb.consumerId); - for (int i = 0; i < sz; i++) { - SMqHbOneTopicBatchRsp innerBatchRsp; - innerBatchRsp.rsps = taosArrayInit(sz, sizeof(SMqHbRsp)); - if (innerBatchRsp.rsps == NULL) { - //TODO - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - SMqHbTopicInfo* topicInfo = taosArrayGet(mqHb.pTopics, i); - SMqConsumerTopic* pConsumerTopic = taosHashGet(pConsumer->topicHash, topicInfo->name, strlen(topicInfo->name)+1); - if (pConsumerTopic->epoch != topicInfo->epoch) { - //add new vgids into rsp - int vgSz = taosArrayGetSize(topicInfo->pVgInfo); - for (int j = 0; j < vgSz; j++) { - SMqHbRsp innerRsp; - SMqHbVgInfo* pVgInfo = taosArrayGet(topicInfo->pVgInfo, i); - SVgObj* pVgObj = mndAcquireVgroup(pMnode, pVgInfo->vgId); - innerRsp.epSet = mndGetVgroupEpset(pMnode, pVgObj); - taosArrayPush(innerBatchRsp.rsps, &innerRsp); - } - } - taosArrayPush(batchRsp.batchRsps, &innerBatchRsp); - } - int32_t tlen = taosEncodeSMqHbBatchRsp(NULL, &batchRsp); - void* buf = taosMemoryMalloc(tlen); - if (buf == NULL) { - //TODO - return NULL; - } - void* abuf = buf; - taosEncodeSMqHbBatchRsp(&abuf, &batchRsp); - pRsp->body = buf; - pRsp->bodyLen = tlen; - return pRsp; -#endif + // return NULL; } From 16750e1401065edf70acd392af4f8aca5c64cd29 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 16 Jul 2022 17:12:45 +0800 Subject: [PATCH 109/124] enh: rsma async commit support --- source/dnode/mnode/impl/src/mndSma.c | 2 +- source/dnode/vnode/src/inc/sma.h | 61 +++++------ source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/sma/smaCommit.c | 125 ++++++++++++++++------- source/dnode/vnode/src/sma/smaEnv.c | 12 +-- source/dnode/vnode/src/sma/smaRollup.c | 103 ++++++++++++++----- source/dnode/vnode/src/sma/smaUtil.c | 95 +++++++++++++++++ source/dnode/vnode/src/vnd/vnodeCommit.c | 8 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- 9 files changed, 299 insertions(+), 110 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 88629ebc69..bff3f19e99 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -709,7 +709,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("sma:%s, failed to create since %s", createReq.name, terrstr(terrno)); + mError("sma:%s, failed to create since %s", createReq.name, terrstr()); } mndReleaseStb(pMnode, pStb); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index cfeb4ebeb2..1c8434594d 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -32,6 +32,8 @@ extern "C" { #define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on +#define RSMA_TASK_INFO_HASH_SLOT 8 + typedef struct SSmaEnv SSmaEnv; typedef struct SSmaStat SSmaStat; typedef struct STSmaStat STSmaStat; @@ -41,7 +43,7 @@ typedef struct SRSmaInfo SRSmaInfo; typedef struct SRSmaInfoItem SRSmaInfoItem; struct SSmaEnv { - TdThreadRwlock lock; + SRWLatch lock; int8_t type; SSmaStat *pStat; }; @@ -52,7 +54,7 @@ typedef struct { void *tmrHandle; // shared by all fetch tasks } SSmaMgmt; -#define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_LOCK(env) (&(env)->lock) #define SMA_ENV_TYPE(env) ((env)->type) #define SMA_ENV_STAT(env) ((env)->pStat) @@ -64,12 +66,14 @@ struct STSmaStat { struct SRSmaStat { SSma *pSma; - int64_t submitVer; - int64_t refId; // shared by fetch tasks - int8_t triggerStat; // shared by fetch tasks - int8_t commitStat; // 0 not in committing, 1 in committing - SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; - SHashObj *iRsmaInfoHash; // key: stbUid, value: SRSmaInfo; immutable rsmaInfoHash + int64_t commitAppliedVer; // vnode applied version for async commit + int64_t commitSubmitVer; // rsma submit version for async commit + int64_t submitVer; // latest submit version + int64_t refId; // shared by fetch tasks + int8_t triggerStat; // shared by fetch tasks + int8_t commitStat; // 0 not in committing, 1 in committing + SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; + SHashObj *iRsmaInfoHash; // key: stbUid, value: SRSmaInfo; immutable rsmaInfoHash }; struct SSmaStat { @@ -113,7 +117,13 @@ enum { TASK_TRIGGER_STAT_DROPPED = 5, }; -#define RSMA_TASK_INFO_HASH_SLOT 8 +enum { + RSMA_ROLE_CREATE = 0, + RSMA_ROLE_DROP = 1, + RSMA_ROLE_FETCH = 2, + RSMA_ROLE_SUBMIT = 3, + RSMA_ROLE_ITERATE = 4, +}; void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -133,33 +143,6 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); int32_t tdLockSma(SSma *pSma); int32_t tdUnLockSma(SSma *pSma); -static FORCE_INLINE int32_t tdRLockSmaEnv(SSmaEnv *pEnv) { - int code = taosThreadRwlockRdlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int32_t tdWLockSmaEnv(SSmaEnv *pEnv) { - int code = taosThreadRwlockWrlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int32_t tdUnLockSmaEnv(SSmaEnv *pEnv) { - int code = taosThreadRwlockUnlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - static FORCE_INLINE int8_t tdSmaStat(STSmaStat *pTStat) { if (pTStat) { return atomic_load_8(&pTStat->state); @@ -204,11 +187,13 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { atomic_or_fetch_8(&pTStat->state, TSDB_SMA_STAT_DROPPED); } } + +int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pDest, SRSmaInfo *pSrc); void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); -void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); -int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat); +void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); +int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdProcessRSmaRestoreImpl(SSma *pSma); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 3b32ca9c60..9186b4794a 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -168,6 +168,7 @@ int32_t smaSyncPreCommit(SSma* pSma); int32_t smaSyncCommit(SSma* pSma); int32_t smaSyncPostCommit(SSma* pSma); int32_t smaAsyncPreCommit(SSma* pSma); +int32_t smaAsyncCommit(SSma* pSma); int32_t smaAsyncPostCommit(SSma* pSma); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 4729187e8c..10f750f58d 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -15,11 +15,13 @@ #include "sma.h" -static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma); -static int32_t tdProcessRSmaCommitImpl(SSma *pSma); -static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); +static int32_t tdClearupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); /** * @brief Only applicable to Rollup SMA @@ -27,7 +29,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); * @param pSma * @return int32_t */ -int32_t smaSyncPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); } +int32_t smaSyncPreCommit(SSma *pSma) { return tdProcessRSmaSyncPreCommitImpl(pSma); } /** * @brief Only applicable to Rollup SMA @@ -35,7 +37,7 @@ int32_t smaSyncPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); * @param pSma * @return int32_t */ -int32_t smaSyncCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); } +int32_t smaSyncCommit(SSma *pSma) { return tdProcessRSmaSyncCommitImpl(pSma); } /** * @brief Only applicable to Rollup SMA @@ -43,7 +45,7 @@ int32_t smaSyncCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); } * @param pSma * @return int32_t */ -int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); } +int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaSyncPostCommitImpl(pSma); } /** * @brief Only applicable to Rollup SMA @@ -53,6 +55,14 @@ int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma) */ int32_t smaAsyncPreCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); } +/** + * @brief Only applicable to Rollup SMA + * + * @param pSma + * @return int32_t + */ +int32_t smaAsyncCommit(SSma *pSma) { return tdProcessRSmaAsyncCommitImpl(pSma); } + /** * @brief Only applicable to Rollup SMA * @@ -106,7 +116,7 @@ int32_t smaBegin(SSma *pSma) { * @param pSma * @return int32_t */ -static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) { +static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma) { SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); if (!pSmaEnv) { return TSDB_CODE_SUCCESS; @@ -135,7 +145,9 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) { } // step 3: perform persist task for qTaskInfo - tdRSmaPersistExecImpl(pRSmaStat); + pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; + pRSmaStat->commitSubmitVer = pRSmaStat->submitVer; + tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); smaDebug("vgId:%d, rsma pre commit success", SMA_VID(pSma)); @@ -148,7 +160,7 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) { * @param pSma * @return int32_t */ -static int32_t tdProcessRSmaCommitImpl(SSma *pSma) { +static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma) { SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); if (!pSmaEnv) { return TSDB_CODE_SUCCESS; @@ -156,21 +168,9 @@ static int32_t tdProcessRSmaCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } -/** - * @brief post-commit for rollup sma - * 1) clean up the outdated qtaskinfo files - * - * @param pSma - * @return int32_t - */ -static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) { - SVnode *pVnode = pSma->pVnode; - - if (!VND_IS_RSMA(pVnode)) { - return TSDB_CODE_SUCCESS; - } - - int64_t committed = pVnode->state.committed; +static int32_t tdClearupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) { + SVnode *pVnode = pSma->pVnode; + int64_t committed = pRSmaStat->commitAppliedVer; TdDirPtr pDir = NULL; TdDirEntryPtr pDirEntry = NULL; char dir[TSDB_FILENAME_LEN]; @@ -238,6 +238,29 @@ static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) { taosCloseDir(&pDir); regfree(®ex); + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief post-commit for rollup sma + * 1) clean up the outdated qtaskinfo files + * + * @param pSma + * @return int32_t + */ +static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { + SVnode *pVnode = pSma->pVnode; + if (!VND_IS_RSMA(pVnode)) { + return TSDB_CODE_SUCCESS; + } + + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(SMA_ENV_STAT(pSmaEnv)); + + // cleanup outdated qtaskinfo files + tdClearupQTaskInfoFiles(pSma, pRSmaStat); + return TSDB_CODE_SUCCESS; } @@ -259,8 +282,9 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); - // step 1: set rsma stat paused + // step 1: set rsma stat atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); + atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 1); // step 2: wait all triggered fetch tasks finished int32_t nLoops = 0; @@ -285,19 +309,45 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { RSMA_IMU_INFO_HASH(pRSmaStat) = RSMA_INFO_HASH(pRSmaStat); RSMA_INFO_HASH(pRSmaStat) = taosHashInit(RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + if (!RSMA_INFO_HASH(pRSmaStat)) { smaError("vgId:%d, rsma async commit failed since %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; } + // step 4: others + pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; + pRSmaStat->commitSubmitVer = pRSmaStat->submitVer; + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief commit for rollup sma + * + * @param pSma + * @return int32_t + */ +static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + if (!pSmaEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + + // perform persist task for qTaskInfo + tdRSmaPersistExecImpl(pRSmaStat, RSMA_IMU_INFO_HASH(pRSmaStat)); + return TSDB_CODE_SUCCESS; } /** * @brief Migrate rsmaInfo from iRsmaInfo to rsmaInfo if rsmaInfoHash not empty. - * - * @param pSma - * @return int32_t + * + * @param pSma + * @return int32_t */ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); @@ -309,9 +359,10 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); // step 1: merge rsmaInfoHash and iRsmaInfoHash + taosWLockLatch(SMA_ENV_LOCK(pSmaEnv)); if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) { - // TODO: + // TODO: optimization - just switch the hash pointer if rsmaInfoHash is empty } void *pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), NULL); @@ -320,15 +371,12 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { if (!taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t))) { taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); - smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), *pSuid); + smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), + *pSuid); } else { + // free the resources SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - SRSmaInfoItem *pItem = &pRSmaInfo->items[i]; - if (pItem->taskInfo) { - tdFreeQTaskInfo(pItem->taskInfo, SMA_VID(pSma), i + 1); - } - } + tdFreeRSmaInfo(pSma, pRSmaInfo, false); smaDebug("vgId:%d, rsma async post commit, free rsma info since already COW for table:%" PRIi64, SMA_VID(pSma), *pSuid); } @@ -339,5 +387,10 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { taosHashCleanup(RSMA_IMU_INFO_HASH(pRSmaStat)); RSMA_IMU_INFO_HASH(pRSmaStat) = NULL; + taosWUnLockLatch(SMA_ENV_LOCK(pSmaEnv)); + + // step 2: cleanup outdated qtaskinfo files + tdClearupQTaskInfoFiles(pSma, pRSmaStat); + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 77377cf089..577d5fd4fa 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -108,12 +108,7 @@ static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path) SMA_ENV_TYPE(pEnv) = smaType; - int code = taosThreadRwlockInit(&(pEnv->lock), NULL); - if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - taosMemoryFree(pEnv); - return NULL; - } + taosInitRWLatch(&(pEnv->lock)); if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma) != TSDB_CODE_SUCCESS) { tdFreeSmaEnv(pEnv); @@ -147,7 +142,6 @@ static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaEn void tdDestroySmaEnv(SSmaEnv *pSmaEnv) { if (pSmaEnv) { pSmaEnv->pStat = tdFreeSmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv)); - taosThreadRwlockDestroy(&(pSmaEnv->lock)); } } @@ -259,7 +253,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL); while (infoHash) { SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash; - tdFreeRSmaInfo(pSma, pSmaInfo); + tdFreeRSmaInfo(pSma, pSmaInfo, true); infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash); } } @@ -359,7 +353,7 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { } break; default: - smaError("vgId:%d, undefined smaType:%", SMA_VID(pSma), smaType); + smaError("vgId:%d undefined smaType:%", SMA_VID(pSma), smaType); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 300a0ec4be..32d6dee57f 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -91,6 +91,7 @@ static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskInfoIter *pIter) { void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC + if (!taskHandle) return; qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { smaDebug("vgId:%d, free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); @@ -98,14 +99,23 @@ void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { } else { smaDebug("vgId:%d, not free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); } + // TODO: clear files related to qTaskInfo? } -void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { +/** + * @brief general function to free rsmaInfo + * + * @param pSma + * @param pInfo + * @param isDeepFree Only stop tmrId and free pTSchema for deep free + * @return void* + */ +void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { if (pInfo) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (pItem->taskInfo) { - if (pItem->tmrId) { + if (isDeepFree && pItem->tmrId) { smaDebug("vgId:%d, table %" PRIi64 " stop fetch timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId, i + 1); taosTmrStopA(&pItem->tmrId); @@ -116,7 +126,9 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { pInfo->suid, i + 1); } } - taosMemoryFree(pInfo->pTSchema); + if (isDeepFree) { + taosMemoryFree(pInfo->pTSchema); + } taosMemoryFree(pInfo); } @@ -138,7 +150,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) if (!suid || !tbUids) { terrno = TSDB_CODE_INVALID_PTR; - smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); return TSDB_CODE_FAILED; } @@ -152,7 +164,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) if (pRSmaInfo->items[0].taskInfo) { if ((qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); return TSDB_CODE_FAILED; } else { smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), @@ -162,7 +174,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) if (pRSmaInfo->items[1].taskInfo) { if ((qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); return TSDB_CODE_FAILED; } else { smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), @@ -247,9 +259,10 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat if ((param->qmsgLen > 0) && param->qmsg[idx]) { SRetention *pRetention = SMA_RETENTION(pSma); STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); + SVnode *pVnode = pSma->pVnode; SReadHandle handle = { - .meta = pSma->pVnode->pMeta, - .vnode = pSma->pVnode, + .meta = pVnode->pMeta, + .vnode = pVnode, .initTqReader = 1, }; @@ -274,7 +287,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat pItem->level = idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2; smaInfo("vgId:%d, table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%" PRIi32, - SMA_VID(pSma), pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); + TD_VID(pVnode), pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); } return TSDB_CODE_SUCCESS; } @@ -341,7 +354,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con return TSDB_CODE_SUCCESS; _err: - tdFreeRSmaInfo(pSma, pRSmaInfo); + tdFreeRSmaInfo(pSma, pRSmaInfo, true); return TSDB_CODE_FAILED; } @@ -635,9 +648,18 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType return TSDB_CODE_SUCCESS; } +/** + * @brief During async commit, the SRSmaInfo object would be COW from iRSmaInfoHash and write lock should be applied. + * + * @param pSma + * @param suid + * @return SRSmaInfo* + */ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = NULL; + SRSmaInfo *pRSmaInfo = NULL; + if (!pEnv) { // only applicable when rsma env exists return NULL; @@ -648,11 +670,37 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { return NULL; } - SRSmaInfo *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); - if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + if (pRSmaInfo && (pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + return pRSmaInfo; + } + + if (RSMA_COMMIT_STAT(pStat) == 0) { return NULL; } - return pRSmaInfo; + + // clone the SRSmaInfo from iRsmaInfoHash to rsmaInfoHash if in committing stat + SRSmaInfo *pCowRSmaInfo = NULL; + // lock + taosWLockLatch(SMA_ENV_LOCK(pEnv)); + void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + if (iRSmaInfo) { + SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; + if (pIRSmaInfo) { + if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) { + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); + return NULL; + } + if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) { + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + return NULL; + } + } + } + // unlock + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + return pCowRSmaInfo; } static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { @@ -881,6 +929,13 @@ _err: return TSDB_CODE_FAILED; } +/** + * @brief Restore from SRSmaQTaskInfoItem + * + * @param pSma + * @param pItem + * @return int32_t + */ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) { SRSmaInfo *pRSmaInfo = NULL; void *qTaskInfo = NULL; @@ -906,7 +961,7 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem * if (qDeserializeTaskStatus(qTaskInfo, pItem->qTaskInfo, pItem->len) < 0) { smaError("vgId:%d, restore rsma task failed for table:%" PRIi64 " level %d since %s", SMA_VID(pSma), pItem->suid, - pItem->type, terrstr(terrno)); + pItem->type, terrstr()); return TSDB_CODE_FAILED; } smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, @@ -1053,26 +1108,27 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) { return TSDB_CODE_SUCCESS; } -int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) { +int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { SSma *pSma = pRSmaStat->pSma; SVnode *pVnode = pSma->pVnode; int32_t vid = SMA_VID(pSma); int64_t toffset = 0; bool isFileCreated = false; - if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) { + if (taosHashGetSize(pInfoHash) <= 0) { return TSDB_CODE_SUCCESS; } - void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); + void *infoHash = taosHashIterate(pInfoHash, NULL); if (!infoHash) { return TSDB_CODE_SUCCESS; } STFile tFile = {0}; - if (RSMA_SUBMIT_VER(pRSmaStat) > 0) { +#if 0 + if (pRSmaStat->commitAppliedVer > 0) { char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFName(vid, pSma->pVnode->state.applied, qTaskInfoFName); + tdRSmaQTaskInfoGetFName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr()); goto _err; @@ -1085,6 +1141,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) { isFileCreated = true; } +#endif while (infoHash) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; @@ -1100,7 +1157,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) { int8_t type = (int8_t)(i + 1); if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) { smaError("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo failed since %s", vid, pRSmaInfo->suid, - i + 1, terrstr(terrno)); + i + 1, terrstr()); goto _err; } if (!pOutput || len <= 0) { @@ -1116,7 +1173,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) { if (!isFileCreated) { char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFName(vid, pSma->pVnode->state.applied, qTaskInfoFName); + tdRSmaQTaskInfoGetFName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr()); goto _err; @@ -1149,11 +1206,11 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) { taosMemoryFree(pOutput); } - infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash); + infoHash = taosHashIterate(pInfoHash, infoHash); } if (isFileCreated) { - tFile.info.qTaskInfo.submitVer = atomic_load_64(&pRSmaStat->submitVer); + tFile.info.qTaskInfo.submitVer = atomic_load_64(&pRSmaStat->commitSubmitVer); if (tdUpdateTFileHeader(&tFile) < 0) { smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile), tstrerror(terrno)); diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 22d82e8df0..13befabed5 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -313,4 +313,99 @@ int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t return TSDB_CODE_SUCCESS; } + +static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t srcTaskInfo, SRSmaParam *param, + tb_uid_t suid, int8_t idx) { + SVnode *pVnode = pSma->pVnode; + char *pOutput = NULL; + int32_t len = 0; + + if (qSerializeTaskStatus(srcTaskInfo, &pOutput, &len) < 0) { + smaError("vgId:%d, rsma clone, table %" PRIi64 " serialize qTaskInfo failed since %s", TD_VID(pVnode), suid, + terrstr()); + goto _err; + } + + SReadHandle handle = { + .meta = pVnode->pMeta, + .vnode = pVnode, + .initTqReader = 1, + }; + ASSERT(!dstTaskInfo); + dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); + if (!dstTaskInfo) { + terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; + goto _err; + } + + if (qDeserializeTaskStatus(dstTaskInfo, pOutput, len) < 0) { + smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, + terrstr()); + goto _err; + } + + smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); + + taosMemoryFreeClear(pOutput); + return TSDB_CODE_SUCCESS; +_err: + taosMemoryFreeClear(pOutput); + tdFreeQTaskInfo(dstTaskInfo, TD_VID(pVnode), idx + 1); + return TSDB_CODE_FAILED; +} + +/** + * @brief pTSchema is shared + * + * @param pSma + * @param pDest + * @param pSrc + * @return int32_t + */ +int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pDest, SRSmaInfo *pSrc) { + SVnode *pVnode = pSma->pVnode; + SRSmaParam *param = NULL; + if (!pSrc) { + return TSDB_CODE_SUCCESS; + } + + if (!pDest) { + pDest = taosMemoryCalloc(1, sizeof(SRSmaInfo)); + if (!pDest) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + + memcpy(pDest, pSrc, sizeof(SRSmaInfo)); + + SMetaReader mr = {0}; + metaReaderInit(&mr, SMA_META(pSma), 0); + smaDebug("vgId:%d, rsma clone, suid is %" PRIi64, TD_VID(pVnode), pSrc->suid); + if (metaGetTableEntryByUid(&mr, pSrc->suid) < 0) { + smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), pSrc->suid, + terrstr()); + goto _err; + } + ASSERT(mr.me.type == TSDB_SUPER_TABLE); + ASSERT(mr.me.uid == pSrc->suid); + if (TABLE_IS_ROLLUP(mr.me.flags)) { + param = &mr.me.stbEntry.rsmaParam; + for (int i = 0; i < TSDB_RETENTION_L2; ++i) { + SRSmaInfoItem *pItem = &pSrc->items[i]; + if (pItem->taskInfo) { + tdCloneQTaskInfo(pSma, pDest->items[i].taskInfo, pItem->taskInfo, param, pSrc->suid, i); + } + } + smaDebug("vgId:%d, rsma clone env success for %" PRIi64, TD_VID(pVnode), pSrc->suid); + } + + metaReaderClear(&mr); + return TSDB_CODE_SUCCESS; +_err: + metaReaderClear(&mr); + tdFreeRSmaInfo(pSma, pDest, false); + return TSDB_CODE_FAILED; +} + // ... \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 4e55c05a47..c969861241 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -233,7 +233,8 @@ int vnodeCommit(SVnode *pVnode) { walBeginSnapshot(pVnode->pWal, pVnode->state.applied); // preCommit - smaSyncPreCommit(pVnode->pSma); + // smaSyncPreCommit(pVnode->pSma); + smaAsyncPreCommit(pVnode->pSma); // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { @@ -242,6 +243,8 @@ int vnodeCommit(SVnode *pVnode) { } if (VND_IS_RSMA(pVnode)) { + smaAsyncCommit(pVnode->pSma); + if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { ASSERT(0); return -1; @@ -276,7 +279,8 @@ int vnodeCommit(SVnode *pVnode) { pVnode->state.committed = info.state.committed; // postCommit - smaSyncPostCommit(pVnode->pSma); + // smaSyncPostCommit(pVnode->pSma); + smaAsyncPostCommit(pVnode->pSma); // apply the commit (TODO) walEndSnapshot(pVnode->pWal); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index cca212a4e4..3761516bda 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -900,7 +900,7 @@ static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void * _err: tDecoderClear(&coder); vError("vgId:%d, failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s", - TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr(terrno)); + TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr()); return -1; } From 0144ef524f0f049847842b45f34a167072270466 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 16 Jul 2022 17:16:57 +0800 Subject: [PATCH 110/124] other: api rename --- source/dnode/vnode/src/sma/smaCommit.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 10f750f58d..cdaaf2bbdb 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -21,7 +21,7 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); -static int32_t tdClearupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); +static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); /** * @brief Only applicable to Rollup SMA @@ -168,7 +168,7 @@ static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } -static int32_t tdClearupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) { +static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) { SVnode *pVnode = pSma->pVnode; int64_t committed = pRSmaStat->commitAppliedVer; TdDirPtr pDir = NULL; @@ -259,7 +259,7 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { SRSmaStat *pRSmaStat = SMA_RSMA_STAT(SMA_ENV_STAT(pSmaEnv)); // cleanup outdated qtaskinfo files - tdClearupQTaskInfoFiles(pSma, pRSmaStat); + tdCleanupQTaskInfoFiles(pSma, pRSmaStat); return TSDB_CODE_SUCCESS; } @@ -390,7 +390,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { taosWUnLockLatch(SMA_ENV_LOCK(pSmaEnv)); // step 2: cleanup outdated qtaskinfo files - tdClearupQTaskInfoFiles(pSma, pRSmaStat); + tdCleanupQTaskInfoFiles(pSma, pRSmaStat); return TSDB_CODE_SUCCESS; } From ed2384036fb6c5fb75f357b19ef593079803de28 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 16 Jul 2022 17:19:21 +0800 Subject: [PATCH 111/124] test:update the test case. --- tests/system-test/2-query/unique.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index fce0d21f4d..3e6e14be38 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -426,7 +426,7 @@ class TDTestCase: tdSql.query(" select unique(t1+c1) from stb1 ") tdSql.checkRows(13) tdSql.query(" select unique(t1+c1) from stb1 partition by tbname ") - tdSql.checkRows(13) + tdSql.checkRows(20) tdSql.query(" select unique(t1) from stb1 partition by tbname ") tdSql.checkRows(2) From 76f451744a42c2509405a1a374c082adf0345859 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 16 Jul 2022 17:20:37 +0800 Subject: [PATCH 112/124] fix(query): fix memory leak. --- source/libs/executor/src/sortoperator.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 9795907404..229d7dd670 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -234,9 +234,9 @@ void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); + tsortDestroySortHandle(pInfo->pSortHandle); taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pColMatchInfo); - taosMemoryFreeClear(param); } @@ -539,14 +539,12 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { } pInfo->startTs = taosGetTimestampUs(); - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pInputBlock, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); - tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort); for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { @@ -556,7 +554,6 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { } int32_t code = tsortOpen(pInfo->pSortHandle); - if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, terrno); } @@ -674,6 +671,7 @@ void destroyMultiwayMergeOperatorInfo(void* param, int32_t numOfOutput) { pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock); + tsortDestroySortHandle(pInfo->pSortHandle); taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pColMatchInfo); From 81c25a1a3014057f9e45f05fca40644f725da952 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 16 Jul 2022 17:32:10 +0800 Subject: [PATCH 113/124] fix: fix mnode close issue --- source/dnode/mnode/impl/src/mndMain.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 120b98d85a..041fc2a2d1 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -42,7 +42,6 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#include "qworker.h" static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -387,7 +386,6 @@ void mndPreClose(SMnode *pMnode) { void mndClose(SMnode *pMnode) { if (pMnode != NULL) { mDebug("start to close mnode"); - qWorkerDestroy((void **)&pMnode->pQuery); mndCleanupSteps(pMnode, -1); taosMemoryFreeClear(pMnode->path); taosMemoryFreeClear(pMnode); From 069778c6760638732938fb063611ef4f39de6cda Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sat, 16 Jul 2022 17:32:40 +0800 Subject: [PATCH 114/124] feat(stream): add log --- source/common/src/tdatablock.c | 2 +- source/libs/executor/src/timewindowoperator.c | 52 ++++++++++--------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index cd21de52ed..3b7663c4f4 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1752,7 +1752,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; int32_t len = 0; - len += snprintf(dumpBuf + len, size - len, "%s |block type %d |child id %d|group id:%" PRIu64 "| uid:%ld|======\n", "dumpBlockData", + len += snprintf(dumpBuf + len, size - len, "===stream===%s |block type %d |child id %d|group id:%" PRIu64 "| uid:%ld|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId, pDataBlock->info.uid); if (len >= size - 1) return dumpBuf; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 71eb221c64..0fa29fca74 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1353,13 +1353,13 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, if (chIds && pPullDataMap) { SArray* chAy = *(SArray**)chIds; int32_t size = taosArrayGetSize(chAy); - qDebug("window %" PRId64 " wait child size:%d", win.skey, size); + qDebug("===stream===window %" PRId64 " wait child size:%d", win.skey, size); for (int32_t i = 0; i < size; i++) { - qDebug("window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); + qDebug("===stream===window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); } continue; } else if (pPullDataMap) { - qDebug("close window %" PRId64, win.skey); + qDebug("===stream===close window %" PRId64, win.skey); } SResultRowPosition* pPos = (SResultRowPosition*)pIte; if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { @@ -2482,7 +2482,9 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId}; // add pull data request taosArrayPush(pInfo->pPullWins, &pull); - addPullWindow(pInfo->pPullDataMap, &winRes, taosArrayGetSize(pInfo->pChildren)); + int32_t size = taosArrayGetSize(pInfo->pChildren); + addPullWindow(pInfo->pPullDataMap, &winRes, size); + qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size); } else { int32_t index = -1; SArray* chArray = NULL; @@ -2492,14 +2494,14 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc chId = getChildIndex(pSDataBlock); index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); } - if (index != -1 && pSDataBlock->info.type == STREAM_PULL_DATA) { - qDebug("======delete child id %d", chId); - taosArrayRemove(chArray, index); - if (taosArrayGetSize(chArray) == 0) { - // pull data is over - taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); - } - } + // if (index != -1 && pSDataBlock->info.type == STREAM_PULL_DATA) { + // qDebug("===stream===delete child id %d", chId); + // taosArrayRemove(chArray, index); + // if (taosArrayGetSize(chArray) == 0) { + // // pull data is over + // taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); + // } + // } if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) { ignore = false; } @@ -2623,6 +2625,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { SArray* chArray = *(SArray**)chIds; int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); if (index != -1) { + qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId); taosArrayRemove(chArray, index); if (taosArrayGetSize(chArray) == 0) { // pull data is over @@ -2641,7 +2644,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; - qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -2657,18 +2660,18 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } return NULL; } - printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->binfo.pRes; } else { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows != 0) { - printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->binfo.pRes; } if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) { pInfo->returnUpdate = false; ASSERT(!IS_FINAL_OP(pInfo)); - printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); // process the rest of the data return pInfo->pUpdateRes; } @@ -2676,13 +2679,13 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->pPullDataRes->info.rows != 0) { // process the rest of the data ASSERT(IS_FINAL_OP(pInfo)); - printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->pPullDataRes; } doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows != 0) { // process the rest of the data - printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->pDelRes; } } @@ -2693,10 +2696,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { clearSpecialDataBlock(pInfo->pUpdateRes); removeDeleteResults(pUpdated, pInfo->pDelWins); pOperator->status = OP_RES_TO_RETURN; - qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); break; } - printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv"); + printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv"); maxTs = TMAX(maxTs, pBlock->info.window.ekey); if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA || @@ -2771,6 +2774,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pTmpInfo = pChildOp->info; pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; taosArrayPush(pInfo->pChildren, &pChildOp); + qDebug("===stream===add child, id:%d", chIndex); } SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; @@ -2795,14 +2799,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows != 0) { - printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->binfo.pRes; } if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) { pInfo->returnUpdate = false; ASSERT(!IS_FINAL_OP(pInfo)); - printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); // process the rest of the data return pInfo->pUpdateRes; } @@ -2811,14 +2815,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->pPullDataRes->info.rows != 0) { // process the rest of the data ASSERT(IS_FINAL_OP(pInfo)); - printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->pPullDataRes; } doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows != 0) { // process the rest of the data - printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->pDelRes; } // ASSERT(false); From e5e9252388756a0ce2c5ea9edce602a0d79e1b8b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 16 Jul 2022 17:40:54 +0800 Subject: [PATCH 115/124] some problems of planner --- source/libs/nodes/src/nodesUtilFuncs.c | 29 ++++++++++++------- source/libs/parser/src/parTranslater.c | 8 +++-- source/libs/planner/src/planLogicCreater.c | 2 +- source/libs/planner/src/planOptimizer.c | 21 +++++++++----- source/libs/planner/test/planJoinTest.cpp | 6 ++++ source/libs/planner/test/planOptimizeTest.cpp | 6 ++++ 6 files changed, 52 insertions(+), 20 deletions(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 7265e7ee78..db294dce1b 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1543,7 +1543,7 @@ typedef struct SCollectFuncsCxt { int32_t errCode; FFuncClassifier classifier; SNodeList* pFuncs; - SHashObj* pAliasName; + SHashObj* pFuncsSet; } SCollectFuncsCxt; static EDealRes collectFuncs(SNode* pNode, void* pContext) { @@ -1551,28 +1551,37 @@ static EDealRes collectFuncs(SNode* pNode, void* pContext) { if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId) && !(((SExprNode*)pNode)->orderAlias)) { SExprNode* pExpr = (SExprNode*)pNode; - if (NULL == taosHashGet(pCxt->pAliasName, pExpr->aliasName, strlen(pExpr->aliasName))) { + if (NULL == taosHashGet(pCxt->pFuncsSet, &pExpr, POINTER_BYTES)) { pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode)); - taosHashPut(pCxt->pAliasName, pExpr->aliasName, strlen(pExpr->aliasName), &pExpr, POINTER_BYTES); + taosHashPut(pCxt->pFuncsSet, &pExpr, POINTER_BYTES, &pExpr, POINTER_BYTES); } return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } return DEAL_RES_CONTINUE; } +static uint32_t funcNodeHash(const char* pKey, uint32_t len) { + SExprNode* pExpr = *(SExprNode**)pKey; + return MurmurHash3_32(pExpr->aliasName, strlen(pExpr->aliasName)); +} + +static int32_t funcNodeEqual(const void* pLeft, const void* pRight, size_t len) { + return nodesEqualNode(*(const SNode**)pLeft, *(const SNode**)pRight) ? 0 : 1; +} + int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs) { if (NULL == pSelect || NULL == pFuncs) { return TSDB_CODE_FAILED; } - SCollectFuncsCxt cxt = { - .errCode = TSDB_CODE_SUCCESS, - .classifier = classifier, - .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs), - .pAliasName = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, false)}; - if (NULL == cxt.pFuncs) { + SCollectFuncsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, + .classifier = classifier, + .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs), + .pFuncsSet = taosHashInit(4, funcNodeHash, false, false)}; + if (NULL == cxt.pFuncs || NULL == cxt.pFuncsSet) { return TSDB_CODE_OUT_OF_MEMORY; } + taosHashSetEqualFp(cxt.pFuncsSet, funcNodeEqual); *pFuncs = NULL; nodesWalkSelectStmt(pSelect, clause, collectFuncs, &cxt); if (TSDB_CODE_SUCCESS == cxt.errCode) { @@ -1584,7 +1593,7 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifi } else { nodesDestroyList(cxt.pFuncs); } - taosHashCleanup(cxt.pAliasName); + taosHashCleanup(cxt.pFuncsSet); return cxt.errCode; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8998971c74..4e5c2c3e7e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -868,7 +868,8 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - if (strict && (pVal->node.resType.bytes > targetDt.bytes - VARSTR_HEADER_SIZE)) { + if (strict && (!IS_VAR_DATA_TYPE(pVal->node.resType.type) || + pVal->node.resType.bytes > targetDt.bytes - VARSTR_HEADER_SIZE)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1); @@ -888,6 +889,9 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, break; } case TSDB_DATA_TYPE_NCHAR: { + if (strict && !IS_VAR_DATA_TYPE(pVal->node.resType.type)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + } pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1); if (NULL == pVal->datum.p) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); @@ -1168,7 +1172,7 @@ static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* p return TSDB_CODE_SUCCESS; } if (isSelectStmt(pCxt->pCurrStmt)) { - //select percentile() without from clause is also valid + // select percentile() without from clause is also valid if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index e0b020ce21..4903beddbe 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -282,7 +282,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pScan->hasNormalCols = true; } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && SCAN_TYPE_SYSTEM_TABLE != pScan->scanType) { code = addPrimaryKeyCol(pScan->tableId, &pScan->pScanCols); } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index cdbe74bf04..4473c7c8b6 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1050,8 +1050,11 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS } } - int32_t code = - replaceLogicNode(pLogicSubplan, (SLogicNode*)pSort, (SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0)); + SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0); + if (NULL == pSort->node.pParent) { + TSWAP(pSort->node.pTargets, pChild->pTargets); + } + int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pSort, pChild); if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pSort->node.pChildren); nodesDestroyNode((SNode*)pSort); @@ -1982,11 +1985,15 @@ static int32_t rewriteUniqueOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog } static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || !(((SAggLogicNode*)pNode)->hasLastRow) || - NULL != ((SAggLogicNode*)pNode)->pGroupKeys || 1 != LIST_LENGTH(pNode->pChildren) || - QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0)) || - NULL != ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->node.pConditions || - 0 == ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->cacheLastMode) { + if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) || + QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0))) { + return false; + } + + SAggLogicNode* pAgg = (SAggLogicNode*)pNode; + SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0); + if (!pAgg->hasLastRow || NULL != pAgg->pGroupKeys || NULL != pScan->node.pConditions || 0 == pScan->cacheLastMode || + IS_TSWINDOW_SPECIFIED(pScan->scanRange)) { return false; } diff --git a/source/libs/planner/test/planJoinTest.cpp b/source/libs/planner/test/planJoinTest.cpp index 8c321e75be..66ef4d3f19 100644 --- a/source/libs/planner/test/planJoinTest.cpp +++ b/source/libs/planner/test/planJoinTest.cpp @@ -45,6 +45,12 @@ TEST_F(PlanJoinTest, withWhere) { "WHERE t1.c1 > t2.c1 AND t1.c2 = 'abc' AND t2.c2 = 'qwe'"); } +TEST_F(PlanJoinTest, withAggAndOrderBy) { + useDb("root", "test"); + + run("SELECT t1.ts, TOP(t2.c1, 10) FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts ORDER BY t2.ts"); +} + TEST_F(PlanJoinTest, multiJoin) { useDb("root", "test"); diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 75b4aa0dc1..770ac94e5b 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -84,6 +84,12 @@ TEST_F(PlanOptimizeTest, eliminateProjection) { // run("select 1-abs(c1) from (select unique(c1) c1 from st1s3) order by 1 nulls first"); } +TEST_F(PlanOptimizeTest, mergeProjects) { + useDb("root", "test"); + + run("SELECT * FROM (SELECT * FROM t1 WHERE c1 > 10 ORDER BY ts) ORDER BY ts"); +} + TEST_F(PlanOptimizeTest, pushDownProjectCond) { useDb("root", "test"); run("select 1-abs(c1) from (select unique(c1) c1 from st1s3) where 1-c1>5 order by 1 nulls first"); From 0c4ea9b37905cbff230c975d6ab575893aa0a831 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 16 Jul 2022 17:45:19 +0800 Subject: [PATCH 116/124] fix: destroy rsp.pArray after encoding --- source/dnode/vnode/src/vnd/vnodeSvr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index cca212a4e4..24e932bd0a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -689,6 +689,7 @@ _exit: tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen); tEncodeSVDropTbBatchRsp(&encoder, &rsp); tEncoderClear(&encoder); + taosArrayDestroy(rsp.pArray); return 0; } From ba87537a3e74d9557dd3821131fd29ec8830a020 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 16 Jul 2022 17:45:23 +0800 Subject: [PATCH 117/124] feat: update taostools for3.0 (#14998) * feat: update taos-tools for 3.0 [TD-14141] * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index bd496f76b6..b7b922268c 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit bd496f76b64931c66da2f8b0f24143a98a881cde +Subproject commit b7b922268c4a06d9db77ffdfde0726f3d9900b72 From 43852a0e690a3a8eff70fdfcf54a057d5bd67d39 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 16 Jul 2022 19:15:14 +0800 Subject: [PATCH 118/124] some problems of planner --- source/libs/nodes/src/nodesUtilFuncs.c | 3 +++ tests/script/tsim/tag/5.sim | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index db294dce1b..2636b7ebe0 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1566,6 +1566,9 @@ static uint32_t funcNodeHash(const char* pKey, uint32_t len) { } static int32_t funcNodeEqual(const void* pLeft, const void* pRight, size_t len) { + if (0 != strcmp((*(const SExprNode**)pLeft)->aliasName, (*(const SExprNode**)pRight)->aliasName)) { + return 1; + } return nodesEqualNode(*(const SNode**)pLeft, *(const SNode**)pRight) ? 0 : 1; } diff --git a/tests/script/tsim/tag/5.sim b/tests/script/tsim/tag/5.sim index 92695ddfcf..319d9c7bc0 100644 --- a/tests/script/tsim/tag/5.sim +++ b/tests/script/tsim/tag/5.sim @@ -24,7 +24,7 @@ sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 tinyint, tgcol2 int, $i = 0 while $i < 5 $tb = $tbPrefix . $i - sql create table $tb using $mt tags( 0, 0, 0, 0, 0 ) + sql create table $tb using $mt tags( 0, 0, 0, 0, '0' ) $x = 0 while $x < $rowNum $ms = $x . m @@ -35,7 +35,7 @@ while $i < 5 endw while $i < 10 $tb = $tbPrefix . $i - sql create table $tb using $mt tags( 1, 1, 1, 1, 1 ) + sql create table $tb using $mt tags( 1, 1, 1, 1, '1' ) $x = 0 while $x < $rowNum $ms = $x . m From 478cdfe7a96c34e500c50b709270c8518a77b3a9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jul 2022 19:42:11 +0800 Subject: [PATCH 119/124] test: valgrind case --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4f64908bd6..3e2d7a38f0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -309,7 +309,7 @@ ./test.sh -f tsim/valgrind/checkError1.sim ./test.sh -f tsim/valgrind/checkError2.sim ./test.sh -f tsim/valgrind/checkError3.sim -# jira ./test.sh -f tsim/valgrind/checkError4.sim +./test.sh -f tsim/valgrind/checkError4.sim # --- vnode # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim From 5a5e30c4560a2a8b135e26a4c29853127a83f28d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 16 Jul 2022 19:44:08 +0800 Subject: [PATCH 120/124] fix(query): remove invalid free. --- source/libs/executor/src/executorimpl.c | 4 +++- source/libs/executor/src/tsort.c | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 28faa473db..b8612960e4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2217,6 +2217,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx if (completed == totalSources) { return setAllSourcesCompleted(pOperator, startTs); } + + sched_yield(); } _error: @@ -3743,7 +3745,7 @@ void doDestroyExchangeOperatorInfo(void* param) { taosArrayDestroy(pExInfo->pSources); taosArrayDestroy(pExInfo->pSourceDataInfo); if (pExInfo->pResult != NULL) { - blockDataDestroy(pExInfo->pResult); + pExInfo->pResult = blockDataDestroy(pExInfo->pResult); } tsem_destroy(&pExInfo->ready); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 9b15cc7adb..caa3ba3cf3 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -119,7 +119,6 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { blockDataDestroy(pSortHandle->pDataBlock); for (size_t i = 0; i < taosArrayGetSize(pSortHandle->pOrderedSource); i++){ SSortSource** pSource = taosArrayGet(pSortHandle->pOrderedSource, i); - blockDataDestroy((*pSource)->src.pBlock); taosMemoryFreeClear(*pSource); } taosArrayDestroy(pSortHandle->pOrderedSource); From 6756c6515a8693da6f3e577ff142e60e05cd9025 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 16 Jul 2022 19:54:14 +0800 Subject: [PATCH 121/124] fix(query): add null pointer check. --- source/libs/executor/src/tsort.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index caa3ba3cf3..cd8470883b 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -109,6 +109,10 @@ static int32_t sortComparClearup(SMsortComparParam* cmpParam) { } void tsortDestroySortHandle(SSortHandle* pSortHandle) { + if (pSortHandle == NULL) { + return; + } + tsortClose(pSortHandle); if (pSortHandle->pMergeTree != NULL) { tMergeTreeDestroy(pSortHandle->pMergeTree); From cb1b6794293121492b5f60c9d45aec4a524c850a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jul 2022 20:13:44 +0800 Subject: [PATCH 122/124] test: valgrind case --- tests/script/jenkins/basic.txt | 27 +++---- tests/script/tsim/compute/last_row.sim | 2 - tests/script/tsim/stable/alter_insert1.sim | 3 - tests/script/tsim/stable/alter_insert2.sim | 2 - tests/script/tsim/stable/dnode3.sim | 2 - tests/script/tsim/tag/add.sim | 28 ++++---- tests/script/tsim/tag/change.sim | 37 +++++----- tests/script/tsim/valgrind/checkError4.sim | 1 - tests/script/tsim/valgrind/checkError5.sim | 82 ++++++++++++++++++++++ 9 files changed, 128 insertions(+), 56 deletions(-) create mode 100644 tests/script/tsim/valgrind/checkError5.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 3e2d7a38f0..7347446556 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -266,26 +266,27 @@ ./test.sh -f tsim/tmq/snapshot1.sim # --- stable +./test.sh -f tsim/stable/alter_comment.sim +./test.sh -f tsim/stable/alter_count.sim +./test.sh -f tsim/stable/alter_import.sim +./test.sh -f tsim/stable/alter_insert1.sim +./test.sh -f tsim/stable/alter_insert2.sim +./test.sh -f tsim/stable/alter_metrics.sim +./test.sh -f tsim/stable/column_add.sim +./test.sh -f tsim/stable/column_drop.sim +./test.sh -f tsim/stable/column_modify.sim ./test.sh -f tsim/stable/disk.sim ./test.sh -f tsim/stable/dnode3.sim ./test.sh -f tsim/stable/metrics.sim ./test.sh -f tsim/stable/refcount.sim ./test.sh -f tsim/stable/show.sim -./test.sh -f tsim/stable/values.sim -./test.sh -f tsim/stable/vnode3.sim -./test.sh -f tsim/stable/column_add.sim -./test.sh -f tsim/stable/column_drop.sim -./test.sh -f tsim/stable/column_modify.sim ./test.sh -f tsim/stable/tag_add.sim ./test.sh -f tsim/stable/tag_drop.sim +./test.sh -f tsim/stable/tag_filter.sim ./test.sh -f tsim/stable/tag_modify.sim ./test.sh -f tsim/stable/tag_rename.sim -./test.sh -f tsim/stable/alter_comment.sim -./test.sh -f tsim/stable/alter_count.sim -./test.sh -f tsim/stable/alter_insert1.sim -./test.sh -f tsim/stable/alter_insert2.sim -./test.sh -f tsim/stable/alter_import.sim -./test.sh -f tsim/stable/tag_filter.sim +./test.sh -f tsim/stable/values.sim +./test.sh -f tsim/stable/vnode3.sim # --- for multi process mode ./test.sh -f tsim/user/basic.sim -m @@ -367,7 +368,7 @@ ./test.sh -f tsim/compute/diff2.sim ./test.sh -f tsim/compute/first.sim ./test.sh -f tsim/compute/interval.sim -# jira ./test.sh -f tsim/compute/last_row.sim +./test.sh -f tsim/compute/last_row.sim ./test.sh -f tsim/compute/last.sim ./test.sh -f tsim/compute/leastsquare.sim ./test.sh -f tsim/compute/max.sim @@ -416,7 +417,7 @@ ./test.sh -f tsim/tag/4.sim ./test.sh -f tsim/tag/5.sim # jira ./test.sh -f tsim/tag/6.sim -# jira ./test.sh -f tsim/tag/add.sim +./test.sh -f tsim/tag/add.sim ./test.sh -f tsim/tag/bigint.sim ./test.sh -f tsim/tag/binary_binary.sim ./test.sh -f tsim/tag/binary.sim diff --git a/tests/script/tsim/compute/last_row.sim b/tests/script/tsim/compute/last_row.sim index 590fada86a..70fb6626cd 100644 --- a/tests/script/tsim/compute/last_row.sim +++ b/tests/script/tsim/compute/last_row.sim @@ -65,8 +65,6 @@ if $data00 != 19 then return -1 endi - - print =============== step7 sql select last_row(tbcol) from $mt print ===> $data00 diff --git a/tests/script/tsim/stable/alter_insert1.sim b/tests/script/tsim/stable/alter_insert1.sim index 82781f2fe5..bcea0b48c4 100644 --- a/tests/script/tsim/stable/alter_insert1.sim +++ b/tests/script/tsim/stable/alter_insert1.sim @@ -934,11 +934,8 @@ if $data79 != null then endi print ======== step9 - system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql select * from tb order by ts asc if $rows != 8 then diff --git a/tests/script/tsim/stable/alter_insert2.sim b/tests/script/tsim/stable/alter_insert2.sim index a30175f398..faae24d32a 100644 --- a/tests/script/tsim/stable/alter_insert2.sim +++ b/tests/script/tsim/stable/alter_insert2.sim @@ -604,9 +604,7 @@ sql_error alter table tb drop column a print ======== step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql select * from tb order by ts desc if $rows != 7 then diff --git a/tests/script/tsim/stable/dnode3.sim b/tests/script/tsim/stable/dnode3.sim index 03e8df26b7..4a2d148738 100644 --- a/tests/script/tsim/stable/dnode3.sim +++ b/tests/script/tsim/stable/dnode3.sim @@ -69,8 +69,6 @@ if $rows != 2 then return -1 endi -sleep 100 - print =============== step2 $i = 1 $tb = $tbPrefix . $i diff --git a/tests/script/tsim/tag/add.sim b/tests/script/tsim/tag/add.sim index 423224b1d6..a22d337eff 100644 --- a/tests/script/tsim/tag/add.sim +++ b/tests/script/tsim/tag/add.sim @@ -250,8 +250,8 @@ sql alter table $mt add tag tgcol6 binary(10) sql reset query cache sql alter table $tb set tag tgcol4=false -sql alter table $tb set tag tgcol5=5 -sql alter table $tb set tag tgcol6=6 +sql alter table $tb set tag tgcol5='5' +sql alter table $tb set tag tgcol6='6' sql reset query cache sql select * from $mt where tgcol5 = '5' @@ -390,8 +390,8 @@ sql alter table $mt add tag tgcol5 binary(17) sql alter table $mt add tag tgcol6 bool sql reset query cache sql alter table $tb set tag tgcol4=4 -sql alter table $tb set tag tgcol5=5 -sql alter table $tb set tag tgcol6=1 +sql alter table $tb set tag tgcol5='5' +sql alter table $tb set tag tgcol6='1' sql reset query cache sql select * from $mt where tgcol5 = '5' @@ -409,7 +409,7 @@ endi if $data03 != 5 then return -1 endi -if $data04 != 1 then +if $data04 != 0 then return -1 endi @@ -425,7 +425,7 @@ $i = 9 $mt = $mtPrefix . $i $tb = $tbPrefix . $i sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 double, tgcol2 binary(10), tgcol3 binary(10)) -sql create table $tb using $mt tags( 1, 2, '3' ) +sql create table $tb using $mt tags( 1, '2', '3' ) sql insert into $tb values(now, 1) sql select * from $mt where tgcol2 = '2' if $rows != 1 then @@ -520,7 +520,7 @@ sql alter table $mt add tag tgcol4 binary(10) sql alter table $mt add tag tgcol5 bool sql reset query cache -sql alter table $tb set tag tgcol4=4 +sql alter table $tb set tag tgcol4='4' sql alter table $tb set tag tgcol5=false sql reset query cache @@ -598,9 +598,9 @@ sql alter table $mt add tag tgcol7 bigint sql alter table $mt add tag tgcol8 smallint sql reset query cache -sql alter table $tb set tag tgcol4=4 +sql alter table $tb set tag tgcol4='4' sql alter table $tb set tag tgcol5=5 -sql alter table $tb set tag tgcol6=6 +sql alter table $tb set tag tgcol6='6' sql alter table $tb set tag tgcol7=7 sql alter table $tb set tag tgcol8=8 sql reset query cache @@ -687,11 +687,11 @@ sql alter table $mt add tag tgcol5 bigint sql reset query cache sql alter table $tb set tag tgcol1=false -sql alter table $tb set tag tgcol2=5 +sql alter table $tb set tag tgcol2='5' sql alter table $tb set tag tgcol3=4 -sql alter table $tb set tag tgcol4=3 +sql alter table $tb set tag tgcol4='3' sql alter table $tb set tag tgcol5=2 -sql alter table $tb set tag tgcol6=1 +sql alter table $tb set tag tgcol6='1' sql reset query cache sql select * from $mt where tgcol4 = '3' @@ -783,8 +783,8 @@ sql alter table $mt add tag tgcol4 int sql alter table $mt add tag tgcol6 bigint sql reset query cache -sql alter table $tb set tag tgcol1=7 -sql alter table $tb set tag tgcol2=8 +sql alter table $tb set tag tgcol1='7' +sql alter table $tb set tag tgcol2='8' sql alter table $tb set tag tgcol3=9 sql alter table $tb set tag tgcol4=10 sql alter table $tb set tag tgcol5=11 diff --git a/tests/script/tsim/tag/change.sim b/tests/script/tsim/tag/change.sim index f58e387217..236ad8ea67 100644 --- a/tests/script/tsim/tag/change.sim +++ b/tests/script/tsim/tag/change.sim @@ -40,19 +40,19 @@ if $data03 != 2 then return -1 endi -sql alter table $mt change tag tagcx tgcol3 -x step21 +sql alter table $mt rename tag tagcx tgcol3 -x step21 return -1 step21: -sql alter table $mt change tag tgcol1 tgcol2 -x step22 +sql alter table $mt rename tag tgcol1 tgcol2 -x step22 return -1 step22: -#sql alter table $mt change tag tgcol1 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -x step20 +#sql alter table $mt rename tag tgcol1 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -x step20 # return -1 #step20: -sql alter table $mt change tag tgcol1 tgcol3 -sql alter table $mt change tag tgcol2 tgcol4 -sql alter table $mt change tag tgcol4 tgcol3 -x step23 +sql alter table $mt rename tag tgcol1 tgcol3 +sql alter table $mt rename tag tgcol2 tgcol4 +sql alter table $mt rename tag tgcol4 tgcol3 -x step23 return -1 step23: @@ -77,8 +77,8 @@ if $data03 != 2 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol3 -sql alter table $mt change tag tgcol2 tgcol4 +sql alter table $mt rename tag tgcol1 tgcol3 +sql alter table $mt rename tag tgcol2 tgcol4 print =============== step4 $i = 4 @@ -101,8 +101,8 @@ if $data03 != 2.00000 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol3 -sql alter table $mt change tag tgcol2 tgcol4 +sql alter table $mt rename tag tgcol1 tgcol3 +sql alter table $mt rename tag tgcol2 tgcol4 print =============== step5 $i = 5 @@ -125,8 +125,8 @@ if $data03 != 2 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol3 -sql alter table $mt change tag tgcol2 tgcol4 +sql alter table $mt rename tag tgcol1 tgcol3 +sql alter table $mt rename tag tgcol2 tgcol4 print =============== step6 $i = 6 @@ -163,15 +163,14 @@ endi sql alter table $mt drop tag tgcol3 sql reset query cache -sql alter table $mt change tag tgcol4 tgcol3 -sql alter table $mt change tag tgcol1 tgcol7 -sql alter table $mt change tag tgcol2 tgcol8 +sql alter table $mt rename tag tgcol4 tgcol3 +sql alter table $mt rename tag tgcol1 tgcol7 +sql alter table $mt rename tag tgcol2 tgcol8 sql reset query cache -sql alter table $mt change tag tgcol3 tgcol9 -sql alter table $mt change tag tgcol5 tgcol10 -sql alter table $mt change tag tgcol6 tgcol11 +sql alter table $mt rename tag tgcol3 tgcol9 +sql alter table $mt rename tag tgcol5 tgcol10 +sql alter table $mt rename tag tgcol6 tgcol11 -sleep 3000 sql reset query cache print =============== step2 diff --git a/tests/script/tsim/valgrind/checkError4.sim b/tests/script/tsim/valgrind/checkError4.sim index be1dc003ad..745ff40517 100644 --- a/tests/script/tsim/valgrind/checkError4.sim +++ b/tests/script/tsim/valgrind/checkError4.sim @@ -57,7 +57,6 @@ sql drop table d1.stb2 sql drop database d1 sql drop database d2 -goto _OVER print =============== step7: repeat sql create database d1 vgroups 2 buffer 3 sql create database d2 vgroups 2 buffer 3 diff --git a/tests/script/tsim/valgrind/checkError5.sim b/tests/script/tsim/valgrind/checkError5.sim new file mode 100644 index 0000000000..61964d1c42 --- /dev/null +++ b/tests/script/tsim/valgrind/checkError5.sim @@ -0,0 +1,82 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 +system sh/exec.sh -n dnode1 -s start -v +sql connect + +print =============== step1: create drop show dnodes +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not ready! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +if $rows != 1 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database db +sql use db +sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" + +print =============== step3: alter stb +sql_error alter table db.stb add column ts int +sql alter table db.stb add column c3 int +sql alter table db.stb add column c4 bigint +sql alter table db.stb add column c5 binary(12) +sql alter table db.stb drop column c1 +sql alter table db.stb drop column c4 +sql alter table db.stb MODIFY column c2 binary(32) +sql alter table db.stb add tag t4 bigint +sql alter table db.stb add tag c1 int +sql alter table db.stb add tag t5 binary(12) +sql alter table db.stb drop tag c1 +sql alter table db.stb drop tag t5 +sql alter table db.stb MODIFY tag t3 binary(32) +sql alter table db.stb rename tag t1 tx + +sql alter table db.stb comment 'abcde' ; +goto _OVER + +print =============== step4: alter tb +sql create table tb (ts timestamp, a int) +sql insert into tb values(now-28d, -28) +sql select count(a) from tb +sql alter table tb add column b smallint +sql insert into tb values(now-25d, -25, 0) +sql select count(b) from tb +sql alter table tb add column c tinyint +sql insert into tb values(now-22d, -22, 3, 0) +sql select count(c) from tb +sql alter table tb add column d int +sql insert into tb values(now-19d, -19, 6, 0, 0) +sql select count(d) from tb +sql alter table tb add column e bigint +sql alter table tb add column f float +sql alter table tb add column g double +sql alter table tb add column h binary(10) +sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb +sql select * from tb order by ts desc + +_OVER: +system sh/exec.sh -n dnode1 -s stop -x SIGINT +print =============== check +$null= + +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content > 0 then + return -1 +endi + +if $system_content == $null then + return -1 +endi From 312f3a3cd56658bae9dc5d1a74a5a08af9fb9c8b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jul 2022 20:47:18 +0800 Subject: [PATCH 123/124] fix: definite lost while alter stable comment --- source/dnode/mnode/impl/src/mndStb.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index bff33af5af..1306a0252f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1689,6 +1689,9 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p _OVER: taosMemoryFreeClear(stbObj.pTags); taosMemoryFreeClear(stbObj.pColumns); + if (pAlter->commentLen > 0) { + taosMemoryFreeClear(stbObj.comment); + } return code; } @@ -1733,7 +1736,7 @@ _OVER: mndReleaseStb(pMnode, pStb); mndReleaseDb(pMnode, pDb); - taosArrayDestroy(alterReq.pFields); + tFreeSMAltertbReq(&alterReq); return code; } From f76b7ce165382affcd708009e0c62869b1811219 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 16 Jul 2022 21:23:01 +0800 Subject: [PATCH 124/124] test: valgrind case --- tests/script/tsim/valgrind/checkError6.sim | 77 ++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/script/tsim/valgrind/checkError6.sim diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim new file mode 100644 index 0000000000..a9f66647f9 --- /dev/null +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -0,0 +1,77 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 +system sh/exec.sh -n dnode1 -s start -v +sql connect + +print =============== step1: create drop show dnodes +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not ready! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +if $rows != 1 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi + +$tbPrefix = tb +$tbNum = 5 +$rowNum = 10 + +print =============== step2: prepare data +sql create database db vgroups 2 +sql use db +sql create table if not exists stb (ts timestamp, tbcol int, tbcol2 float, tbcol3 double) tags (tgcol int unsigned) + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + sql insert into $tb values ($ms , $x , $x , $x ) + $x = $x + 1 + endw + $i = $i + 1 +endw + +print =============== step3: avg +sql select avg(tbcol) from tb1 +sql select avg(tbcol) from tb1 where ts <= 1601481840000 +sql select avg(tbcol) as b from tb1 +sql select avg(tbcol) as b from tb1 interval(1d) +goto _OVER +sql select avg(tbcol) as b from tb1 where ts <= 1601481840000s interval(1m) +sql select avg(tbcol) as c from stb +sql select avg(tbcol) as c from stb where ts <= 1601481840000 +sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000 +sql select avg(tbcol) as c from stb interval(1m) +sql select avg(tbcol) as c from stb interval(1d) +sql select avg(tbcol) as b from stb where ts <= 1601481840000s interval(1m) +sql select avg(tbcol) as c from stb group by tgcol +sql select avg(tbcol) as b from stb where ts <= 1601481840000s partition by tgcol interval(1m) + +_OVER: +system sh/exec.sh -n dnode1 -s stop -x SIGINT +print =============== check +$null= + +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content > 0 then + return -1 +endi + +if $system_content == $null then + return -1 +endi