From 77b890e263b85585228d7508a3f8c199a8ad83e8 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 20 May 2022 19:59:35 +0800 Subject: [PATCH 01/73] add fsync case --- tests/system-test/0-others/fsync.py | 287 ++++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 tests/system-test/0-others/fsync.py diff --git a/tests/system-test/0-others/fsync.py b/tests/system-test/0-others/fsync.py new file mode 100644 index 0000000000..8ee724175b --- /dev/null +++ b/tests/system-test/0-others/fsync.py @@ -0,0 +1,287 @@ +import datetime + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +PRIMARY_COL = "ts" + +INT_COL = "c1" +BINT_COL = "c2" +SINT_COL = "c3" +TINT_COL = "c4" +FLOAT_COL = "c5" +DOUBLE_COL = "c6" +BOOL_COL = "c7" + +BINARY_COL = "c8" +NCHAR_COL = "c9" +TS_COL = "c10" + +NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] +CHAR_COL = [ BINARY_COL, NCHAR_COL, ] +BOOLEAN_COL = [ BOOL_COL, ] +TS_TYPE_COL = [ TS_COL, ] + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def __kill_process(self, process_name): + killCmd = f"ps -ef|grep -w {process_name}| grep -v grep | awk '{{print $2}}' | xargs kill -TERM > /dev/null 2>&1" + + psCmd = f"ps -ef|grep -w {process_name}| grep -v grep | awk '{{print $2}}'" + while processID := subprocess.check_output(psCmd, shell=True): + os.system(killCmd) + time.sleep(1) + + def test_fsync_current(self): + wal_index = 0 + fsync_index = 0 + tdSql.query("show databases") + for i in range(tdSql.queryCols): + if tdSql.cursor.description[i] == "wal": + wal_index = i + if tdSql.cursor.description[i] == "fsync": + fsync_index = i + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 wal 1") + tdSql.query("show datsbases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, wal_index, 1) + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 wal 2") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 fsync 0") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 0) + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 fsync 3000") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 fsync 180000") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 180000) + + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 wal 1 fsync 3000") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 180000) + tdSql.checkData(i, wal_index, 1) + + tdSql.execute("drop database if exists db1") + tdSql.execute("create database db1 wal 2 fsync 3000") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("alter database db1 wal 1") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + tdSql.checkData(i, wal_index, 1) + + tdSql.execute("alter database db1 wal 2") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("alter database db1 fsync 0") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 0) + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("alter database db1 fsync 3000") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("alter database db1 fsync 18000") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 18000) + tdSql.checkData(i, wal_index, 2) + + tdSql.execute("alter database db1 wal 1 fsync 3000") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "db1": + tdSql.checkData(i, fsync_index, 3000) + tdSql.checkData(i, wal_index, 1) + + @property + def fsync_create_err(self): + return [ + "create database db1 wal 0", + "create database db1 wal 3", + "create database db1 wal null", + "create database db1 wal true", + "create database db1 wal 1.1", + "create database db1 fsync -1", + "create database db1 fsync 180001", + "create database db1 fsync 10.111", + "create database db1 fsync true", + ] + + @property + def fsync_alter_err(self): + return [ + "alter database db1 wal 0", + "alter database db1 wal 3", + "alter database db1 wal null", + "alter database db1 wal true", + "alter database db1 wal 1.1", + "alter database db1 fsync -1", + "alter database db1 fsync 180001", + "alter database db1 fsync 10.111", + "alter database db1 fsync true", + ] + + def test_fsync_err(self): + for sql in self.fsync_create_err: + tdSql.error(sql) + tdSql.query("create database db1") + for sql in self.fsync_alter_err: + tdSql.error(sql) + tdSql.query("drop database db1") + + def all_test(self): + self.test_fsync_err() + self.test_fsync_current() + + # def __create_tb(self): + + # tdLog.printNoPrefix("==========step1:create table") + # create_stb_sql = f'''create table stb1( + # ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + # {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + # {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + # ) tags (t1 int) + # ''' + # create_ntb_sql = f'''create table t1( + # ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + # {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + # {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + # ) + # ''' + # tdSql.execute(create_stb_sql) + # tdSql.execute(create_ntb_sql) + + # for i in range(4): + # tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + # { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2} + + # def __insert_data(self, rows): + # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + # for i in range(rows): + # tdSql.execute( + # f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + # ) + # tdSql.execute( + # f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + # ) + # tdSql.execute( + # f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + # ) + # tdSql.execute( + # f'''insert into ct1 values + # ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) + # ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) + # ''' + # ) + + # tdSql.execute( + # f'''insert into ct4 values + # ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( + # { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, + # { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000} + # ) + # ( + # { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, + # { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000} + # ) + # ''' + # ) + + # tdSql.execute( + # f'''insert into ct2 values + # ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( + # { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, + # { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + # ) + # ( + # { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, + # { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + # ) + # ''' + # ) + + # for i in range(rows): + # insert_data = f'''insert into t1 values + # ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, + # "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) + # ''' + # tdSql.execute(insert_data) + # tdSql.execute( + # f'''insert into t1 values + # ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + # ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, + # { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, + # "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + # ) + # ( + # { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, + # { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, + # "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + # ) + # ''' + # ) + + def run(self): + tdSql.prepare() + + self.all_test() + + tdDnodes.stop(1) + tdDnodes.start(1) + + # tdSql.execute("use db") + + tdLog.printNoPrefix("==========step4:after wal, all check again ") + self.all_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 8d3bad977b7ee0271c3fa268f3d288d18716da95 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 20 May 2022 20:01:17 +0800 Subject: [PATCH 02/73] fix case --- tests/system-test/0-others/fsync.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/system-test/0-others/fsync.py b/tests/system-test/0-others/fsync.py index 8ee724175b..aab8cccb3b 100644 --- a/tests/system-test/0-others/fsync.py +++ b/tests/system-test/0-others/fsync.py @@ -50,31 +50,35 @@ class TDTestCase: tdSql.execute("drop database if exists db1") tdSql.execute("create database db1 wal 1") - tdSql.query("show datsbases") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, wal_index, 1) tdSql.execute("drop database if exists db1") tdSql.execute("create database db1 wal 2") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, wal_index, 2) tdSql.execute("drop database if exists db1") tdSql.execute("create database db1 fsync 0") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, fsync_index, 0) tdSql.execute("drop database if exists db1") tdSql.execute("create database db1 fsync 3000") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, fsync_index, 3000) tdSql.execute("drop database if exists db1") tdSql.execute("create database db1 fsync 180000") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, fsync_index, 180000) @@ -82,6 +86,7 @@ class TDTestCase: tdSql.execute("drop database if exists db1") tdSql.execute("create database db1 wal 1 fsync 3000") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, fsync_index, 180000) @@ -89,42 +94,49 @@ class TDTestCase: tdSql.execute("drop database if exists db1") tdSql.execute("create database db1 wal 2 fsync 3000") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, fsync_index, 3000) tdSql.checkData(i, wal_index, 2) tdSql.execute("alter database db1 wal 1") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, fsync_index, 3000) tdSql.checkData(i, wal_index, 1) tdSql.execute("alter database db1 wal 2") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, fsync_index, 3000) tdSql.checkData(i, wal_index, 2) tdSql.execute("alter database db1 fsync 0") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, fsync_index, 0) tdSql.checkData(i, wal_index, 2) tdSql.execute("alter database db1 fsync 3000") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, fsync_index, 3000) tdSql.checkData(i, wal_index, 2) tdSql.execute("alter database db1 fsync 18000") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, fsync_index, 18000) tdSql.checkData(i, wal_index, 2) tdSql.execute("alter database db1 wal 1 fsync 3000") + tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": tdSql.checkData(i, fsync_index, 3000) From 6d09a3ed8450dcf33fc5ab2f8067478f3fd774f2 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 20 May 2022 20:04:25 +0800 Subject: [PATCH 03/73] fix case --- tests/system-test/0-others/fsync.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/fsync.py b/tests/system-test/0-others/fsync.py index aab8cccb3b..084e3854a6 100644 --- a/tests/system-test/0-others/fsync.py +++ b/tests/system-test/0-others/fsync.py @@ -42,10 +42,11 @@ class TDTestCase: wal_index = 0 fsync_index = 0 tdSql.query("show databases") + print(tdSql.cursor.description) for i in range(tdSql.queryCols): - if tdSql.cursor.description[i] == "wal": + if tdSql.cursor.description[i][0] == "wal": wal_index = i - if tdSql.cursor.description[i] == "fsync": + if tdSql.cursor.description[i][0] == "fsync": fsync_index = i tdSql.execute("drop database if exists db1") From 460e3c7a58a3d52f8ca8005d932e1b62a9bce072 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 20 May 2022 20:04:52 +0800 Subject: [PATCH 04/73] fix case --- tests/system-test/0-others/fsync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/system-test/0-others/fsync.py b/tests/system-test/0-others/fsync.py index 084e3854a6..d70d71b7f1 100644 --- a/tests/system-test/0-others/fsync.py +++ b/tests/system-test/0-others/fsync.py @@ -42,7 +42,6 @@ class TDTestCase: wal_index = 0 fsync_index = 0 tdSql.query("show databases") - print(tdSql.cursor.description) for i in range(tdSql.queryCols): if tdSql.cursor.description[i][0] == "wal": wal_index = i From c3c30034fdd4c670e96fee3ea3e51a9f7ca97389 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 20 May 2022 20:18:35 +0800 Subject: [PATCH 05/73] fix case --- tests/system-test/0-others/fsync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/fsync.py b/tests/system-test/0-others/fsync.py index d70d71b7f1..ac0b8bb58e 100644 --- a/tests/system-test/0-others/fsync.py +++ b/tests/system-test/0-others/fsync.py @@ -85,11 +85,11 @@ class TDTestCase: tdSql.execute("drop database if exists db1") - tdSql.execute("create database db1 wal 1 fsync 3000") + tdSql.execute("create database db1 wal 1 fsync 6000") tdSql.query("show databases") for i in range(tdSql.queryRows): if tdSql.queryResult[i][0] == "db1": - tdSql.checkData(i, fsync_index, 180000) + tdSql.checkData(i, fsync_index, 6000) tdSql.checkData(i, wal_index, 1) tdSql.execute("drop database if exists db1") From 00441b4e06765c9b489d120bb5a3e9755105edb3 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 20 May 2022 20:21:32 +0800 Subject: [PATCH 06/73] fix case --- tests/system-test/0-others/fsync.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/system-test/0-others/fsync.py b/tests/system-test/0-others/fsync.py index ac0b8bb58e..964550cdbc 100644 --- a/tests/system-test/0-others/fsync.py +++ b/tests/system-test/0-others/fsync.py @@ -142,6 +142,8 @@ class TDTestCase: tdSql.checkData(i, fsync_index, 3000) tdSql.checkData(i, wal_index, 1) + tdSql.execute("drop database db1 ") + @property def fsync_create_err(self): return [ From 8ad2efeedf1028b8281453ceff53d42abcfab142 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 20 May 2022 20:22:51 +0800 Subject: [PATCH 07/73] add fsync to CI --- tests/system-test/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 06be29a636..527cc1f8a3 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -11,6 +11,7 @@ python3 ./test.py -f 0-others/udfTest.py # TODO privilege has error # python3 ./test.py -f 0-others/user_control.py +python3 ./test.py -f 0-others/fsync.py #python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py From 9a6710b465b16f97141f7327dc6bfa0cc30d519a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 May 2022 23:39:52 +0800 Subject: [PATCH 08/73] enh: add invert index to vnode --- cmake/cmake.options | 1 + source/dnode/vnode/CMakeLists.txt | 7 ++- source/dnode/vnode/src/inc/meta.h | 25 ++++++----- source/dnode/vnode/src/meta/metaIdx.c | 53 +++++++++++----------- source/dnode/vnode/src/meta/metaOpen.c | 21 +++++++++ source/dnode/vnode/src/meta/metaTable.c | 59 +++++++++++++++---------- source/libs/index/inc/indexInt.h | 44 ++++-------------- source/libs/index/src/indexCache.c | 3 -- source/libs/index/src/indexComm.c | 25 ++++++++++- 9 files changed, 137 insertions(+), 101 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index d83ab49fd5..5f394c7426 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -142,3 +142,4 @@ option( "If use invertedIndex" ON ) + diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a8e3860ed1..b10bee4732 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -76,9 +76,14 @@ target_link_libraries( #PUBLIC scalar PUBLIC transport PUBLIC stream + PUBLIC index ) target_compile_definitions(vnode PUBLIC -DMETA_REFACT) - +if (${BUILD_WITH_INVERTEDINDEX}) + add_definitions(-DUSE_INVERTED_INDEX) +endif(${BUILD_WITH_INVERTEDINDEX}) if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) + + diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 693f4a0a2b..3340bbb91c 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -17,6 +17,7 @@ #define _TD_VNODE_META_H_ #include "vnodeInt.h" +#include "index.h" #ifdef __cplusplus extern "C" { @@ -61,16 +62,20 @@ static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64() struct SMeta { TdThreadRwlock lock; - char* path; - SVnode* pVnode; - TDB* pEnv; - TXN txn; - TTB* pTbDb; - TTB* pSkmDb; - TTB* pUidIdx; - TTB* pNameIdx; - TTB* pCtbIdx; - TTB* pTagIdx; + char* path; + SVnode* pVnode; + TDB* pEnv; + TXN txn; + TTB* pTbDb; + TTB* pSkmDb; + TTB* pUidIdx; + TTB* pNameIdx; + TTB* pCtbIdx; +#ifdef USE_INVERTED_INDEX + void* pTagIvtIdx; +#else + TTB* pTagIdx; +#endif TTB* pTtlIdx; TTB* pSmaIdx; SMetaIdx* pIdx; diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 3f52071315..efa06d2d1f 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -53,10 +53,10 @@ int metaOpenIdx(SMeta *pMeta) { #endif #ifdef USE_INVERTED_INDEX - SIndexOpts opts; - if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { - return -1; - } + // SIndexOpts opts; + // if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) { + // return -1; + //} #endif return 0; @@ -71,36 +71,37 @@ void metaCloseIdx(SMeta *pMeta) { /* TODO */ #endif #ifdef USE_INVERTED_INDEX - SIndexOpts opts; - if (indexClose(pMeta->pIdx->pIdx) != 0) { - return -1; - } + // SIndexOpts opts; + // if (indexClose(pMeta->pIdx->pIdx) != 0) { + // return -1; + //} + // return 0; #endif } int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) { #ifdef USE_INVERTED_INDEX - if (pTbCfgs->type == META_CHILD_TABLE) { - char buf[8] = {0}; - int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId; - sprintf(buf, "%d", colId); // colname + // if (pTbCfgs->type == META_CHILD_TABLE) { + // char buf[8] = {0}; + // int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId; + // sprintf(buf, "%d", colId); // colname - char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId); + // char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId); - tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id - tb_uid_t tuid = 0; // child table uid - SIndexMultiTerm *terms = indexMultiTermCreate(); - SIndexTerm *term = - indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid); - indexMultiTermAdd(terms, term); + // tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id + // tb_uid_t tuid = 0; // child table uid + // SIndexMultiTerm *terms = indexMultiTermCreate(); + // SIndexTerm *term = + // indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid); + // indexMultiTermAdd(terms, term); - int ret = indexPut(pMeta->pIdx->pIdx, terms); - indexMultiTermDestroy(terms); - return ret; - } else { - return DB_DONOTINDEX; - } + // int ret = indexPut(pMeta->pIdx->pIdx, terms); + // indexMultiTermDestroy(terms); + // return ret; + //} else { + // return DB_DONOTINDEX; + //} #endif // TODO return 0; @@ -112,4 +113,4 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) { #endif // TODO return 0; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 9a97357b97..f23e7f8805 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -93,11 +93,24 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { } // open pTagIdx +#ifdef USE_INVERTED_INDEX + // TODO(yihaoDeng), refactor later + char indexFullPath[128] = {0}; + sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); + taosMkDir(indexFullPath); + ret = indexOpen(indexOptsCreate(), indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); + if (ret < 0) { + metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + +#else ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); if (ret < 0) { metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } +#endif // open pTtlIdx ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); @@ -128,7 +141,11 @@ _err: if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); +#ifdef USE_INVERTED_INDEX + if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); +#else if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); +#endif if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); @@ -145,7 +162,11 @@ int metaClose(SMeta *pMeta) { if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); +#ifdef USE_INVERTED_INDEX + if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); +#else if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); +#endif if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 2bcfd01904..c0eeeaedfa 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -30,9 +30,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int vLen = 0; const void *pKey = NULL; const void *pVal = NULL; - void *pBuf = NULL; + void * pBuf = NULL; int32_t szBuf = 0; - void *p = NULL; + void * p = NULL; SMetaReader mr = {0}; // validate req @@ -71,9 +71,9 @@ _err: } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - TBC *pNameIdxc = NULL; - TBC *pUidIdxc = NULL; - TBC *pCtbIdxc = NULL; + TBC * pNameIdxc = NULL; + TBC * pUidIdxc = NULL; + TBC * pCtbIdxc = NULL; SCtbIdxKey *pCtbIdxKey; const void *pKey = NULL; int nKey; @@ -134,8 +134,8 @@ _err: int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; SMetaEntry nStbEntry = {0}; - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; + TBC * pUidIdxc = NULL; + TBC * pTbDbc = NULL; const void *pData; int nData; int64_t oversion; @@ -256,9 +256,9 @@ _err: } int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) { - TBC *pTbDbc = NULL; - TBC *pUidIdxc = NULL; - TBC *pNameIdxc = NULL; + TBC * pTbDbc = NULL; + TBC * pUidIdxc = NULL; + TBC * pNameIdxc = NULL; const void *pData; int nData; tb_uid_t uid; @@ -377,14 +377,14 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi } static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - void *pVal = NULL; + void * pVal = NULL; int nVal = 0; - const void *pData = NULL; + const void * pData = NULL; int nData = 0; int ret = 0; tb_uid_t uid; int64_t oversion; - SSchema *pColumn = NULL; + SSchema * pColumn = NULL; SMetaEntry entry = {0}; SSchemaWrapper *pSchema; int c; @@ -530,7 +530,7 @@ _err: static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { SMetaEntry ctbEntry = {0}; SMetaEntry stbEntry = {0}; - void *pVal = NULL; + void * pVal = NULL; int nVal = 0; int ret; int c; @@ -561,7 +561,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA oversion = *(int64_t *)pData; // search table.db - TBC *pTbDbc = NULL; + TBC * pTbDbc = NULL; SDecoder dc = {0}; /* get ctbEntry */ @@ -586,7 +586,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tDecoderClear(&dc); SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - SSchema *pColumn = NULL; + SSchema * pColumn = NULL; int32_t iCol = 0; for (;;) { pColumn = NULL; @@ -678,8 +678,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { STbDbKey tbDbKey; - void *pKey = NULL; - void *pVal = NULL; + void * pKey = NULL; + void * pVal = NULL; int kLen = 0; int vLen = 0; SEncoder coder = {0}; @@ -794,14 +794,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { } static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { - void *pData = NULL; + void * pData = NULL; int nData = 0; STbDbKey tbDbKey = {0}; SMetaEntry stbEntry = {0}; - STagIdxKey *pTagIdxKey = NULL; + STagIdxKey * pTagIdxKey = NULL; int32_t nTagIdxKey; const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; - const void *pTagData = NULL; // + const void * pTagData = NULL; // SDecoder dc = {0}; // get super table @@ -817,22 +817,33 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { pTagData = tdGetKVRowValOfCol((const SKVRow)pCtbEntry->ctbEntry.pTags, pTagColumn->colId); // update tag index +#ifdef USE_INVERTED_INDEX + tb_uid_t suid = pCtbEntry->ctbEntry.suid; + tb_uid_t tuid = pCtbEntry->uid; + + SIndexMultiTerm *tmGroup = indexMultiTermCreate(); + + SIndexTerm *tm = indexTermCreate(suid, ADD_VALUE, pTagColumn->type, pTagColumn->name, sizeof(pTagColumn->name), + pTagData, pTagData == NULL ? 0 : strlen(pTagData)); + indexMultiTermAdd(tmGroup, tm); + int ret = indexPut((SIndex *)pMeta->pTagIvtIdx, tmGroup, tuid); + indexMultiTermDestroy(tmGroup); +#else if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, pTagColumn->type, pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { return -1; } tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); - +#endif tDecoderClear(&dc); tdbFree(pData); - return 0; } static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { SEncoder coder = {0}; - void *pVal = NULL; + void * pVal = NULL; int vLen = 0; int rcode = 0; SSkmDbKey skmDbKey = {0}; diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 27c380beaf..3c05f38d05 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -131,42 +131,14 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); -#define indexFatal(...) \ - do { \ - if (sDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("index FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ - } \ - } while (0) -#define indexError(...) \ - do { \ - if (sDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("index ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ - } \ - } while (0) -#define indexWarn(...) \ - do { \ - if (sDebugFlag & DEBUG_WARN) { \ - taosPrintLog("index WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ - } \ - } while (0) -#define indexInfo(...) \ - do { \ - if (sDebugFlag & DEBUG_INFO) { \ - taosPrintLog("index ", DEBUG_INFO, 255, __VA_ARGS__); \ - } \ - } while (0) -#define indexDebug(...) \ - do { \ - if (sDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("index ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__); \ - } \ - } while (0) -#define indexTrace(...) \ - do { \ - if (sDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("index ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__); \ - } \ - } while (0) +// clang-format off +#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while (0) +#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while (0) +#define indexWarn(...) do { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while (0) +#define indexInfo(...) do { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", DEBUG_INFO, 255, __VA_ARGS__); } } while (0) +#define indexDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__);} } while (0) +#define indexTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__);} } while (0) +// clang-format on #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 9a2e487df1..65986199d4 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -709,9 +709,6 @@ static bool indexCacheIteratorNext(Iterate* itera) { iv->type = ct->operaType; iv->ver = ct->version; iv->colVal = tstrdup(ct->colVal); - // printf("col Val: %s\n", iv->colVal); - // iv->colType = cv->colType; - taosArrayPush(iv->val, &ct->uid); } return next; diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 4c23e4ba4b..78c7babb68 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -22,6 +22,29 @@ #include "ttypes.h" #include "tvariant.h" +#define INDEX_DATA_BOOL_NULL 0x02 +#define INDEX_DATA_TINYINT_NULL 0x80 +#define INDEX_DATA_SMALLINT_NULL 0x8000 +#define INDEX_DATA_INT_NULL 0x80000000L +#define INDEX_DATA_BIGINT_NULL 0x8000000000000000L +#define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL + +#define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF +#define INDEX_DATA_BINARY_NULL 0xFF +#define INDEX_DATA_JSON_NULL 0xFFFFFFFF +#define INDEX_DATA_JSON_null 0xFFFFFFFE +#define INDEX_DATA_JSON_NOT_NULL 0x01 + +#define INDEX_DATA_UTINYINT_NULL 0xFF +#define INDEX_DATA_USMALLINT_NULL 0xFFFF +#define INDEX_DATA_UINT_NULL 0xFFFFFFFF +#define INDEX_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL + +#define INDEX_DATA_NULL_STR "NULL" +#define INDEX_DATA_NULL_STR_L "null" + char JSON_COLUMN[] = "JSON"; char JSON_VALUE_DELIM = '&'; @@ -372,7 +395,7 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); - *dst = (char*) * dst - tlen; + *dst = (char*)*dst - tlen; break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY From 92e2d9612f6de1ce3c0490b6db7a73bf228369ed Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 15:54:05 +0800 Subject: [PATCH 09/73] refactor: free rpcCont on error occured --- include/util/taoserror.h | 4 +- source/dnode/mgmt/node_mgmt/src/dmProc.c | 24 ++++--- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 69 +++++-------------- source/dnode/mgmt/node_util/inc/dmUtil.h | 1 + source/dnode/mgmt/node_util/src/dmEps.c | 11 +++ source/dnode/mnode/impl/src/mndTrans.c | 3 - 6 files changed, 48 insertions(+), 64 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a89808852d..e28618d940 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -282,8 +282,8 @@ int32_t* taosGetErrno(); // dnode #define TSDB_CODE_NODE_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0400) #define TSDB_CODE_NODE_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0401) -#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0403) -#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0404) +#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0402) +#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0403) // vnode #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index d044ad3b2e..72878d0d85 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -142,13 +142,14 @@ static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg queue->tail = headLen + bodyLen; } else if (remain < 8 + headLen) { memcpy(queue->pBuffer + queue->tail + 8, pHead, remain - 8); - memcpy(queue->pBuffer, (char*)pHead + remain - 8, rawHeadLen - (remain - 8)); + memcpy(queue->pBuffer, (char *)pHead + remain - 8, rawHeadLen - (remain - 8)); if (rawBodyLen > 0) memcpy(queue->pBuffer + headLen - (remain - 8), pBody, rawBodyLen); queue->tail = headLen - (remain - 8) + bodyLen; } else if (remain < 8 + headLen + bodyLen) { memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen); if (rawBodyLen > 0) memcpy(queue->pBuffer + queue->tail + 8 + headLen, pBody, remain - 8 - headLen); - if (rawBodyLen > 0) memcpy(queue->pBuffer, (char*)pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); + if (rawBodyLen > 0) + memcpy(queue->pBuffer, (char *)pBody + remain - 8 - headLen, rawBodyLen - (remain - 8 - headLen)); queue->tail = bodyLen - (remain - 8 - headLen); } else { memcpy(queue->pBuffer + queue->tail + 8, pHead, rawHeadLen); @@ -312,12 +313,7 @@ static void *dmConsumChildQueue(void *param) { code = dmProcessNodeMsg(pWrapper, pMsg); if (code != 0) { dError("node:%s, failed to process msg:%p since %s, put into pqueue", proc->name, pMsg, terrstr()); - SRpcMsg rsp = { - .code = (terrno != 0 ? terrno : code), - .pCont = pMsg->info.rsp, - .contLen = pMsg->info.rspLen, - .info = pMsg->info, - }; + SRpcMsg rsp = {.code = (terrno != 0 ? terrno : code), .info = pMsg->info}; dmPutToProcPQueue(proc, &rsp, DND_FUNC_RSP); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); @@ -469,8 +465,18 @@ void dmPutToProcPQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype) { taosMsleep(retry); } } + + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + pMsg->contLen = 0; } int32_t dmPutToProcCQueue(SProc *proc, SRpcMsg *pMsg, EProcFuncType ftype) { - return dmPushToProcQueue(proc, proc->cqueue, pMsg, ftype); + int32_t code = dmPushToProcQueue(proc, proc->cqueue, pMsg, ftype); + if (code == 0) { + dTrace("msg:%p, is freed after push to cqueue", pMsg); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + } + return code; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 3dc0572927..b5fb0e1d7a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -16,6 +16,9 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" +static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet); +static inline void dmSendRsp(SRpcMsg *pMsg); + static inline int32_t dmBuildNodeMsg(SRpcMsg *pMsg, SRpcMsg *pRpc) { SRpcConnInfo connInfo = {0}; if (IsReq(pRpc) && rpcGetConnInfo(pRpc->info.handle, &connInfo) != 0) { @@ -122,27 +125,28 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { } _OVER: - if (code == 0) { - if (pWrapper != NULL && InParentProc(pWrapper)) { - dTrace("msg:%p, is freed after push to cqueue", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pRpc->pCont); - } - } else { + if (code != 0) { dError("msg:%p, failed to process since %s", pMsg, terrstr()); if (terrno != 0) code = terrno; if (IsReq(pRpc)) { - if (code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_NODE_OFFLINE) { - if (pRpc->msgType > TDMT_MND_MSG && pRpc->msgType < TDMT_VND_MSG) { - code = TSDB_CODE_NODE_REDIRECT; + if ((code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) && pRpc->msgType > TDMT_MND_MSG && + pRpc->msgType < TDMT_VND_MSG) { + SEpSet epset = {0}; + dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &epset); + SRpcMsg rsp = {.code = TSDB_CODE_NODE_REDIRECT, .info = pRpc->info}; + if (pWrapper != NULL) { + dmSendRedirectRsp(&rsp, &epset); + } else { + rpcSendRedirectRsp(&rsp, &epset); } - } - SRpcMsg rspMsg = {.code = code, .info = pRpc->info}; - if (pWrapper != NULL) { - tmsgSendRsp(&rspMsg); } else { - rpcSendResponse(&rspMsg); + SRpcMsg rsp = {.code = code, .info = pRpc->info}; + if (pWrapper != NULL) { + dmSendRsp(&rsp); + } else { + rpcSendResponse(&rsp); + } } } @@ -198,8 +202,6 @@ static inline void dmSendRsp(SRpcMsg *pMsg) { SMgmtWrapper *pWrapper = pMsg->info.wrapper; if (InChildProc(pWrapper)) { dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_RSP); - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; } else { rpcSendResponse(pMsg); } @@ -226,8 +228,6 @@ static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { SMgmtWrapper *pWrapper = pMsg->info.wrapper; if (InChildProc(pWrapper)) { dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_REGIST); - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; } else { rpcRegisterBrokenLinkArg(pMsg); } @@ -322,34 +322,3 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) { }; return msgCb; } - -static void dmSendMnodeRedirectRsp(SRpcMsg *pMsg) { - SDnode *pDnode = dmInstance(); - SEpSet epSet = {0}; - dmGetMnodeEpSet(&pDnode->data, &epSet); - - dDebug("msg:%p, is redirected, num:%d use:%d", pMsg, epSet.numOfEps, epSet.inUse); - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); - if (strcmp(epSet.eps[i].fqdn, tsLocalFqdn) == 0 && epSet.eps[i].port == tsServerPort) { - epSet.inUse = (i + 1) % epSet.numOfEps; - } - - epSet.eps[i].port = htons(epSet.eps[i].port); - } - - SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; - SMEpSet msg = {.epSet = epSet}; - int32_t contLen = tSerializeSMEpSet(NULL, 0, &msg); - rsp.pCont = rpcMallocCont(contLen); - if (rsp.pCont == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - } else { - tSerializeSMEpSet(rsp.pCont, contLen, &msg); - rsp.contLen = contLen; - } - - dmSendRsp(&rsp); - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; -} diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index e7256a3a87..4946669678 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -173,6 +173,7 @@ int32_t dmReadEps(SDnodeData *pData); int32_t dmWriteEps(SDnodeData *pData); void dmUpdateEps(SDnodeData *pData, SArray *pDnodeEps); void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); +void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet); void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); #ifdef __cplusplus diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 94fa569557..e0af20e41b 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -314,6 +314,17 @@ void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) { taosThreadRwlockUnlock(&pData->lock); } +void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet) { + dmGetMnodeEpSet(pData, pEpSet); + dDebug("msg:%p, is redirected, num:%d use:%d", pMsg, pEpSet->numOfEps, pEpSet->inUse); + for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { + dDebug("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + if (strcmp(pEpSet->eps[i].fqdn, tsLocalFqdn) == 0 && pEpSet->eps[i].port == tsServerPort) { + pEpSet->inUse = (i + 1) % pEpSet->numOfEps; + } + } +} + void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) { taosThreadRwlockWrlock(&pData->lock); pData->mnodeEps = *pEpSet; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 35ecaa748e..6210fe3fcf 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -994,9 +994,6 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgSent = 0; pAction->msgReceived = 0; pAction->errCode = terrno; - if (terrno == TSDB_CODE_INVALID_PTR || terrno == TSDB_CODE_NODE_OFFLINE) { - rpcFreeCont(rpcMsg.pCont); - } mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; } From 733d2216997365cf487e2f875960d79d0cffb3c0 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 21 May 2022 16:04:25 +0800 Subject: [PATCH 10/73] fix stmt bug --- include/libs/qcom/query.h | 2 ++ source/client/src/clientImpl.c | 25 +++++++++++++++++++++++++ source/client/src/clientStmt.c | 10 ++++++++-- source/libs/catalog/src/catalog.c | 5 +++++ source/libs/parser/src/parTranslater.c | 1 + source/libs/qworker/src/qworker.c | 6 +++++- tests/script/api/batchprepare.c | 20 ++++++++++++-------- tests/system-test/0-others/udfTest.py | 2 +- 8 files changed, 59 insertions(+), 12 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 389fe25780..bb57946082 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -198,6 +198,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_HANDLE_ERROR(_code) \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) +#define NEED_CLIENT_RM_TBLMETA_REQ(_type) ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB \ + || (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB) #define NEED_SCHEDULER_RETRY_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 386283b5b5..8a03043a99 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -345,6 +345,10 @@ int32_t validateSversion(SRequestObj* pRequest, void* res) { for (int32_t i = 0; i < pRsp->nBlocks; ++i) { SSubmitBlkRsp* blk = pRsp->pBlocks + i; + if (NULL == blk->tblFName || 0 == blk->tblFName[0]) { + continue; + } + STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; taosArrayPush(pArray, &tbSver); } @@ -499,6 +503,23 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { return code; } +int32_t removeMeta(STscObj* pTscObj, SArray* tbList) { + SCatalog* pCatalog = NULL; + int32_t tbNum = taosArrayGetSize(tbList); + int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + for (int32_t i = 0; i < tbNum; ++i) { + SName* pTbName = taosArrayGet(tbList, i); + catalogRemoveTableMeta(pCatalog, pTbName); + } + + return TSDB_CODE_SUCCESS; +} + + SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { SRequestObj* pRequest = NULL; int32_t retryNum = 0; @@ -518,6 +539,10 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { } } while (retryNum++ < REQUEST_MAX_TRY_TIMES); + if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) { + removeMeta(pTscObj, pRequest->tableList); + } + return pRequest; } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 764571e71e..a9b4e234e8 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -47,8 +47,14 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { } break; case STMT_EXECUTE: - if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) { - code = TSDB_CODE_TSC_STMT_API_ERROR; + if (STMT_TYPE_QUERY == pStmt->sql.type) { + if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS) && STMT_STATUS_NE(BIND) && STMT_STATUS_NE(BIND_COL)) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } + } else { + if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) { + code = TSDB_CODE_TSC_STMT_API_ERROR; + } } break; default: diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d5301e3624..85d9b14c60 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2800,6 +2800,7 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { CTG_ERR_JRET(ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, true)); } + ctgDebug("table meta %s.%s removed", dbFName, pTableName->tname); _return: @@ -2974,6 +2975,10 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm int32_t tbNum = taosArrayGetSize(pTables); for (int32_t i = 0; i < tbNum; ++i) { STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); + if (NULL == pTb->tbFName || 0 == pTb->tbFName[0]) { + continue; + } + tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (CTG_IS_SYS_DBNAME(name.dbname)) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f6d53dd15a..bc16bd89e6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2726,6 +2726,7 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm SName tableName; tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); + collectUseTable(&tableName, pCxt->pTables); return buildRollupAst(pCxt, pStmt, pReq); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index cb7ff08fa3..5b358fb008 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -725,7 +725,11 @@ void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx) { qGetQueriedTableSchemaVersion(pTaskInfo, dbFName, tbName, &ctx->tbInfo.sversion, &ctx->tbInfo.tversion); - sprintf(ctx->tbInfo.tbFName, "%s.%s", dbFName, tbName); + if (dbFName[0] && tbName[0]) { + sprintf(ctx->tbInfo.tbFName, "%s.%s", dbFName, tbName); + } else { + ctx->tbInfo.tbFName[0] = 0; + } } int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index ba1cd00fcb..2ded58a979 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -1825,10 +1825,12 @@ int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) { if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { exit(1); } - - if (taos_stmt_add_batch(stmt)) { - printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); - exit(1); + + if (rand() % 2 == 0) { + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } } if (taos_stmt_execute(stmt) != 0) { @@ -1871,10 +1873,12 @@ int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) { if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { exit(1); } - - if (taos_stmt_add_batch(stmt)) { - printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); - exit(1); + + if (rand() % 2 == 0) { + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } } if (taos_stmt_execute(stmt) != 0) { diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index af3245df3d..679b415098 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) From c7e7694a55d55476c89bd4282c23c4d07aa1bb0c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 21 May 2022 16:21:56 +0800 Subject: [PATCH 11/73] fea: add null tag support in schemaless --- source/client/src/clientSml.c | 16 +++++++--------- source/client/test/smlTest.cpp | 3 ++- source/common/src/tdataformat.c | 8 +++++--- source/common/src/tname.c | 7 ++----- source/libs/parser/src/parInsert.c | 6 +++--- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index e884748fff..61282ae76a 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -263,7 +263,7 @@ static int32_t smlBuildColumnDescription(SSmlKv* field, char* buf, int32_t bufSi memcpy(tname, field->key, field->keyLen); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { int32_t bytes = field->length > CHAR_SAVE_LENGTH ? (2*field->length) : CHAR_SAVE_LENGTH; - int out = snprintf(buf, bufSize,"`%s` %s(%d)", + int out = snprintf(buf, bufSize, "`%s` %s(%d)", tname, tDataTypes[field->type].name, bytes); *outBytes = out; } else { @@ -400,6 +400,12 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; } + if(taosArrayGetSize(cols) == 0){ + outBytes = snprintf(pos, freeBytes,"`%s` %s(%d)", + TAG, tDataTypes[TSDB_DATA_TYPE_NCHAR].name, CHAR_SAVE_LENGTH); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } pos--; ++freeBytes; outBytes = snprintf(pos, freeBytes, ")"); TAOS_RES* res = taos_query(info->taos, result); @@ -1112,14 +1118,6 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, char *childTableName, bool isTag, SHashObj *dumplicateKey, SSmlMsgBuf *msg){ if(isTag && len == 0){ - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if(!kv) return TSDB_CODE_OUT_OF_MEMORY; - kv->key = TAG; - kv->keyLen = TAG_LEN; - kv->value = TAG_VALUE; - kv->length = TAG_VALUE_LEN; - kv->type = TSDB_DATA_TYPE_NCHAR; - if(cols) taosArrayPush(cols, &kv); return TSDB_CODE_SUCCESS; } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index c7935b351c..2165189153 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1207,7 +1207,8 @@ TEST(testCase, sml_TD15662_Test) { ASSERT_NE(info, nullptr); const char *sql[] = { - "hetrey,id=sub_table_0123456,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64", + "hetrey c0=f,c1=127i8 1626006833639", + "hetrey,t1=r c0=f,c1=127i8 1626006833640", }; int ret = smlProcess(info, (char **)sql, sizeof(sql) / sizeof(sql[0])); ASSERT_EQ(ret, 0); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 8aa8ed2f14..fe112ba67e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1077,7 +1077,7 @@ void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) { SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { int tlen = sizeof(SColIdx) * pBuilder->nCols + pBuilder->size; - if (tlen == 0) return NULL; + // if (tlen == 0) return NULL; // nCols == 0 means no tags tlen += TD_KV_ROW_HEAD_SIZE; @@ -1087,8 +1087,10 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { kvRowSetNCols(row, pBuilder->nCols); kvRowSetLen(row, tlen); - memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols); - memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); + if(pBuilder->nCols > 0){ + memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols); + memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); + } return row; } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 0764ea84b9..104dee261c 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -308,13 +308,10 @@ static int compareKv(const void* p1, const void* p2) { * use stable name and tags to grearate child table name */ void buildChildTableName(RandTableName* rName) { - int32_t size = taosArrayGetSize(rName->tags); - ASSERT(size > 0); - taosArraySort(rName->tags, compareKv); - SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, rName->sTableName, rName->sTableNameLen); - for (int j = 0; j < size; ++j) { + taosArraySort(rName->tags, compareKv); + for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { SSmlKv* tagKv = taosArrayGetP(rName->tags, j); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); if(IS_VAR_DATA_TYPE(tagKv->type)){ diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index d1b4a745b9..795bed3a05 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -818,7 +818,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); if (NULL == row) { - return buildInvalidOperationMsg(&pCxt->msg, "tag value expected"); + return buildInvalidOperationMsg(&pCxt->msg, "out of memory"); } tdSortKVRowByColIdx(row); @@ -1333,7 +1333,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN SKVRow row = tdGetKVRowFromBuilder(&tagBuilder); if (NULL == row) { tdDestroyKVRowBuilder(&tagBuilder); - return buildInvalidOperationMsg(&pBuf, "tag value expected"); + return buildInvalidOperationMsg(&pBuf, "out of memory"); } tdSortKVRowByColIdx(row); @@ -1682,7 +1682,7 @@ static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedD *row = tdGetKVRowFromBuilder(tagsBuilder); if (*row == NULL) { - return TSDB_CODE_SML_INVALID_DATA; + return TSDB_CODE_OUT_OF_MEMORY; } tdSortKVRowByColIdx(*row); return TSDB_CODE_SUCCESS; From 44d9cc8d57521e051c3602bbfb99835b86bc6552 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 16:24:24 +0800 Subject: [PATCH 12/73] refactor: free rpcCont on error occured --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 41 +++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index b5fb0e1d7a..8dfcb798ac 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -16,8 +16,9 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" -static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet); -static inline void dmSendRsp(SRpcMsg *pMsg); +static void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet); +static void dmSendRsp(SRpcMsg *pMsg); +static void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg); static inline int32_t dmBuildNodeMsg(SRpcMsg *pMsg, SRpcMsg *pRpc) { SRpcConnInfo connInfo = {0}; @@ -130,23 +131,17 @@ _OVER: if (terrno != 0) code = terrno; if (IsReq(pRpc)) { + SRpcMsg rsp = {.code = code, .info = pRpc->info}; + if ((code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) && pRpc->msgType > TDMT_MND_MSG && pRpc->msgType < TDMT_VND_MSG) { - SEpSet epset = {0}; - dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &epset); - SRpcMsg rsp = {.code = TSDB_CODE_NODE_REDIRECT, .info = pRpc->info}; - if (pWrapper != NULL) { - dmSendRedirectRsp(&rsp, &epset); - } else { - rpcSendRedirectRsp(&rsp, &epset); - } + dmBuildMnodeRedirectRsp(pDnode, &rsp); + } + + if (pWrapper != NULL) { + dmSendRsp(&rsp); } else { - SRpcMsg rsp = {.code = code, .info = pRpc->info}; - if (pWrapper != NULL) { - dmSendRsp(&rsp); - } else { - rpcSendResponse(&rsp); - } + rpcSendResponse(&rsp); } } @@ -207,6 +202,20 @@ static inline void dmSendRsp(SRpcMsg *pMsg) { } } +static void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { + SMEpSet msg = {0}; + dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &msg.epSet); + + int32_t contLen = tSerializeSMEpSet(NULL, 0, &msg); + pMsg->pCont = rpcMallocCont(contLen); + if (pMsg->pCont == NULL) { + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + tSerializeSMEpSet(pMsg->pCont, contLen, &msg); + pMsg->contLen = contLen; + } +} + static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) { SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; SMEpSet msg = {.epSet = *pNewEpSet}; From 62ecc87c3ec4e8d5a85182857d2c78173f7b80c2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 21 May 2022 16:24:41 +0800 Subject: [PATCH 13/73] fix: memory error --- include/common/tmsg.h | 4 +- source/libs/catalog/src/catalog.c | 1136 ++++++++++++++--------------- source/util/src/thash.c | 145 ++-- 3 files changed, 635 insertions(+), 650 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9104e8a423..bd3ca6ae8d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1352,7 +1352,7 @@ typedef struct { typedef struct { int32_t code; - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t sversion; int32_t tversion; } SResReadyRsp; @@ -2524,7 +2524,7 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); buf = taosDecodeFixedI32(buf, &pRsp->blockNum); pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); if (pRsp->blockNum != 0) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d5301e3624..d3edc90e9c 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "trpc.h" -#include "query.h" -#include "tname.h" #include "catalogInt.h" +#include "query.h" #include "systable.h" +#include "tname.h" +#include "trpc.h" int32_t ctgActUpdateVg(SCtgMetaAction *action); int32_t ctgActUpdateTbl(SCtgMetaAction *action); @@ -27,38 +27,11 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action); int32_t ctgActUpdateUser(SCtgMetaAction *action); extern SCtgDebug gCTGDebug; -SCatalogMgmt gCtgMgmt = {0}; -SCtgAction gCtgAction[CTG_ACT_MAX] = {{ - CTG_ACT_UPDATE_VG, - "update vgInfo", - ctgActUpdateVg - }, - { - CTG_ACT_UPDATE_TBL, - "update tbMeta", - ctgActUpdateTbl - }, - { - CTG_ACT_REMOVE_DB, - "remove DB", - ctgActRemoveDB - }, - { - CTG_ACT_REMOVE_STB, - "remove stbMeta", - ctgActRemoveStb - }, - { - CTG_ACT_REMOVE_TBL, - "remove tbMeta", - ctgActRemoveTbl - }, - { - CTG_ACT_UPDATE_USER, - "update user", - ctgActUpdateUser - } -}; +SCatalogMgmt gCtgMgmt = {0}; +SCtgAction gCtgAction[CTG_ACT_MAX] = { + {CTG_ACT_UPDATE_VG, "update vgInfo", ctgActUpdateVg}, {CTG_ACT_UPDATE_TBL, "update tbMeta", ctgActUpdateTbl}, + {CTG_ACT_REMOVE_DB, "remove DB", ctgActRemoveDB}, {CTG_ACT_REMOVE_STB, "remove stbMeta", ctgActRemoveStb}, + {CTG_ACT_REMOVE_TBL, "remove tbMeta", ctgActRemoveTbl}, {CTG_ACT_UPDATE_USER, "update user", ctgActUpdateUser}}; void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { if (NULL == mgmt->slots) { @@ -76,11 +49,10 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { taosMemoryFreeClear(mgmt->slots); } - void ctgFreeTableMetaCache(SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { - int32_t stblNum = taosHashGetSize(cache->stbCache); + int32_t stblNum = taosHashGetSize(cache->stbCache); taosHashCleanup(cache->stbCache); cache->stbCache = NULL; CTG_CACHE_STAT_SUB(stblNum, stblNum); @@ -106,7 +78,7 @@ void ctgFreeVgInfo(SDBVgInfo *vgInfo) { taosHashCleanup(vgInfo->vgHash); vgInfo->vgHash = NULL; } - + taosMemoryFreeClear(vgInfo); } @@ -116,7 +88,7 @@ void ctgFreeDbCache(SCtgDBCache *dbCache) { } CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - ctgFreeVgInfo (dbCache->vgInfo); + ctgFreeVgInfo(dbCache->vgInfo); CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); ctgFreeTableMetaCache(&dbCache->tbCache); @@ -128,13 +100,13 @@ void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) { taosHashCleanup(userCache->writeDbs); } -void ctgFreeHandle(SCatalog* pCtg) { +void ctgFreeHandle(SCatalog *pCtg) { ctgFreeMetaRent(&pCtg->dbRent); ctgFreeMetaRent(&pCtg->stbRent); - + if (pCtg->dbCache) { int32_t dbNum = taosHashGetSize(pCtg->dbCache); - + void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; @@ -142,42 +114,40 @@ void ctgFreeHandle(SCatalog* pCtg) { atomic_store_8(&dbCache->deleted, 1); ctgFreeDbCache(dbCache); - + pIter = taosHashIterate(pCtg->dbCache, pIter); - } + } taosHashCleanup(pCtg->dbCache); - + CTG_CACHE_STAT_SUB(dbNum, dbNum); } if (pCtg->userCache) { int32_t userNum = taosHashGetSize(pCtg->userCache); - + void *pIter = taosHashIterate(pCtg->userCache, NULL); while (pIter) { SCtgUserAuth *userCache = pIter; ctgFreeSCtgUserAuth(userCache); - + pIter = taosHashIterate(pCtg->userCache, pIter); - } + } taosHashCleanup(pCtg->userCache); - + CTG_CACHE_STAT_SUB(userNum, userNum); } - + taosMemoryFree(pCtg); } - - void ctgWaitAction(SCtgMetaAction *action) { while (true) { tsem_wait(&gCtgMgmt.queue.rspSem); - - if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + + if (atomic_load_8((int8_t *)&gCtgMgmt.exit)) { tsem_post(&gCtgMgmt.queue.rspSem); break; } @@ -193,19 +163,18 @@ void ctgWaitAction(SCtgMetaAction *action) { void ctgPopAction(SCtgMetaAction **action) { SCtgQNode *orig = gCtgMgmt.queue.head; - + SCtgQNode *node = gCtgMgmt.queue.head->next; gCtgMgmt.queue.head = gCtgMgmt.queue.head->next; CTG_QUEUE_SUB(); - + taosMemoryFreeClear(orig); *action = &node->action; } - -int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { +int32_t ctgPushAction(SCatalog *pCtg, SCtgMetaAction *action) { SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); @@ -213,7 +182,7 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { } action->seqId = atomic_add_fetch_64(&gCtgMgmt.queue.seqId, 1); - + node->action = *action; CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); @@ -235,10 +204,9 @@ int32_t ctgPushAction(SCatalog* pCtg, SCtgMetaAction *action) { return TSDB_CODE_SUCCESS; } - -int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_DB}; +int32_t ctgPushRmDBMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_REMOVE_DB}; SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); @@ -266,10 +234,10 @@ _return: CTG_RET(code); } - -int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq}; +int32_t ctgPushRmStbMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, + bool syncReq) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq}; SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); @@ -294,11 +262,9 @@ _return: CTG_RET(code); } - - -int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq}; +int32_t ctgPushRmTblMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq}; SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); @@ -322,9 +288,9 @@ _return: CTG_RET(code); } -int32_t ctgPushUpdateVgMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq}; +int32_t ctgPushUpdateVgMsgInQueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDBVgInfo *dbInfo, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_UPDATE_VG, .syncReq = syncReq}; SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); @@ -355,9 +321,9 @@ _return: CTG_RET(code); } -int32_t ctgPushUpdateTblMsgInQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq}; +int32_t ctgPushUpdateTblMsgInQueue(SCatalog *pCtg, STableMetaOutput *output, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq}; SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); @@ -377,17 +343,17 @@ int32_t ctgPushUpdateTblMsgInQueue(SCatalog* pCtg, STableMetaOutput *output, boo CTG_ERR_JRET(ctgPushAction(pCtg, &action)); return TSDB_CODE_SUCCESS; - + _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } -int32_t ctgPushUpdateUserMsgInQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { - int32_t code = 0; - SCtgMetaAction action= {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; +int32_t ctgPushUpdateUserMsgInQueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action = {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); @@ -402,38 +368,37 @@ int32_t ctgPushUpdateUserMsgInQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool CTG_ERR_JRET(ctgPushAction(pCtg, &action)); return TSDB_CODE_SUCCESS; - + _return: tFreeSGetUserAuthRsp(pAuth); taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_LOCK(CTG_READ, &dbCache->vgLock); - + if (dbCache->deleted) { CTG_UNLOCK(CTG_READ, &dbCache->vgLock); - ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); - + ctgDebug("db is dropping, dbId:%" PRIx64, dbCache->dbId); + *inCache = false; return TSDB_CODE_SUCCESS; } - if (NULL == dbCache->vgInfo) { CTG_UNLOCK(CTG_READ, &dbCache->vgLock); *inCache = false; - ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); + ctgDebug("db vgInfo is empty, dbId:%" PRIx64, dbCache->dbId); return TSDB_CODE_SUCCESS; } *inCache = true; - + return TSDB_CODE_SUCCESS; } @@ -441,7 +406,7 @@ int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { CTG_LOCK(CTG_WRITE, &dbCache->vgLock); if (dbCache->deleted) { - ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + ctgDebug("db is dropping, dbId:%" PRIx64, dbCache->dbId); CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } @@ -449,20 +414,13 @@ int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { return TSDB_CODE_SUCCESS; } -void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { - taosHashRelease(pCtg->dbCache, dbCache); -} +void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { taosHashRelease(pCtg->dbCache, dbCache); } -void ctgReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); -} +void ctgReleaseVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->vgLock); } -void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); -} +void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); } - -int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { +int32_t ctgAcquireDBCacheImpl(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { char *p = strchr(dbFName, '.'); if (p && CTG_IS_SYS_DBNAME(p + 1)) { dbFName = p + 1; @@ -474,7 +432,7 @@ int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache * } else { dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); } - + if (NULL == dbCache) { *pCache = NULL; ctgDebug("db not in cache, dbFName:%s", dbFName); @@ -484,28 +442,27 @@ int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache * if (dbCache->deleted) { if (acquire) { ctgReleaseDBCache(pCtg, dbCache); - } - + } + *pCache = NULL; ctgDebug("db is removing from cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } *pCache = dbCache; - + return TSDB_CODE_SUCCESS; } -int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { +int32_t ctgAcquireDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); } -int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { +int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } - -int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { +int32_t ctgAcquireVgInfoFromCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache, bool *inCache) { SCtgDBCache *dbCache = NULL; if (NULL == pCtg->dbCache) { @@ -514,11 +471,11 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac } ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { + if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; } - + ctgAcquireVgInfo(pCtg, dbCache, inCache); if (!(*inCache)) { ctgDebug("vgInfo of db %s not in cache", dbFName); @@ -531,7 +488,7 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac CTG_CACHE_STAT_ADD(vgHitNum, 1); ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); - + return TSDB_CODE_SUCCESS; _return: @@ -544,12 +501,12 @@ _return: *inCache = false; CTG_CACHE_STAT_ADD(vgMissNum, 1); - + return TSDB_CODE_SUCCESS; } -int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray *out) { - char *msg = NULL; +int32_t ctgGetQnodeListFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SArray *out) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get qnode list from mnode, mgmtEpInUse:%d", pMgmtEps->inUse); @@ -559,16 +516,16 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt ctgError("Build qnode list msg failed, error:%s", tstrerror(code)); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_QNODE_LIST, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { ctgError("error rsp for qnode list, error:%s", tstrerror(rpcRsp.code)); CTG_ERR_RET(rpcRsp.code); @@ -585,9 +542,9 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt return TSDB_CODE_SUCCESS; } - -int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { - char *msg = NULL; +int32_t ctgGetDBVgInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SBuildUseDBInput *input, + SUseDbOutput *out) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); @@ -597,16 +554,16 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtE ctgError("Build use db msg failed, code:%x, db:%s", code, input->db); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_USE_DB, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { ctgError("error rsp for use db, error:%s, db:%s", tstrerror(rpcRsp.code), input->db); CTG_ERR_RET(rpcRsp.code); @@ -623,8 +580,8 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtE return TSDB_CODE_SUCCESS; } -int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *dbFName, SDbCfgInfo *out) { - char *msg = NULL; +int32_t ctgGetDBCfgFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SDbCfgInfo *out) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get db cfg from mnode, dbFName:%s", dbFName); @@ -634,16 +591,16 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, ctgError("Build get db cfg msg failed, code:%x, db:%s", code, dbFName); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_GET_DB_CFG, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { ctgError("error rsp for get db cfg, error:%s, db:%s", tstrerror(rpcRsp.code), dbFName); CTG_ERR_RET(rpcRsp.code); @@ -660,8 +617,9 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, return TSDB_CODE_SUCCESS; } -int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *indexName, SIndexInfo *out) { - char *msg = NULL; +int32_t ctgGetIndexInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *indexName, + SIndexInfo *out) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get index from mnode, indexName:%s", indexName); @@ -671,16 +629,16 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt ctgError("Build get index msg failed, code:%x, db:%s", code, indexName); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_GET_INDEX, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { ctgError("error rsp for get index, error:%s, indexName:%s", tstrerror(rpcRsp.code), indexName); CTG_ERR_RET(rpcRsp.code); @@ -697,8 +655,9 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt return TSDB_CODE_SUCCESS; } -int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *funcName, SFuncInfo **out) { - char *msg = NULL; +int32_t ctgGetUdfInfoFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *funcName, + SFuncInfo **out) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get udf info from mnode, funcName:%s", funcName); @@ -708,23 +667,23 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEp ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_RETRIEVE_FUNC, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (TSDB_CODE_MND_FUNC_NOT_EXIST == rpcRsp.code) { ctgDebug("funcName %s not exist in mnode", funcName); taosMemoryFreeClear(*out); CTG_RET(TSDB_CODE_SUCCESS); } - + ctgError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rpcRsp.code), funcName); CTG_ERR_RET(rpcRsp.code); } @@ -740,8 +699,9 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEp return TSDB_CODE_SUCCESS; } -int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *user, SGetUserAuthRsp *authRsp) { - char *msg = NULL; +int32_t ctgGetUserDbAuthFromMnode(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user, + SGetUserAuthRsp *authRsp) { + char *msg = NULL; int32_t msgLen = 0; ctgDebug("try to get user auth from mnode, user:%s", user); @@ -751,16 +711,16 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm ctgError("Build get user auth msg failed, code:%x, db:%s", code, user); CTG_ERR_RET(code); } - + SRpcMsg rpcMsg = { .msgType = TDMT_MND_GET_USER_AUTH, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pRpc, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { ctgError("error rsp for get user auth, error:%s, user:%s", tstrerror(rpcRsp.code), user); CTG_ERR_RET(rpcRsp.code); @@ -777,8 +737,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm return TSDB_CODE_SUCCESS; } - -int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { +int32_t ctgIsTableMetaExistInCache(SCatalog *pCtg, char *dbFName, char *tbName, int32_t *exist) { if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); @@ -793,13 +752,13 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, } size_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - + if (NULL == tbMeta) { ctgReleaseDBCache(pCtg, dbCache); - + *exist = 0; ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; @@ -808,14 +767,14 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, *exist = 1; ctgReleaseDBCache(pCtg, dbCache); - + ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); - + return TSDB_CODE_SUCCESS; } - -int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, bool *inCache, int32_t flag, uint64_t *dbId) { +int32_t ctgGetTableMetaFromCache(SCatalog *pCtg, const SName *pTableName, STableMeta **pTableMeta, bool *inCache, + int32_t flag, uint64_t *dbId) { if (NULL == pCtg->dbCache) { ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); goto _return; @@ -836,10 +795,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable ctgDebug("db %s not in cache", pTableName->tname); goto _return; } - + int32_t sz = 0; CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), (void **)pTableMeta, &sz); + int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), + (void **)pTableMeta, &sz); CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); if (NULL == *pTableMeta) { @@ -852,7 +812,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable *dbId = dbCache->dbId; } - STableMeta* tbMeta = *pTableMeta; + STableMeta *tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); @@ -860,34 +820,36 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable *inCache = true; CTG_CACHE_STAT_ADD(tblHitNum, 1); - + return TSDB_CODE_SUCCESS; } - ctgDebug("Got subtable meta from cache, type:%d, dbFName:%s, tbName:%s, suid:%" PRIx64, tbMeta->tableType, dbFName, pTableName->tname, tbMeta->suid); - + ctgDebug("Got subtable meta from cache, type:%d, dbFName:%s, tbName:%s, suid:%" PRIx64, tbMeta->tableType, dbFName, + pTableName->tname, tbMeta->suid); + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); - ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); + ctgError("stb not in stbCache, suid:%" PRIx64, tbMeta->suid); taosMemoryFreeClear(*pTableMeta); goto _return; } - if ((*stbMeta)->suid != tbMeta->suid) { + if ((*stbMeta)->suid != tbMeta->suid) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); taosMemoryFreeClear(*pTableMeta); - ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); + ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, tbMeta->suid, + (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = CTG_META_SIZE(*stbMeta); *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); - if (NULL == *pTableMeta) { + if (NULL == *pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); ctgError("realloc size[%d] failed", metaSize); @@ -904,23 +866,23 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable CTG_CACHE_STAT_ADD(tblHitNum, 1); ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, pTableName->tname); - + return TSDB_CODE_SUCCESS; _return: *inCache = false; CTG_CACHE_STAT_ADD(tblMissNum, 1); - + return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { +int32_t ctgGetTableTypeFromCache(SCatalog *pCtg, const char *dbFName, const char *tableName, int32_t *tbType) { if (NULL == pCtg->dbCache) { - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); + ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); return TSDB_CODE_SUCCESS; } - + SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { @@ -932,9 +894,9 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char if (NULL == pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); + ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); ctgReleaseDBCache(pCtg, dbCache); - + return TSDB_CODE_SUCCESS; } @@ -946,17 +908,18 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); - + ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); + return TSDB_CODE_SUCCESS; } -int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { +int32_t ctgChkAuthFromCache(SCatalog *pCtg, const char *user, const char *dbFName, AUTH_TYPE type, bool *inCache, + bool *pass) { if (NULL == pCtg->userCache) { ctgDebug("empty user auth cache, user:%s", user); goto _return; } - + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); if (NULL == pUser) { ctgDebug("user not in cache, user:%s", user); @@ -967,7 +930,7 @@ int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFNam ctgDebug("Got user from cache, user:%s", user); CTG_CACHE_STAT_ADD(userHitNum, 1); - + if (pUser->superUser) { *pass = true; return TSDB_CODE_SUCCESS; @@ -979,32 +942,33 @@ int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFNam CTG_UNLOCK(CTG_READ, &pUser->lock); return TSDB_CODE_SUCCESS; } - + if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { *pass = true; } - + if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { *pass = true; } CTG_UNLOCK(CTG_READ, &pUser->lock); - + return TSDB_CODE_SUCCESS; _return: *inCache = false; CTG_CACHE_STAT_ADD(userMissNum, 1); - + return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnodeImpl(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, char *dbFName, char *tbName, + STableMetaOutput *output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; - char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; - int32_t msgLen = 0; + char *msg = NULL; + SEpSet *pVnodeEpSet = NULL; + int32_t msgLen = 0; ctgDebug("try to get table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); @@ -1016,22 +980,23 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* SRpcMsg rpcMsg = { .msgType = TDMT_MND_TABLE_META, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - + rpcSendRecv(pTrans, (SEpSet *)pMgmtEps, &rpcMsg, &rpcRsp); + if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { SET_META_TYPE_NULL(output->metaType); ctgDebug("stablemeta not exist in mnode, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } - - ctgError("error rsp for stablemeta from mnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tbName); + + ctgError("error rsp for stablemeta from mnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, + tbName); CTG_ERR_RET(rpcRsp.code); } @@ -1046,15 +1011,18 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnode(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + STableMetaOutput *output) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); return ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, dbFName, (char *)pTableName->tname, output); } -int32_t ctgGetTableMetaFromVnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { - if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { +int32_t ctgGetTableMetaFromVnodeImpl(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + SVgroupInfo *vgroupInfo, STableMetaOutput *output) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || + NULL == output) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1063,39 +1031,43 @@ int32_t ctgGetTableMetaFromVnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* ctgDebug("try to get table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); - SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; - char *msg = NULL; + SBuildTableMetaInput bInput = { + .vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; + char *msg = NULL; int32_t msgLen = 0; int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen); if (code) { - ctgError("Build vnode tablemeta msg failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tNameGetTableName(pTableName)); + ctgError("Build vnode tablemeta msg failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, + tNameGetTableName(pTableName)); CTG_ERR_RET(code); } SRpcMsg rpcMsg = { .msgType = TDMT_VND_TABLE_META, - .pCont = msg, + .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); - + if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { SET_META_TYPE_NULL(output->metaType); ctgDebug("tablemeta not exist in vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); return TSDB_CODE_SUCCESS; } - - ctgError("error rsp for table meta from vnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tNameGetTableName(pTableName)); + + ctgError("error rsp for table meta from vnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, + tNameGetTableName(pTableName)); CTG_ERR_RET(rpcRsp.code); } code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { - ctgError("Process vnode tablemeta rsp failed, code:%s, dbFName:%s, tbName:%s", tstrerror(code), dbFName, tNameGetTableName(pTableName)); + ctgError("Process vnode tablemeta rsp failed, code:%s, dbFName:%s, tbName:%s", tstrerror(code), dbFName, + tNameGetTableName(pTableName)); CTG_ERR_RET(code); } @@ -1103,17 +1075,18 @@ int32_t ctgGetTableMetaFromVnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromVnode(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + SVgroupInfo *vgroupInfo, STableMetaOutput *output) { int32_t code = 0; int32_t retryNum = 0; - + while (retryNum < CTG_DEFAULT_MAX_RETRY_TIMES) { code = ctgGetTableMetaFromVnodeImpl(pCtg, pTrans, pMgmtEps, pTableName, vgroupInfo, output); if (code) { if (TSDB_CODE_VND_HASH_MISMATCH == code) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - + code = catalogRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName); if (code != TSDB_CODE_SUCCESS) { break; @@ -1140,17 +1113,17 @@ int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { return TSDB_CODE_SUCCESS; } -int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) { - SHashObj *vgroupHash = NULL; +int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray **pList) { + SHashObj *vgroupHash = NULL; SVgroupInfo *vgInfo = NULL; - SArray *vgList = NULL; - int32_t code = 0; - int32_t vgNum = taosHashGetSize(vgHash); + SArray *vgList = NULL; + int32_t code = 0; + int32_t vgNum = taosHashGetSize(vgHash); vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); if (NULL == vgList) { ctgError("taosArrayInit failed, num:%d", vgNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } void *pIter = taosHashIterate(vgHash, NULL); @@ -1159,10 +1132,10 @@ int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList) { if (NULL == taosArrayPush(vgList, vgInfo)) { ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); - taosHashCancelIterate(vgHash, pIter); + taosHashCancelIterate(vgHash, pIter); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - + pIter = taosHashIterate(vgHash, pIter); vgInfo = NULL; } @@ -1184,9 +1157,9 @@ _return: int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { int32_t code = 0; - + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); - char db[TSDB_DB_FNAME_LEN] = {0}; + char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); if (vgNum <= 0) { @@ -1195,7 +1168,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName } tableNameHashFp fp = NULL; - SVgroupInfo *vgInfo = NULL; + SVgroupInfo *vgInfo = NULL; CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); @@ -1211,13 +1184,14 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName taosHashCancelIterate(dbInfo->vgHash, pIter); break; } - + pIter = taosHashIterate(dbInfo->vgHash, pIter); vgInfo = NULL; } if (NULL == vgInfo) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash)); + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, + taosHashGetSize(dbInfo->vgHash)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1226,54 +1200,53 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } -int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { - if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { +int32_t ctgStbVersionSearchCompare(const void *key1, const void *key2) { + if (*(uint64_t *)key1 < ((SSTableMetaVersion *)key2)->suid) { return -1; - } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { + } else if (*(uint64_t *)key1 > ((SSTableMetaVersion *)key2)->suid) { return 1; } else { return 0; } } -int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) { - if (*(int64_t *)key1 < ((SDbVgVersion*)key2)->dbId) { +int32_t ctgDbVgVersionSearchCompare(const void *key1, const void *key2) { + if (*(int64_t *)key1 < ((SDbVgVersion *)key2)->dbId) { return -1; - } else if (*(int64_t *)key1 > ((SDbVgVersion*)key2)->dbId) { + } else if (*(int64_t *)key1 > ((SDbVgVersion *)key2)->dbId) { return 1; } else { return 0; } } -int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { +int32_t ctgStbVersionSortCompare(const void *key1, const void *key2) { + if (((SSTableMetaVersion *)key1)->suid < ((SSTableMetaVersion *)key2)->suid) { return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + } else if (((SSTableMetaVersion *)key1)->suid > ((SSTableMetaVersion *)key2)->suid) { return 1; } else { return 0; } } -int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) { - if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { +int32_t ctgDbVgVersionSortCompare(const void *key1, const void *key2) { + if (((SDbVgVersion *)key1)->dbId < ((SDbVgVersion *)key2)->dbId) { return -1; - } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { + } else if (((SDbVgVersion *)key1)->dbId > ((SDbVgVersion *)key2)->dbId) { return 1; } else { return 0; } } - int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; mgmt->type = type; size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; - + mgmt->slots = taosMemoryCalloc(1, msgSize); if (NULL == mgmt->slots) { qError("calloc %d failed", (int32_t)msgSize); @@ -1281,34 +1254,34 @@ int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { } qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); - + return TSDB_CODE_SUCCESS; } - int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - + int32_t code = 0; + CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size); if (NULL == slot->meta) { - qError("taosArrayInit %d failed, id:%"PRIx64", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, mgmt->type); + qError("taosArrayInit %d failed, id:%" PRIx64 ", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, + mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } if (NULL == taosArrayPush(slot->meta, meta)) { - qError("taosArrayPush meta to rent failed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("taosArrayPush meta to rent failed, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } slot->needSort = true; - qDebug("add meta to rent, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("add meta to rent, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); _return: @@ -1316,20 +1289,22 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { +int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, + __compar_fn_t searchCompare) { int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; + int32_t code = 0; CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { - qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("empty meta slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (slot->needSort) { - qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, + (int32_t)taosArrayGetSize(slot->meta)); taosArraySort(slot->meta, sortCompare); slot->needSort = false; qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); @@ -1337,20 +1312,22 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); if (NULL == orig) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + qError("meta not found in slot, id:%" PRIx64 ", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, + (int32_t)taosArrayGetSize(slot->meta)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } memcpy(orig, meta, size); - qDebug("meta in rent updated, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("meta in rent updated, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); _return: CTG_UNLOCK(CTG_WRITE, &slot->lock); if (code) { - qWarn("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); + qWarn("meta in rent update failed, will try to add it, code:%x, id:%" PRIx64 ", slot idx:%d, type:%d", code, id, + widx, mgmt->type); CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); } @@ -1361,11 +1338,11 @@ int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortComp int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - + int32_t code = 0; + CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { - qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("empty meta slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1377,13 +1354,13 @@ int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortComp int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ); if (idx < 0) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("meta not found in slot, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } taosArrayRemove(slot->meta, idx); - qDebug("meta in rent removed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("meta in rent removed, id:%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); _return: @@ -1392,7 +1369,6 @@ _return: CTG_RET(code); } - int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1); if (ridx >= mgmt->slotNum) { @@ -1401,8 +1377,8 @@ int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_ } SCtgRentSlot *slot = &mgmt->slots[ridx]; - int32_t code = 0; - + int32_t code = 0; + CTG_LOCK(CTG_READ, &slot->lock); if (NULL == slot->meta) { qDebug("empty meta in slot:%d, type:%d", ridx, mgmt->type); @@ -1468,13 +1444,15 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { SCtgDBCache newDBCache = {0}; newDBCache.dbId = dbId; - newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == newDBCache.tbCache.metaCache) { ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); if (NULL == newDBCache.tbCache.stbCache) { ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); @@ -1486,21 +1464,21 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { ctgDebug("db already in cache, dbFName:%s", dbFName); goto _return; } - + ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_ADD(dbNum, 1); - + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + ctgDebug("db added to cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbId); CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%" PRIx64, dbFName, vgVersion.vgVersion, dbId); return TSDB_CODE_SUCCESS; @@ -1511,8 +1489,7 @@ _return: CTG_RET(code); } - -void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { +void ctgRemoveStbRent(SCatalog *pCtg, SCtgTbMetaCache *cache) { CTG_LOCK(CTG_WRITE, &cache->stbLock); if (cache->stbCache) { void *pIter = taosHashIterate(cache->stbCache, NULL); @@ -1520,21 +1497,21 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { uint64_t *suid = NULL; suid = taosHashGetKey(pIter, NULL); - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { - ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); + if (TSDB_CODE_SUCCESS == + ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { + ctgDebug("stb removed from rent, suid:%" PRIx64, *suid); } - + pIter = taosHashIterate(cache->stbCache, pIter); } } CTG_UNLOCK(CTG_WRITE, &cache->stbLock); } - -int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { +int32_t ctgRemoveDB(SCatalog *pCtg, SCtgDBCache *dbCache, const char *dbFName) { uint64_t dbId = dbCache->dbId; - - ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + ctgInfo("start to remove db from cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbCache->dbId); atomic_store_8(&dbCache->deleted, 1); @@ -1543,8 +1520,8 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { ctgFreeDbCache(dbCache); CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + + ctgDebug("db removed from rent, dbFName:%s, dbId:%" PRIx64, dbFName, dbCache->dbId); if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); @@ -1553,19 +1530,18 @@ int32_t ctgRemoveDB(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { CTG_CACHE_STAT_SUB(dbNum, 1); - ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - + ctgInfo("db removed from cache, dbFName:%s, dbId:%" PRIx64, dbFName, dbId); + return TSDB_CODE_SUCCESS; } - -int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { - int32_t code = 0; +int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { + int32_t code = 0; SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, dbFName, &dbCache); - + if (dbCache) { - // TODO OPEN IT + // TODO OPEN IT #if 0 if (dbCache->dbId == dbId) { *pCache = dbCache; @@ -1582,7 +1558,7 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt *pCache = dbCache; return TSDB_CODE_SUCCESS; } - + if (dbCache->dbId == dbId) { *pCache = dbCache; return TSDB_CODE_SUCCESS; @@ -1590,7 +1566,7 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt #endif CTG_ERR_RET(ctgRemoveDB(pCtg, dbCache, dbFName)); } - + CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); ctgGetDBCache(pCtg, dbFName, &dbCache); @@ -1600,46 +1576,47 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } - -int32_t ctgUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { - int32_t code = 0; - SDBVgInfo* dbInfo = *pDbInfo; +int32_t ctgUpdateDBVgInfo(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SDBVgInfo **pDbInfo) { + int32_t code = 0; + SDBVgInfo *dbInfo = *pDbInfo; if (NULL == dbInfo->vgHash) { return TSDB_CODE_SUCCESS; } - + if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", - dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", dbFName, dbInfo->vgHash, + dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - bool newAdded = false; + bool newAdded = false; SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; SCtgDBCache *dbCache = NULL; CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, dbFName, dbId); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } SDBVgInfo *vgInfo = NULL; CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); - + if (dbCache->vgInfo) { if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) { - ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); + ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, + dbCache->vgInfo->vgVersion); ctgWReleaseVgInfo(dbCache); - + return TSDB_CODE_SUCCESS; } if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) { - ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); + ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, + dbInfo->numOfTable); ctgWReleaseVgInfo(dbCache); - + return TSDB_CODE_SUCCESS; } @@ -1650,53 +1627,55 @@ int32_t ctgUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SD *pDbInfo = NULL; - ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); ctgWReleaseVgInfo(dbCache); dbCache = NULL; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), + ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + CTG_RET(code); } - -int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { +int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, + STableMeta *meta, int32_t metaSize) { SCtgTbMetaCache *tbCache = &dbCache->tbCache; CTG_LOCK(CTG_READ, &tbCache->metaLock); if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("db is dropping, dbId:%" PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } - int8_t origType = 0; - uint64_t origSuid = 0; - bool isStb = meta->tableType == TSDB_SUPER_TABLE; + int8_t origType = 0; + uint64_t origSuid = 0; + bool isStb = meta->tableType == TSDB_SUPER_TABLE; STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); if (orig) { origType = orig->tableType; - if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && + orig->tversion >= meta->tversion) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; } - + if (origType == TSDB_SUPER_TABLE) { if ((!isStb) || orig->suid != meta->suid) { CTG_LOCK(CTG_WRITE, &tbCache->stbLock); if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%" PRIx64, dbFName, tbName, orig->suid); } else { CTG_CACHE_STAT_SUB(stblNum, 1); } CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%" PRIx64, dbFName, tbName, orig->suid); + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); } @@ -1707,13 +1686,13 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui if (isStb) { CTG_LOCK(CTG_WRITE, &tbCache->stbLock); } - + if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { if (isStb) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); } - - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1722,35 +1701,38 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui CTG_CACHE_STAT_ADD(tblNum, 1); } - ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64, dbFName, tbName, meta->tableType, meta->suid); + ctgDebug("tbmeta updated to cache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64, dbFName, tbName, meta->tableType, + meta->suid); ctgdShowTableMeta(pCtg, tbName, meta); if (!isStb) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; } STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + ctgError("taosHashPut stable to stable cache failed, suid:%" PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_ADD(stblNum, 1); - + CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64 ",ma:%p", dbFName, tbName, meta->tableType, meta->suid, tbMeta); + ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d, suid:%" PRIx64 ",ma:%p", dbFName, tbName, + meta->tableType, meta->suid, tbMeta); - SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; + SSTableMetaVersion metaRent = { + .dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; strcpy(metaRent.dbFName, dbFName); strcpy(metaRent.stbName, tbName); CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); - + return TSDB_CODE_SUCCESS; } @@ -1772,7 +1754,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { } int32_t *vgId = NULL; - void *pIter = taosHashIterate(src->vgHash, NULL); + void *pIter = taosHashIterate(src->vgHash, NULL); while (pIter) { vgId = taosHashGetKey(pIter, NULL); @@ -1783,18 +1765,16 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) { taosMemoryFreeClear(*dst); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - + pIter = taosHashIterate(src->vgHash, pIter); } - return TSDB_CODE_SUCCESS; } - - -int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { - bool inCache = false; +int32_t ctgGetDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SCtgDBCache **dbCache, + SDBVgInfo **pInfo) { + bool inCache = false; int32_t code = 0; CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, dbCache, &inCache)); @@ -1803,7 +1783,7 @@ int32_t ctgGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const return TSDB_CODE_SUCCESS; } - SUseDbOutput DbOut = {0}; + SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); @@ -1829,18 +1809,18 @@ _return: taosMemoryFreeClear(*pInfo); *pInfo = DbOut.dbVgroup; - + CTG_RET(code); } -int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName) { - bool inCache = false; - int32_t code = 0; - SCtgDBCache* dbCache = NULL; +int32_t ctgRefreshDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName) { + bool inCache = false; + int32_t code = 0; + SCtgDBCache *dbCache = NULL; CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); - SUseDbOutput DbOut = {0}; + SUseDbOutput DbOut = {0}; SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); @@ -1850,7 +1830,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, c ctgReleaseVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } - + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; input.numOfTable = 0; @@ -1869,8 +1849,6 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, c return TSDB_CODE_SUCCESS; } - - int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) { *pOutput = taosMemoryMalloc(sizeof(STableMetaOutput)); if (NULL == *pOutput) { @@ -1895,15 +1873,14 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) return TSDB_CODE_SUCCESS; } - - -int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t flag, STableMetaOutput **pOutput, bool syncReq) { +int32_t ctgRefreshTblMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, int32_t flag, + STableMetaOutput **pOutput, bool syncReq) { if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } SVgroupInfo vgroupInfo = {0}; - int32_t code = 0; + int32_t code = 0; if (!CTG_FLAG_IS_SYS_DB(flag)) { CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo)); @@ -1919,7 +1896,8 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (CTG_FLAG_IS_SYS_DB(flag)) { ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(pTableName)); - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, (char *)pTableName->tname, output)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, + (char *)pTableName->tname, output)); } else if (CTG_FLAG_IS_STB(flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); @@ -1939,7 +1917,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pTableName)); taosMemoryFreeClear(output->tbMeta); - + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; @@ -1953,14 +1931,14 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, if (CTG_IS_META_NULL(moutput.metaType)) { SET_META_TYPE_NULL(output->metaType); } - + taosMemoryFreeClear(output->tbMeta); output->tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else { taosMemoryFreeClear(output->tbMeta); - - SET_META_TYPE_CTABLE(output->metaType); + + SET_META_TYPE_CTABLE(output->metaType); } } } @@ -1972,9 +1950,11 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, } if (CTG_IS_META_TABLE(output->metaType)) { - ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, output->tbMeta->tableType); + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d", output->dbFName, output->tbName, + output->tbMeta->tableType); } else { - ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); + ctgDebug("tbmeta got, dbFName:%s, tbName:%s, tbType:%d, stbMetaGot:%d", output->dbFName, output->ctbName, + output->ctbMeta.tableType, CTG_IS_META_BOTH(output->metaType)); } if (pOutput) { @@ -1989,19 +1969,20 @@ _return: taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output); - + CTG_RET(code); } -int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t flag) { +int32_t ctgGetTableMeta(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName, + STableMeta **pTableMeta, int32_t flag) { if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - bool inCache = false; - int32_t code = 0; - uint64_t dbId = 0; - uint64_t suid = 0; + bool inCache = false; + int32_t code = 0; + uint64_t dbId = 0; + uint64_t suid = 0; STableMetaOutput *output = NULL; if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { @@ -2013,7 +1994,8 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons int32_t tbType = 0; if (inCache) { - if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_SYS_DB(flag)))) { + if (CTG_FLAG_MATCH_STB(flag, (*pTableMeta)->tableType) && + ((!CTG_FLAG_IS_FORCE_UPDATE(flag)) || (CTG_FLAG_IS_SYS_DB(flag)))) { goto _return; } @@ -2027,7 +2009,6 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons CTG_FLAG_SET_STB(flag, tbType); } - while (true) { CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output, false)); @@ -2038,7 +2019,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons if (CTG_IS_META_BOTH(output->metaType)) { memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); - + *pTableMeta = output->tbMeta; goto _return; } @@ -2055,11 +2036,11 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons strcpy(stbName.tname, output->tbName); taosMemoryFreeClear(output->tbMeta); - + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &inCache, flag, NULL)); if (!inCache) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); - + continue; } @@ -2095,12 +2076,13 @@ _return: CTG_RET(code); } -int32_t ctgChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { - bool inCache = false; +int32_t ctgChkAuth(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user, const char *dbFName, + AUTH_TYPE type, bool *pass) { + bool inCache = false; int32_t code = 0; - + *pass = false; - + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); if (inCache) { @@ -2109,7 +2091,7 @@ int32_t ctgChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const cha SGetUserAuthRsp authRsp = {0}; CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pRpc, pMgmtEps, user, &authRsp)); - + if (authRsp.superAuth) { *pass = true; goto _return; @@ -2135,53 +2117,52 @@ _return: return TSDB_CODE_SUCCESS; } - int32_t ctgActUpdateVg(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgUpdateVgMsg *msg = action->data; - + CTG_ERR_JRET(ctgUpdateDBVgInfo(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); _return: ctgFreeVgInfo(msg->dbInfo); taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgActRemoveDB(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgRemoveDBMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); if (NULL == dbCache) { goto _return; } - + if (dbCache->dbId != msg->dbId) { - ctgInfo("dbId already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); + ctgInfo("dbId already updated, dbFName:%s, dbId:%" PRIx64 ", targetId:%" PRIx64, msg->dbFName, dbCache->dbId, + msg->dbId); goto _return; } - + CTG_ERR_JRET(ctgRemoveDB(pCtg, dbCache, msg->dbFName)); _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } - int32_t ctgActUpdateTbl(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgUpdateTblMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; - STableMetaOutput* output = msg->output; - SCtgDBCache *dbCache = NULL; + SCatalog *pCtg = msg->pCtg; + STableMetaOutput *output = msg->output; + SCtgDBCache *dbCache = NULL; if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); @@ -2191,22 +2172,24 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action) { if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - + } + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, output->dbFName, output->dbId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { int32_t metaSize = CTG_META_SIZE(output->tbMeta); - - CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); + + CTG_ERR_JRET( + ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); } if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); + CTG_ERR_JRET(ctgUpdateTblMeta(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, + (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); } _return: @@ -2215,17 +2198,16 @@ _return: taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output); } - + taosMemoryFreeClear(msg); - + CTG_RET(code); } - int32_t ctgActRemoveStb(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgRemoveStbMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, msg->dbFName, &dbCache); @@ -2234,44 +2216,46 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { } if (msg->dbId && (dbCache->dbId != msg->dbId)) { - ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", stb:%s, suid:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); + ctgDebug("dbId already modified, dbFName:%s, current:%" PRIx64 ", dbId:%" PRIx64 ", stb:%s, suid:%" PRIx64, + msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); return TSDB_CODE_SUCCESS; } - + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { - ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, + msg->suid); } else { CTG_CACHE_STAT_SUB(stblNum, 1); } CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { - ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { + ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_SUB(tblNum, 1); } CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - - ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + + ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid); CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); - - ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - + + ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%" PRIx64, msg->dbFName, msg->stbName, msg->suid); + _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgActRemoveTbl(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgRemoveTblMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, msg->dbFName, &dbCache); @@ -2280,7 +2264,8 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) { } if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); + ctgDebug("dbId already modified, dbFName:%s, current:%" PRIx64 ", dbId:%" PRIx64 ", tbName:%s", msg->dbFName, + dbCache->dbId, msg->dbId, msg->tbName); return TSDB_CODE_SUCCESS; } @@ -2304,18 +2289,19 @@ _return: } int32_t ctgActUpdateUser(SCtgMetaAction *action) { - int32_t code = 0; + int32_t code = 0; SCtgUpdateUserMsg *msg = action->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; if (NULL == pCtg->userCache) { - pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), + false, HASH_ENTRY_LOCK); if (NULL == pCtg->userCache) { ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } } - + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); if (NULL == pUser) { SCtgUserAuth userAuth = {0}; @@ -2326,7 +2312,7 @@ int32_t ctgActUpdateUser(SCtgMetaAction *action) { userAuth.readDbs = msg->userAuth.readDbs; userAuth.writeDbs = msg->userAuth.writeDbs; - if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { + if (taosHashPut(pCtg->userCache, msg->userAuth.user, sizeof(msg->userAuth.user), &userAuth, sizeof(userAuth))) { ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -2356,30 +2342,28 @@ int32_t ctgActUpdateUser(SCtgMetaAction *action) { _return: - taosHashCleanup(msg->userAuth.createdDbs); taosHashCleanup(msg->userAuth.readDbs); taosHashCleanup(msg->userAuth.writeDbs); - + taosMemoryFreeClear(msg); - + CTG_RET(code); } - -void* ctgUpdateThreadFunc(void* param) { +void *ctgUpdateThreadFunc(void *param) { setThreadName("catalog"); qInfo("catalog update thread started"); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); - + while (true) { if (tsem_wait(&gCtgMgmt.queue.reqSem)) { qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } - - if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + + if (atomic_load_8((int8_t *)&gCtgMgmt.exit)) { tsem_post(&gCtgMgmt.queue.rspSem); break; } @@ -2389,7 +2373,7 @@ void* ctgUpdateThreadFunc(void* param) { SCatalog *pCtg = ((SCtgUpdateMsgHeader *)action->data)->pCtg; ctgDebug("process [%s] action", gCtgAction[action->act].name); - + (*gCtgAction[action->act].func)(action); gCtgMgmt.queue.seqDone = action->seqId; @@ -2398,7 +2382,7 @@ void* ctgUpdateThreadFunc(void* param) { tsem_post(&gCtgMgmt.queue.rspSem); } - CTG_RUNTIME_STAT_ADD(qDoneNum, 1); + CTG_RUNTIME_STAT_ADD(qDoneNum, 1); ctgdShowClusterCache(pCtg); } @@ -2406,11 +2390,10 @@ void* ctgUpdateThreadFunc(void* param) { CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); qInfo("catalog update thread stopped"); - + return NULL; } - int32_t ctgStartUpdateThread() { TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); @@ -2420,27 +2403,28 @@ int32_t ctgStartUpdateThread() { terrno = TAOS_SYSTEM_ERROR(errno); CTG_ERR_RET(terrno); } - + taosThreadAttrDestroy(&thAttr); return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { - STableMeta *tbMeta = NULL; - int32_t code = 0; - SVgroupInfo vgroupInfo = {0}; - SCtgDBCache* dbCache = NULL; - SArray *vgList = NULL; - SDBVgInfo *vgInfo = NULL; +int32_t ctgGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName, + SArray **pVgList) { + STableMeta *tbMeta = NULL; + int32_t code = 0; + SVgroupInfo vgroupInfo = {0}; + SCtgDBCache *dbCache = NULL; + SArray *vgList = NULL; + SDBVgInfo *vgInfo = NULL; *pVgList = NULL; - + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, &tbMeta, CTG_FLAG_UNKNOWN_STB)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - SHashObj *vgHash = NULL; + SHashObj *vgHash = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, db, &dbCache, &vgInfo)); if (dbCache) { @@ -2455,7 +2439,7 @@ int32_t ctgGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps // USE HASH METHOD INSTEAD OF VGID IN TBMETA ctgError("invalid method to get none stb vgInfo, tbType:%d", tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); - + #if 0 int32_t vgId = tbMeta->vgId; if (taosHashGetDup(vgHash, &vgId, sizeof(vgId), &vgroupInfo) != 0) { @@ -2476,7 +2460,7 @@ int32_t ctgGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps *pVgList = vgList; vgList = NULL; -#endif +#endif } _return: @@ -2507,7 +2491,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - atomic_store_8((int8_t*)&gCtgMgmt.exit, false); + atomic_store_8((int8_t *)&gCtgMgmt.exit, false); if (cfg) { memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); @@ -2534,7 +2518,8 @@ int32_t catalogInit(SCatalogCfg *cfg) { gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } - gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), + false, HASH_ENTRY_LOCK); if (NULL == gCtgMgmt.pCluster) { qError("taosHashInit %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -2544,7 +2529,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); } - + if (tsem_init(&gCtgMgmt.queue.rspSem, 0, 0)) { qError("tsem_init failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); CTG_ERR_RET(TSDB_CODE_CTG_SYS_ERROR); @@ -2559,30 +2544,31 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(ctgStartUpdateThread()); - qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", gCtgMgmt.cfg.maxDBCacheNum, + gCtgMgmt.cfg.maxTblCacheNum, gCtgMgmt.cfg.dbRentSec, gCtgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; } -int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { +int32_t catalogGetHandle(uint64_t clusterId, SCatalog **catalogHandle) { if (NULL == catalogHandle) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } if (NULL == gCtgMgmt.pCluster) { - qError("catalog cluster cache are not ready, clusterId:%"PRIx64, clusterId); + qError("catalog cluster cache are not ready, clusterId:%" PRIx64, clusterId); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } - int32_t code = 0; + int32_t code = 0; SCatalog *clusterCtg = NULL; while (true) { - SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); + SCatalog **ctg = (SCatalog **)taosHashGet(gCtgMgmt.pCluster, (char *)&clusterId, sizeof(clusterId)); if (ctg && (*ctg)) { *catalogHandle = *ctg; - qDebug("got catalog handle from cache, clusterId:%"PRIx64", CTG:%p", clusterId, *ctg); + qDebug("got catalog handle from cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, *ctg); return TSDB_CODE_SUCCESS; } @@ -2597,7 +2583,8 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB)); CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); - clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), + false, HASH_ENTRY_LOCK); if (NULL == clusterCtg->dbCache) { qError("taosHashInit %d dbCache failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); @@ -2609,12 +2596,12 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { ctgFreeHandle(clusterCtg); continue; } - - qError("taosHashPut CTG to cache failed, clusterId:%"PRIx64, clusterId); + + qError("taosHashPut CTG to cache failed, clusterId:%" PRIx64, clusterId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("add CTG to cache, clusterId:%"PRIx64", CTG:%p", clusterId, clusterCtg); + qDebug("add CTG to cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, clusterCtg); break; } @@ -2622,36 +2609,36 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { *catalogHandle = clusterCtg; CTG_CACHE_STAT_ADD(clusterNum, 1); - + return TSDB_CODE_SUCCESS; _return: ctgFreeHandle(clusterCtg); - + CTG_RET(code); } -void catalogFreeHandle(SCatalog* pCtg) { +void catalogFreeHandle(SCatalog *pCtg) { if (NULL == pCtg) { return; } if (taosHashRemove(gCtgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) { - ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%"PRIx64, pCtg->clusterId); + ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%" PRIx64, pCtg->clusterId); return; } CTG_CACHE_STAT_SUB(clusterNum, 1); uint64_t clusterId = pCtg->clusterId; - + ctgFreeHandle(pCtg); - - ctgInfo("handle freed, culsterId:%"PRIx64, clusterId); + + ctgInfo("handle freed, culsterId:%" PRIx64, clusterId); } -int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t *tableNum) { +int32_t catalogGetDBVgVersion(SCatalog *pCtg, const char *dbFName, int32_t *version, int64_t *dbId, int32_t *tableNum) { CTG_API_ENTER(); if (NULL == pCtg || NULL == dbFName || NULL == version || NULL == dbId) { @@ -2659,8 +2646,8 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers } SCtgDBCache *dbCache = NULL; - bool inCache = false; - int32_t code = 0; + bool inCache = false; + int32_t code = 0; CTG_ERR_JRET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache, &inCache)); if (!inCache) { @@ -2684,18 +2671,19 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetDBVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SArray** vgroupList) { +int32_t catalogGetDBVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, + SArray **vgroupList) { CTG_API_ENTER(); if (NULL == pCtg || NULL == dbFName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - SCtgDBCache* dbCache = NULL; - int32_t code = 0; - SArray *vgList = NULL; - SHashObj *vgHash = NULL; - SDBVgInfo *vgInfo = NULL; + SCtgDBCache *dbCache = NULL; + int32_t code = 0; + SArray *vgList = NULL; + SHashObj *vgHash = NULL; + SDBVgInfo *vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName, &dbCache, &vgInfo)); if (dbCache) { vgHash = dbCache->vgInfo->vgHash; @@ -2720,15 +2708,14 @@ _return: taosMemoryFreeClear(vgInfo); } - CTG_API_LEAVE(code); + CTG_API_LEAVE(code); } - -int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo* dbInfo) { +int32_t catalogUpdateDBVgInfo(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SDBVgInfo *dbInfo) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName || NULL == dbInfo) { ctgFreeVgInfo(dbInfo); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); @@ -2741,12 +2728,11 @@ _return: CTG_API_LEAVE(code); } - -int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { +int32_t catalogRemoveDB(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2758,21 +2744,19 @@ int32_t catalogRemoveDB(SCatalog* pCtg, const char* dbFName, uint64_t dbId) { CTG_ERR_JRET(ctgPushRmDBMsgInQueue(pCtg, dbFName, dbId)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); - + _return: CTG_API_LEAVE(code); } -int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet) { - return 0; -} +int32_t catalogUpdateVgEpSet(SCatalog *pCtg, const char *dbFName, int32_t vgId, SEpSet *epSet) { return 0; } -int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { +int32_t catalogRemoveTableMeta(SCatalog *pCtg, const SName *pTableName) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == pTableName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2782,8 +2766,8 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { } STableMeta *tblMeta = NULL; - bool inCache = false; - uint64_t dbId = 0; + bool inCache = false; + uint64_t dbId = 0; CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, pTableName, &tblMeta, &inCache, 0, &dbId)); if (!inCache) { @@ -2793,14 +2777,13 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - + if (TSDB_SUPER_TABLE == tblMeta->tableType) { CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, tblMeta->suid, true)); } else { CTG_ERR_JRET(ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, true)); } - _return: taosMemoryFreeClear(tblMeta); @@ -2808,12 +2791,11 @@ _return: CTG_API_LEAVE(code); } - -int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid) { +int32_t catalogRemoveStbMeta(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *stbName, uint64_t suid) { CTG_API_ENTER(); int32_t code = 0; - + if (NULL == pCtg || NULL == dbFName || NULL == stbName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2825,29 +2807,32 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid, true)); CTG_API_LEAVE(TSDB_CODE_SUCCESS); - + _return: CTG_API_LEAVE(code); } -int32_t catalogGetIndexMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, const char *pIndexName, SIndexMeta** pIndexMeta) { +int32_t catalogGetIndexMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + const char *pIndexName, SIndexMeta **pIndexMeta) { return 0; } -int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + STableMeta **pTableMeta) { CTG_API_ENTER(); CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_UNKNOWN_STB)); } -int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetSTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + STableMeta **pTableMeta) { CTG_API_ENTER(); CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_STB)); } -int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { +int32_t catalogUpdateSTableMeta(SCatalog *pCtg, STableMetaRsp *rspMsg) { CTG_API_ENTER(); if (NULL == pCtg || NULL == rspMsg) { @@ -2859,42 +2844,41 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); } - + int32_t code = 0; strcpy(output->dbFName, rspMsg->dbFName); strcpy(output->tbName, rspMsg->tbName); output->dbId = rspMsg->dbId; - + SET_META_TYPE_TABLE(output->metaType); - + CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output->tbMeta)); CTG_ERR_JRET(ctgPushUpdateTblMsgInQueue(pCtg, output, false)); CTG_API_LEAVE(code); - + _return: taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output); - + CTG_API_LEAVE(code); } - - -int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* sver, int32_t *tbType, uint64_t *suid, char* stbName) { +int32_t ctgGetTbSverFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tbType, uint64_t *suid, + char *stbName) { *sver = -1; - + if (NULL == pCtg->dbCache) { ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } SCtgDBCache *dbCache = NULL; - char dbFName[TSDB_DB_FNAME_LEN] = {0}; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, dbFName); ctgAcquireDBCache(pCtg, dbFName, &dbCache); @@ -2902,9 +2886,9 @@ int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* ctgDebug("db %s not in cache", pTableName->tname); return TSDB_CODE_SUCCESS; } - + CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta* tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); + STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); if (tbMeta) { *tbType = tbMeta->tableType; *suid = tbMeta->suid; @@ -2922,32 +2906,33 @@ int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* if (*tbType != TSDB_CHILD_TABLE) { ctgReleaseDBCache(pCtg, dbCache); ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname); - + return TSDB_CODE_SUCCESS; } ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, *suid); - + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, suid, sizeof(*suid)); if (NULL == stbMeta || NULL == *stbMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("stb not in stbCache, suid:%"PRIx64, *suid); + ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid); return TSDB_CODE_SUCCESS; } if ((*stbMeta)->suid != *suid) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgReleaseDBCache(pCtg, dbCache); - ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, *suid, (*stbMeta)->suid); + ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, *suid, + (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } size_t nameLen = 0; - char* name = taosHashGetKey(*stbMeta, &nameLen); - + char *name = taosHashGetKey(*stbMeta, &nameLen); + strncpy(stbName, name, nameLen); stbName[nameLen] = 0; @@ -2958,38 +2943,38 @@ int32_t ctgGetTbSverFromCache(SCatalog* pCtg, const SName* pTableName, int32_t* ctgReleaseDBCache(pCtg, dbCache); ctgDebug("Got sver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tbType, dbFName, pTableName->tname); - + return TSDB_CODE_SUCCESS; } -int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables) { +int32_t catalogChkTbMetaVersion(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, SArray *pTables) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTables) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - SName name; + SName name; int32_t sver = 0; int32_t tbNum = taosArrayGetSize(pTables); for (int32_t i = 0; i < tbNum; ++i) { - STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); + STbSVersion *pTb = (STbSVersion *)taosArrayGet(pTables, i); tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - + if (CTG_IS_SYS_DBNAME(name.dbname)) { continue; } - int32_t tbType = 0; + int32_t tbType = 0; uint64_t suid = 0; - char stbName[TSDB_TABLE_FNAME_LEN]; + char stbName[TSDB_TABLE_FNAME_LEN]; ctgGetTbSverFromCache(pCtg, &name, &sver, &tbType, &suid, stbName); if (sver >= 0 && sver < pTb->sver) { switch (tbType) { case TSDB_CHILD_TABLE: { SName stb = name; strcpy(stb.tname, stbName); - catalogRemoveTableMeta(pCtg, &stb); + catalogRemoveTableMeta(pCtg, &stb); break; } case TSDB_SUPER_TABLE: @@ -3006,8 +2991,7 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgm CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - -int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName) { +int32_t catalogRefreshDBVgInfo(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const char *dbFName) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == dbFName) { @@ -3017,23 +3001,28 @@ int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmt CTG_API_LEAVE(ctgRefreshDBVgInfo(pCtg, pTrans, pMgmtEps, dbFName)); } -int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { +int32_t catalogRefreshTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + int32_t isSTable) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable), NULL, true)); + CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, + CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable), NULL, true)); } -int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { +int32_t catalogRefreshGetTableMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + STableMeta **pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable))); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, pTableMeta, + CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(isSTable))); } -int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { +int32_t catalogGetTableDistVgInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const SName *pTableName, + SArray **pVgList) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { @@ -3051,12 +3040,13 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm code = ctgGetTableDistVgInfo(pCtg, pRpc, pMgmtEps, pTableName, pVgList); if (code) { if (TSDB_CODE_CTG_VG_META_MISMATCH == code) { - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(CTG_FLAG_UNKNOWN_STB), NULL, true)); + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, + CTG_FLAG_FORCE_UPDATE | CTG_FLAG_MAKE_STB(CTG_FLAG_UNKNOWN_STB), NULL, true)); char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); + tNameGetFullDbName(pTableName, dbFName); CTG_ERR_JRET(ctgRefreshDBVgInfo(pCtg, pRpc, pMgmtEps, dbFName)); - + continue; } } @@ -3069,8 +3059,8 @@ _return: CTG_API_LEAVE(code); } - -int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { +int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, + SVgroupInfo *pVgroup) { CTG_API_ENTER(); if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { @@ -3078,9 +3068,9 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pM CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - SCtgDBCache* dbCache = NULL; - int32_t code = 0; - char db[TSDB_DB_FNAME_LEN] = {0}; + SCtgDBCache *dbCache = NULL; + int32_t code = 0; + char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); SDBVgInfo *vgInfo = NULL; @@ -3103,8 +3093,8 @@ _return: CTG_API_LEAVE(code); } - -int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { +int32_t catalogGetAllMeta(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SCatalogReq *pReq, + SMetaData *pRsp) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { @@ -3126,11 +3116,11 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, ctgError("taosArrayInit %d failed", tbNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - + for (int32_t i = 0; i < tbNum; ++i) { - SName *name = taosArrayGet(pReq->pTableName, i); + SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { @@ -3148,7 +3138,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, CTG_API_LEAVE(TSDB_CODE_SUCCESS); -_return: +_return: if (pRsp->pTableMeta) { int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); @@ -3156,19 +3146,19 @@ _return: STableMeta *pMeta = taosArrayGetP(pRsp->pTableMeta, i); taosMemoryFreeClear(pMeta); } - + taosArrayDestroy(pRsp->pTableMeta); pRsp->pTableMeta = NULL; } - + CTG_API_LEAVE(code); } -int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray* pQnodeList) { +int32_t catalogGetQnodeList(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, SArray *pQnodeList) { CTG_API_ENTER(); - + int32_t code = 0; - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3179,7 +3169,7 @@ _return: CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, uint32_t *num) { +int32_t catalogGetExpiredSTables(SCatalog *pCtg, SSTableMetaVersion **stables, uint32_t *num) { CTG_API_ENTER(); if (NULL == pCtg || NULL == stables || NULL == num) { @@ -3189,9 +3179,9 @@ int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, u CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); } -int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) { +int32_t catalogGetExpiredDBs(SCatalog *pCtg, SDbVgVersion **dbs, uint32_t *num) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == dbs || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3199,9 +3189,9 @@ int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } -int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num) { +int32_t catalogGetExpiredUsers(SCatalog *pCtg, SUserAuthVersion **users, uint32_t *num) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == users || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3215,7 +3205,7 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_ } } - uint32_t i = 0; + uint32_t i = 0; SCtgUserAuth *pAuth = taosHashIterate(pCtg->userCache, NULL); while (pAuth != NULL) { void *key = taosHashGetKey(pAuth, NULL); @@ -3227,10 +3217,9 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_ CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - -int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { +int32_t catalogGetDBCfg(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *dbFName, SDbCfgInfo *pDbCfg) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == dbFName || NULL == pDbCfg) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3238,9 +3227,10 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons CTG_API_LEAVE(ctgGetDBCfgFromMnode(pCtg, pRpc, pMgmtEps, dbFName, pDbCfg)); } -int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo) { +int32_t catalogGetIndexInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *indexName, + SIndexInfo *pInfo) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == indexName || NULL == pInfo) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3248,9 +3238,9 @@ int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, CTG_API_LEAVE(ctgGetIndexInfoFromMnode(pCtg, pRpc, pMgmtEps, indexName, pInfo)); } -int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo) { +int32_t catalogGetUdfInfo(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *funcName, SFuncInfo **pInfo) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -3262,32 +3252,33 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, co } CTG_ERR_JRET(ctgGetUdfInfoFromMnode(pCtg, pRpc, pMgmtEps, funcName, pInfo)); - + _return: if (code) { - taosMemoryFreeClear(*pInfo); + taosMemoryFreeClear(*pInfo); } - + CTG_API_LEAVE(code); } -int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { +int32_t catalogChkAuth(SCatalog *pCtg, void *pRpc, const SEpSet *pMgmtEps, const char *user, const char *dbFName, + AUTH_TYPE type, bool *pass) { CTG_API_ENTER(); - + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; CTG_ERR_JRET(ctgChkAuth(pCtg, pRpc, pMgmtEps, user, dbFName, type, pass)); - + _return: CTG_API_LEAVE(code); } -int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { +int32_t catalogUpdateUserAuthInfo(SCatalog *pCtg, SGetUserAuthRsp *pAuth) { CTG_API_ENTER(); if (NULL == pCtg || NULL == pAuth) { @@ -3297,20 +3288,19 @@ int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { CTG_API_LEAVE(ctgPushUpdateUserMsgInQueue(pCtg, pAuth, false)); } - void catalogDestroy(void) { qInfo("start to destroy catalog"); - - if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + + if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t *)&gCtgMgmt.exit)) { return; } - atomic_store_8((int8_t*)&gCtgMgmt.exit, true); + atomic_store_8((int8_t *)&gCtgMgmt.exit, true); if (tsem_post(&gCtgMgmt.queue.reqSem)) { qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } - + if (tsem_post(&gCtgMgmt.queue.rspSem)) { qError("tsem_post failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } @@ -3318,21 +3308,21 @@ void catalogDestroy(void) { while (CTG_IS_LOCKED(&gCtgMgmt.lock)) { taosUsleep(1); } - + CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock); SCatalog *pCtg = NULL; - void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); while (pIter) { pCtg = *(SCatalog **)pIter; if (pCtg) { catalogFreeHandle(pCtg); } - + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); } - + taosHashCleanup(gCtgMgmt.pCluster); gCtgMgmt.pCluster = NULL; @@ -3341,5 +3331,3 @@ void catalogDestroy(void) { qInfo("catalog destroyed"); } - - diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 4bbf6ccf45..9683655253 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -15,8 +15,8 @@ #define _DEFAULT_SOURCE #include "thash.h" -#include "taoserror.h" #include "os.h" +#include "taoserror.h" #include "tlog.h" // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT @@ -27,36 +27,36 @@ #define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) -#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) -#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) -#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode))) +#define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen) +#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode)) +#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode))) #define FREE_HASH_NODE(_fp, _n) \ do { \ -/* if (_fp != NULL) { \ - (_fp)(_n); \ - }*/ \ + /* if (_fp != NULL) { \ + (_fp)(_n); \ + }*/ \ taosMemoryFreeClear(_n); \ } while (0); struct SHashNode { - SHashNode *next; - uint32_t hashVal; // the hash value of key - uint32_t dataLen; // length of data - uint32_t keyLen; // length of the key - uint16_t refCount; // reference count - int8_t removed; // flag to indicate removed - char data[]; + SHashNode *next; + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + uint16_t refCount; // reference count + int8_t removed; // flag to indicate removed + char data[]; }; typedef struct SHashEntry { - int32_t num; // number of elements in current entry - SRWLatch latch; // entry latch - SHashNode *next; + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SHashNode *next; } SHashEntry; struct SHashObj { - SHashEntry ** hashList; + SHashEntry **hashList; size_t capacity; // number of slots int64_t size; // number of elements in hash table _hash_fn_t hashFp; // hash function @@ -65,7 +65,7 @@ struct SHashObj { SRWLatch lock; // read-write spin lock SHashLockTypeE type; // lock type bool enableUpdate; // enable update - SArray * pMemBlock; // memory block allocated for SHashEntry + SArray *pMemBlock; // memory block allocated for SHashEntry _hash_before_fn_t callbackFp; // function invoked before return the value to caller }; @@ -103,14 +103,14 @@ static FORCE_INLINE void taosHashRUnlock(SHashObj *pHashObj) { taosRUnLockLatch(&pHashObj->lock); } -static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry* pe) { +static FORCE_INLINE void taosHashEntryWLock(const SHashObj *pHashObj, SHashEntry *pe) { if (pHashObj->type == HASH_NO_LOCK) { return; } taosWLockLatch(&pe->latch); } -static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry* pe) { +static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEntry *pe) { if (pHashObj->type == HASH_NO_LOCK) { return; } @@ -118,7 +118,7 @@ static FORCE_INLINE void taosHashEntryWUnlock(const SHashObj *pHashObj, SHashEnt taosWUnLockLatch(&pe->latch); } -static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry* pe) { +static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry *pe) { if (pHashObj->type == HASH_NO_LOCK) { return; } @@ -126,7 +126,7 @@ static FORCE_INLINE void taosHashEntryRLock(const SHashObj *pHashObj, SHashEntry taosRLockLatch(&pe->latch); } -static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry* pe) { +static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEntry *pe) { if (pHashObj->type == HASH_NO_LOCK) { return; } @@ -142,12 +142,11 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { return i; } -static FORCE_INLINE SHashNode * -doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { +static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, + uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && - ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; @@ -186,7 +185,8 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param pNode the old node with requested key * @param pNewNode the new node with requested key */ -static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { +static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SHashNode *prev, SHashNode *pNode, + SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); atomic_sub_fetch_16(&pNode->refCount, 1); @@ -227,9 +227,7 @@ static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj); * @param pHashObj * @return */ -static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { - return taosHashGetSize(pHashObj) == 0; -} +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { return taosHashGetSize(pHashObj) == 0; } SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { if (fn == NULL) { @@ -251,7 +249,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp pHashObj->capacity = taosHashCapacity((int32_t)capacity); pHashObj->equalFp = memcmp; - pHashObj->hashFp = fn; + pHashObj->hashFp = fn; pHashObj->type = type; pHashObj->enableUpdate = update; @@ -305,7 +303,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) { if (pHashObj == NULL) { return 0; } - return (int32_t)atomic_load_64((int64_t*)&pHashObj->size); + return (int32_t)atomic_load_64((int64_t *)&pHashObj->size); } int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) { @@ -340,10 +338,9 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo } #endif - SHashNode* prev = NULL; + SHashNode *prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && - (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 && + if ((pNode->keyLen == keyLen) && (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; @@ -391,27 +388,27 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo } } -static void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef); +static void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef); void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { - void* p = NULL; + void *p = NULL; return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, false); } int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf) { terrno = 0; - /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false); + /*char* p = */ taosHashGetImpl(pHashObj, key, keyLen, &destBuf, 0, false); return terrno; } -int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t* size) { +int32_t taosHashGetDup_m(SHashObj *pHashObj, const void *key, size_t keyLen, void **destBuf, int32_t *size) { terrno = 0; - /*char* p = */taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false); + /*char* p = */ taosHashGetImpl(pHashObj, key, keyLen, destBuf, size, false); return terrno; } -void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** d, int32_t* size, bool addRef) { +void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { return NULL; } @@ -449,15 +446,15 @@ void* taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void** if (size != NULL) { if (*d == NULL) { - *size = pNode->dataLen; + *size = pNode->dataLen; *d = taosMemoryCalloc(1, *size); if (*d == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } } else if (*size < pNode->dataLen) { - *size = pNode->dataLen; - char* tmp = taosMemoryRealloc(*d, *size); + *size = pNode->dataLen; + char *tmp = taosMemoryRealloc(*d, *size); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -508,13 +505,12 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { return -1; } - int code = -1; + int code = -1; SHashNode *pNode = pe->next; SHashNode *prevNode = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && - ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { code = 0; // it is found @@ -598,14 +594,14 @@ void taosHashCleanup(SHashObj *pHashObj) { } // for profile only -int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){ +int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return 0; } int32_t num = 0; - taosHashRLock((SHashObj*) pHashObj); + taosHashRLock((SHashObj *)pHashObj); for (int32_t i = 0; i < pHashObj->size; ++i) { SHashEntry *pEntry = pHashObj->hashList[i]; @@ -616,7 +612,7 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj){ } } - taosHashRUnlock((SHashObj*) pHashObj); + taosHashRUnlock((SHashObj *)pHashObj); return num; } @@ -627,22 +623,22 @@ void taosHashTableResize(SHashObj *pHashObj) { int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); if (newCapacity > HASH_MAX_CAPACITY) { -// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", -// pHashObj->capacity, HASH_MAX_CAPACITY); + // uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", + // pHashObj->capacity, HASH_MAX_CAPACITY); return; } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { -// uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); + // uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; } pHashObj->hashList = pNewEntryList; size_t inc = newCapacity - pHashObj->capacity; - void * p = taosMemoryCalloc(inc, sizeof(SHashEntry)); + void *p = taosMemoryCalloc(inc, sizeof(SHashEntry)); for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); @@ -653,9 +649,9 @@ void taosHashTableResize(SHashObj *pHashObj) { pHashObj->capacity = newCapacity; for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { SHashEntry *pe = pHashObj->hashList[idx]; - SHashNode *pNode; - SHashNode *pNext; - SHashNode *pPrev = NULL; + SHashNode *pNode; + SHashNode *pNext; + SHashNode *pPrev = NULL; if (pe->num == 0) { assert(pe->next == NULL); @@ -688,8 +684,9 @@ void taosHashTableResize(SHashObj *pHashObj) { int64_t et = taosGetTimestampUs(); -// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, -// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); + // uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", + // (int32_t)pHashObj->capacity, + // ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { @@ -700,12 +697,12 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s return NULL; } - pNewNode->keyLen = (uint32_t)keyLen; + pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; pNewNode->dataLen = (uint32_t)dsize; - pNewNode->refCount= 1; + pNewNode->refCount = 1; pNewNode->removed = 0; - pNewNode->next = NULL; + pNewNode->next = NULL; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); @@ -727,11 +724,12 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return 0; } - return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void*))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); + return (pHashObj->capacity * (sizeof(SHashEntry) + sizeof(void *))) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + + sizeof(SHashObj); } -void *taosHashGetKey(void *data, size_t* keyLen) { - SHashNode * node = GET_HASH_PNODE(data); +void *taosHashGetKey(void *data, size_t *keyLen) { + SHashNode *node = GET_HASH_PNODE(data); if (keyLen != NULL) { *keyLen = node->keyLen; } @@ -751,8 +749,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { SHashNode *pNode = pe->next; while (pNode) { - if (pNode == pOld) - break; + if (pNode == pOld) break; prevNode = pNode; pNode = pNode->next; @@ -766,7 +763,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { } atomic_sub_fetch_16(&pOld->refCount, 1); - if (pOld->refCount <=0) { + if (pOld->refCount <= 0) { if (prevNode) { prevNode->next = pOld->next; } else { @@ -778,7 +775,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { FREE_HASH_NODE(pHashObj->freeFp, pOld); } } else { -// uError("pNode:%p data:%p is not there!!!", pNode, p); + // uError("pNode:%p data:%p is not there!!!", pNode, p); } return pNode; @@ -787,7 +784,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { void *taosHashIterate(SHashObj *pHashObj, void *p) { if (pHashObj == NULL) return NULL; - int slot = 0; + int slot = 0; char *data = NULL; // only add the read lock to disable the resize process @@ -865,9 +862,9 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) { taosHashRUnlock(pHashObj); } -//TODO remove it +// TODO remove it void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { - void* p = NULL; + void *p = NULL; return taosHashGetImpl(pHashObj, key, keyLen, &p, 0, true); } From 682a7cab1797f85c29b8d216eb512a9aa3a3fe11 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 16:35:24 +0800 Subject: [PATCH 14/73] fix: avoid memory leak --- include/util/taoserror.h | 5 ++--- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 12 ++++-------- source/dnode/mnode/impl/src/mndBnode.c | 8 ++++---- source/dnode/mnode/impl/src/mndConsumer.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 12 ++++++------ source/dnode/mnode/impl/src/mndDnode.c | 8 ++++---- source/dnode/mnode/impl/src/mndFunc.c | 8 ++++---- source/dnode/mnode/impl/src/mndMnode.c | 8 ++++---- source/dnode/mnode/impl/src/mndOffset.c | 2 +- source/dnode/mnode/impl/src/mndQnode.c | 8 ++++---- source/dnode/mnode/impl/src/mndSma.c | 8 ++++---- source/dnode/mnode/impl/src/mndSnode.c | 8 ++++---- source/dnode/mnode/impl/src/mndStb.c | 12 ++++++------ source/dnode/mnode/impl/src/mndStream.c | 4 ++-- source/dnode/mnode/impl/src/mndTopic.c | 6 +++--- source/dnode/mnode/impl/src/mndTrans.c | 10 +++++----- source/dnode/mnode/impl/src/mndUser.c | 12 ++++++------ source/dnode/mnode/impl/src/mnode.c | 2 +- source/util/src/terror.c | 1 - 19 files changed, 65 insertions(+), 71 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index e28618d940..1d7287ed0e 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -129,9 +129,8 @@ int32_t* taosGetErrno(); // mnode-common #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) #define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0301) -#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0302) -#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) -#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0304) +#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0302) +#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0303) // mnode-show #define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index c4314a57b1..c6233a4f18 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -46,7 +46,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { code = mndProcessMsg(pMsg); } - if (IsReq(pMsg) && pMsg->info.handle != NULL && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (IsReq(pMsg) && pMsg->info.handle != NULL && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && terrno != 0) code = terrno; mmSendRsp(pMsg, code); } @@ -59,18 +59,14 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void mmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; - tmsg_t msgType = pMsg->msgType; - bool isRequest = msgType & 1U; dTrace("msg:%p, get from mnode-query queue", pMsg); pMsg->info.node = pMgmt->pMnode; code = mndProcessMsg(pMsg); - if (isRequest) { - if (pMsg->info.handle != NULL && code != 0) { - if (code != 0 && terrno != 0) code = terrno; - mmSendRsp(pMsg, code); - } + if (IsReq(pMsg) && pMsg->info.handle != NULL && code != 0) { + if (terrno != 0) code = terrno; + mmSendRsp(pMsg, code); } dTrace("msg:%p, is freed, code:0x%x", pMsg, code); diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 924b6db8f7..3316a09462 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -304,10 +304,10 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { } code = mndCreateBnode(pMnode, pReq, pDnode, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("bnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); } @@ -414,10 +414,10 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { } code = mndDropBnode(pMnode, pReq, pObj); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("bnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 57f7b341d4..8b2799833b 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -511,7 +511,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { if (mndTransPrepare(pMnode, pTrans) != 0) goto SUBSCRIBE_OVER; } - code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + code = TSDB_CODE_ACTION_IN_PROGRESS; SUBSCRIBE_OVER: mndTransDrop(pTrans); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 0bf7b240b2..633dda454b 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -605,10 +605,10 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } code = mndCreateDb(pMnode, pReq, &createReq, pUser); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("db:%s, failed to create since %s", createReq.db, terrstr()); } @@ -839,10 +839,10 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); code = mndAlterDb(pMnode, pReq, pDb, &dbObj); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); } @@ -1110,10 +1110,10 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { } code = mndDropDb(pMnode, pReq, pDb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("db:%s, failed to drop since %s", dropReq.db, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 2a4cf01115..01ff08cef9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -504,10 +504,10 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { } code = mndCreateDnode(pMnode, pReq, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; CREATE_DNODE_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr()); } @@ -585,10 +585,10 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } code = mndDropDnode(pMnode, pReq, pDnode); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; DROP_DNODE_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index cf2edb5784..9107dab693 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -330,10 +330,10 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { } code = mndCreateFunc(pMnode, pReq, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("func:%s, failed to create since %s", createReq.name, terrstr()); } @@ -386,10 +386,10 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } code = mndDropFunc(pMnode, pReq, pFunc); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("func:%s, failed to drop since %s", dropReq.name, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 04c0a93485..7f86eb8b32 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -402,10 +402,10 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { } code = mndCreateMnode(pMnode, pReq, pDnode, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("mnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); } @@ -574,10 +574,10 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { } code = mndDropMnode(pMnode, pReq, pObj); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("mnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index b109ede819..6f42d66625 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -205,7 +205,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) { } mndTransDrop(pTrans); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return TSDB_CODE_ACTION_IN_PROGRESS; } static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) { diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index c153d86552..3dc6200229 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -306,10 +306,10 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { } code = mndCreateQnode(pMnode, pReq, pDnode, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("qnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); } @@ -416,10 +416,10 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { } code = mndDropQnode(pMnode, pReq, pObj); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("qnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 0f52f00d4e..b38e901d49 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -562,10 +562,10 @@ static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq) { } code = mndCreateSma(pMnode, pReq, &createReq, pDb, pStb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("sma:%s, failed to create since %s", createReq.name, terrstr(terrno)); } @@ -706,10 +706,10 @@ static int32_t mndProcessMDropSmaReq(SRpcMsg *pReq) { } code = mndDropSma(pMnode, pReq, pDb, pSma); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("sma:%s, failed to drop since %s", dropReq.name, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 5f58f2c890..87b61f59ec 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -312,10 +312,10 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { } code = mndCreateSnode(pMnode, pReq, pDnode, &createReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("snode:%d, failed to create since %s", createReq.dnodeId, terrstr()); return -1; } @@ -424,10 +424,10 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { } code = mndDropSnode(pMnode, pReq, pObj); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("snode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 7485510bc6..33e4828299 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -827,10 +827,10 @@ static int32_t mndProcessMCreateStbReq(SRpcMsg *pReq) { } code = mndCreateStb(pMnode, pReq, &createReq, pDb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stb:%s, failed to create since %s", createReq.name, terrstr()); } @@ -1334,10 +1334,10 @@ static int32_t mndProcessMAlterStbReq(SRpcMsg *pReq) { } code = mndAlterStb(pMnode, pReq, &alterReq, pDb, pStb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stb:%s, failed to alter since %s", alterReq.name, terrstr()); } @@ -1475,10 +1475,10 @@ static int32_t mndProcessMDropStbReq(SRpcMsg *pReq) { } code = mndDropStb(pMnode, pReq, pDb, pStb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stb:%s, failed to drop since %s", dropReq.name, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 61a84fc95c..7b6383a470 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -472,10 +472,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } code = mndCreateStream(pMnode, pReq, &createStreamReq, pDb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; CREATE_STREAM_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index c6eebb5c5d..b9b01a4391 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -457,10 +457,10 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { } code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; CREATE_TOPIC_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); } @@ -547,7 +547,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { return -1; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return TSDB_CODE_ACTION_IN_PROGRESS; } static int32_t mndProcessDropTopicInRsp(SRpcMsg *pRsp) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 6210fe3fcf..5d205197d1 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1035,13 +1035,13 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA } } else { mDebug("trans:%d, %d of %d actions executed", pTrans->id, numOfReceived, numOfActions); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return TSDB_CODE_ACTION_IN_PROGRESS; } } static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to execute redoActions since:%s, code:0x%x", terrstr(), terrno); } return code; @@ -1049,7 +1049,7 @@ static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to execute undoActions since %s", terrstr()); } return code; @@ -1088,7 +1088,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { pTrans->stage = TRN_STAGE_COMMIT; mDebug("trans:%d, stage from redoAction to commit", pTrans->id); continueExec = true; - } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("trans:%d, stage keep on redoAction since %s", pTrans->id, tstrerror(code)); continueExec = false; } else { @@ -1176,7 +1176,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { pTrans->stage = TRN_STAGE_UNDO_LOG; mDebug("trans:%d, stage from undoAction to undoLog", pTrans->id); continueExec = true; - } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("trans:%d, stage keep on undoAction since %s", pTrans->id, tstrerror(code)); continueExec = false; } else { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index f2a42e3083..88e646e765 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -331,10 +331,10 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { } code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("user:%s, failed to create since %s", createReq.user, terrstr()); } @@ -536,10 +536,10 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { } code = mndAlterUser(pMnode, pUser, &newUser, pReq); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("user:%s, failed to alter since %s", alterReq.user, terrstr()); } @@ -613,10 +613,10 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { } code = mndDropUser(pMnode, pReq, pUser); - if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("user:%s, failed to drop since %s", dropReq.user, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 8c805dd8c7..285ae030e1 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -367,7 +367,7 @@ int32_t mndProcessMsg(SRpcMsg *pMsg) { } int32_t code = (*fp)(pMsg); - if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code == TSDB_CODE_ACTION_IN_PROGRESS) { terrno = code; mTrace("msg:%p, in progress, app:%p", pMsg, ahandle); } else if (code != 0) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index f84775f0d7..3890a55ff1 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -134,7 +134,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt cl // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Mnode not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, "Message is progressing") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, "Invalid message connection") From 0000fa8418ee95b7a6ca59fa0c07b75c4d76d4f7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 21 May 2022 16:37:37 +0800 Subject: [PATCH 15/73] fea: add null tag support in schemaless --- include/common/tglobal.h | 1 + source/client/src/clientSml.c | 6 +----- source/common/src/tglobal.c | 2 ++ 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 111241cf03..2a4ef565dd 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -128,6 +128,7 @@ extern bool tsStartUdfd; // schemaless extern char tsSmlChildTableName[]; +extern char tsSmlTagName[]; extern bool tsSmlDataFormat; // internal diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 61282ae76a..8b2f47b295 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -63,10 +63,6 @@ for (int i = 1; i < keyLen; ++i) { \ #define TS "_ts" #define TS_LEN 3 -#define TAG "_tag" -#define TAG_LEN 4 -#define TAG_VALUE "NULL" -#define TAG_VALUE_LEN 4 #define VALUE "value" #define VALUE_LEN 5 @@ -402,7 +398,7 @@ static int32_t smlApplySchemaAction(SSmlHandle* info, SSchemaAction* action) { } if(taosArrayGetSize(cols) == 0){ outBytes = snprintf(pos, freeBytes,"`%s` %s(%d)", - TAG, tDataTypes[TSDB_DATA_TYPE_NCHAR].name, CHAR_SAVE_LENGTH); + tsSmlTagName, tDataTypes[TSDB_DATA_TYPE_NCHAR].name, CHAR_SAVE_LENGTH); pos += outBytes; freeBytes -= outBytes; *pos = ','; ++pos; --freeBytes; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d74d5a4d4e..bd7a04c991 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -78,6 +78,7 @@ char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com"; uint16_t tsTelemPort = 80; // schemaless +char tsSmlTagName[TSDB_COL_NAME_LEN] = "__tag"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; //user defined child table name can be specified in tag value. //If set to empty system will generate table name using MD5 hash. bool tsSmlDataFormat = true; // true means that the name and order of cols in each line are the same(only for influx protocol) @@ -522,6 +523,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { } tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); + tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; From db93c49ab0b67bd16213fdaa267ea933aef891b4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 21 May 2022 16:53:25 +0800 Subject: [PATCH 16/73] fea: add null tag support in schemaless --- source/client/test/smlTest.cpp | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 2165189153..217699e360 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -269,16 +269,7 @@ TEST(testCase, smlParseCols_tag_Test) { ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); size = taosArrayGetSize(cols); - ASSERT_EQ(size, 1); - - // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 0); - ASSERT_EQ(strncasecmp(kv->key, TAG, TAG_LEN), 0); - ASSERT_EQ(kv->keyLen, TAG_LEN); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, TAG_LEN); - ASSERT_EQ(strncasecmp(kv->value, TAG_VALUE, TAG_VALUE_LEN), 0); - taosMemoryFree(kv); + ASSERT_EQ(size, 0); taosArrayDestroy(cols); taosHashCleanup(dumplicateKey); From f8d8beadebbe44b186126ef35112bc40c3e578cc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 21 May 2022 16:54:56 +0800 Subject: [PATCH 17/73] refactor: add assert in hash table. --- source/util/src/thash.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 4bbf6ccf45..8ae0a33f0f 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -192,12 +192,15 @@ static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SH atomic_sub_fetch_16(&pNode->refCount, 1); if (prev != NULL) { prev->next = pNewNode; + ASSERT(prev->next != prev); } else { pe->next = pNewNode; } if (pNode->refCount <= 0) { pNewNode->next = pNode->next; + ASSERT(pNewNode->next != pNewNode); + FREE_HASH_NODE(pHashObj->freeFp, pNode); } else { pNewNode->next = pNode; @@ -525,6 +528,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { pe->next = pNode->next; } else { prevNode->next = pNode->next; + ASSERT(prevNode->next != prevNode); } pe->num--; @@ -719,6 +723,7 @@ void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) { pNode->next = pEntry->next; pEntry->next = pNode; + ASSERT(pNode->next != pNode); pEntry->num += 1; } From ff615a066c41cf2ed2990048fc044181710cf166 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 21 May 2022 16:56:27 +0800 Subject: [PATCH 18/73] refactor: add assert in hash. --- source/util/src/thash.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 8ae0a33f0f..89fc446da9 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -774,8 +774,11 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { if (pOld->refCount <=0) { if (prevNode) { prevNode->next = pOld->next; + ASSERT(prevNode->next != prevNode); } else { pe->next = pOld->next; + SHashNode* x = pe->next; + ASSERT(x->next != x); } pe->num--; From 1a5be99bd476c5fbec7181dbfd5239f209b6880a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 21 May 2022 17:01:18 +0800 Subject: [PATCH 19/73] fix(sync): syncRespMgrGetAndDel --- source/libs/sync/src/syncRespMgr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 47c73745fd..642e572434 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -76,8 +76,8 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index)); if (pTmp != NULL) { memcpy(pStub, pTmp, sizeof(SRespStub)); - taosThreadMutexUnlock(&(pObj->mutex)); taosHashRemove(pObj->pRespHash, &index, sizeof(index)); + taosThreadMutexUnlock(&(pObj->mutex)); return 1; // get one object } taosThreadMutexUnlock(&(pObj->mutex)); @@ -90,4 +90,4 @@ void syncRespClean(SSyncRespMgr *pObj) { taosThreadMutexUnlock(&(pObj->mutex)); } -void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) {} \ No newline at end of file +void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) {} From 0f0060f8c3a3a5de1580011f5790171846755e5d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 21 May 2022 17:04:13 +0800 Subject: [PATCH 20/73] fix: add a null ptr check. --- source/util/src/thash.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 89fc446da9..18404f8bde 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -778,7 +778,9 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { } else { pe->next = pOld->next; SHashNode* x = pe->next; - ASSERT(x->next != x); + if (x != NULL) { + ASSERT(x->next != x); + } } pe->num--; From bbb30eb599c74bb17ade947a3c2c8da28920f5ab Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 21 May 2022 17:07:56 +0800 Subject: [PATCH 21/73] fix case issue --- tests/script/tsim/stable/metrics.sim | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/script/tsim/stable/metrics.sim b/tests/script/tsim/stable/metrics.sim index c49de0e803..e68d95511c 100644 --- a/tests/script/tsim/stable/metrics.sim +++ b/tests/script/tsim/stable/metrics.sim @@ -28,15 +28,17 @@ if $rows != 1 then return -1 endi -print =============== step2 -sql drop table $mt -sql show stables -if $rows != 0 then - return -1 -endi +#TODO OPEN THIS WHEN STABLE DELETE WORKS +#print =============== step2 +#sql drop table $mt +#sql show stables +#if $rows != 0 then +# return -1 +#endi -print =============== step3 -sql create table $mt (ts timestamp, speed int) TAGS(sp int) +#print =============== step3 +#sql create table $mt (ts timestamp, speed int) TAGS(sp int) +#TODO OPEN THIS WHEN STABLE DELETE WORKS sql show stables if $rows != 1 then @@ -134,4 +136,4 @@ if $rows != 2 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT From d14429f3491639e3b591d8512401e1d4a72a3956 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 17:11:24 +0800 Subject: [PATCH 22/73] fix: santizer error in sim --- tests/tsim/src/simSystem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 969332ba5f..2171c8f701 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -92,7 +92,7 @@ SScript *simProcessCallOver(SScript *script) { simScriptPos--; simFreeScript(script); - if (simScriptPos == -1 && simExecSuccess) { + if (simScriptPos == -1 || simExecSuccess) { simInfo("----------------------------------------------------------------------"); simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced); return NULL; From 011169b108bb6a5927dbf321c3dd790d7f4cc8af Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:12:37 +0800 Subject: [PATCH 23/73] fix case --- tests/system-test/0-others/user_control.py | 260 ++++++++++++++++----- 1 file changed, 199 insertions(+), 61 deletions(-) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index 48058af295..d781b24cbb 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -1,7 +1,9 @@ +from tabnanny import check import taos -import sys +import time import inspect import traceback +from dataclasses import dataclass from util.log import * from util.sql import * @@ -12,6 +14,10 @@ PRIVILEGES_ALL = "ALL" PRIVILEGES_READ = "READ" PRIVILEGES_WRITE = "WRITE" +WEIGHT_ALL = 5 +WEIGHT_READ = 2 +WEIGHT_WRITE = 3 + PRIMARY_COL = "ts" INT_COL = "c1" @@ -94,6 +100,7 @@ class TDconnect: self.cursor.close() self._conn.close() + def taos_connect( host = "127.0.0.1", port = 6030, @@ -111,6 +118,15 @@ def taos_connect( config=config ) + +@dataclass +class User: + name : str = None + passwd : str = None + db_set : set = set() + priv : str = None + priv_weight : int = 0 + class TDTestCase: def init(self, conn, logSql): @@ -121,6 +137,21 @@ class TDTestCase: def __user_list(self): return [f"user_test{i}" for i in range(self.users_count) ] + def __users(self): + self.users = [] + self.root_user = User() + self.root_user.name = "root" + self.root_user.passwd = "passwd" + self.root_user.db_set = set("*") + self.root_user.priv = PRIVILEGES_ALL + self.root_user.priv_weight = WEIGHT_ALL + for i in range(self.users_count): + user = User() + user.name = f"user_test{i}" + user.passwd = f"taosdata{i}" + self.users.append(user) + return self.users + @property def __passwd_list(self): return [f"taosdata{i}" for i in range(self.users_count) ] @@ -205,76 +236,166 @@ class TDTestCase: def __grant_user_privileges(self, privilege, dbname=None, user_name="root"): return f"GRANT {privilege} ON {self.__priv_level(dbname)} TO {user_name} " - def grant_check(self, user="root", passwd="taosdata", priv=PRIVILEGES_ALL): - with taos_connect(user=user, passwd=passwd) as user: - user.query("use db") - user.query("show tables") - if priv in [PRIVILEGES_ALL, PRIVILEGES_READ]: - user.query("select * from ct1") - else: - user.error("select * from ct1") - if priv in [PRIVILEGES_ALL, PRIVILEGES_WRITE]: - user.query("insert into t1 (ts) values (now())") - else: - user.error("insert into t1 (ts) values (now())") + def __revoke_user_privileges(self, privilege, dbname=None, user_name="root"): + return f"REVOKE {privilege} ON {self.__priv_level(dbname)} FROM {user_name} " - def test_grant_current(self): + def __user_check(self, user:User=None, check_priv=PRIVILEGES_ALL): + if user is None: + user = self.root_user + with taos_connect(user=user.name, passwd=user.passwd) as use: + use.query("use db") + use.query("show tables") + if check_priv == PRIVILEGES_ALL: + use.query("select * from ct1") + use.query("insert into t1 (ts) values (now())") + elif check_priv == PRIVILEGES_READ: + use.query("select * from ct1") + use.error("insert into t1 (ts) values (now())") + elif check_priv == PRIVILEGES_WRITE: + use.error("select * from ct1") + use.query("insert into t1 (ts) values (now())") + elif check_priv is None: + use.error("select * from ct1") + use.error("insert into t1 (ts) values (now())") + + def __change_user_priv(self, user: User, pre_priv, invoke=False): + if user.priv == pre_priv and invoke : + return + if pre_priv.upper() == PRIVILEGES_ALL: + pre_weight = -5 if invoke else 5 + elif pre_priv.upper() == PRIVILEGES_READ: + pre_weight = -2 if invoke else 2 + elif pre_priv.upper() == PRIVILEGES_WRITE: + pre_weight = -3 if invoke else 3 + else: + return + pre_weight += user.priv_weight + + if pre_weight >= 5: + user.priv = PRIVILEGES_ALL + user.priv_weight = 5 + elif pre_weight == 3: + user.priv = PRIVILEGES_WRITE + user.priv_weight = pre_weight + elif pre_weight == 2: + user.priv_weight = pre_weight + user.priv = PRIVILEGES_READ + elif pre_weight in [1, -1]: + return + elif pre_weight <= 0: + user.priv_weight = 0 + user.priv = "" + + return user + + def grant_user(self, user: User = None, priv=PRIVILEGES_ALL, dbname=None): + if not user: + user = self.root_user + sql = self.__grant_user_privileges(privilege=priv, dbname=dbname, user_name=user.name) + tdLog.info(sql) + if user not in self.users or user.name != "root" or priv not in (PRIVILEGES_ALL, PRIVILEGES_READ, PRIVILEGES_WRITE): + tdSql.error(sql) + tdSql.query(sql) + self.__change_user_priv(user=user, pre_priv=priv) + user.db_set.add(dbname) + time.sleep(2) + + def revoke_user(self, user: User = None, priv=PRIVILEGES_ALL, dbname=None): + sql = self.__revoke_user_privileges(privilege=priv, dbname=dbname, user_name=user.name) + tdLog.info(sql) + if not user or priv not in(): + tdSql.error(sql) + tdSql.query(sql) + if user.name == "root": + return + self.__change_user_priv(user=user, pre_priv=priv, invoke=True) + user.db_set.remove(dbname) + time.sleep(2) + + def test_priv_change_current(self): tdLog.printNoPrefix("==========step 1.0: if do not grant, can not read/write") - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=None) + self.__user_check() + self.__user_check(user=self.users[0], check_priv=None) tdLog.printNoPrefix("==========step 1.1: grant read, can read, can not write") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_READ, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_READ) + self.grant_user(user=self.users[0], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_READ) - tdLog.printNoPrefix("==========step 1.2: grant write, can write, can not read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_WRITE, user_name=self.__user_list[1]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[1], passwd=self.__passwd_list[1], priv=PRIVILEGES_WRITE) + tdLog.printNoPrefix("==========step 1.2: grant write, can write") + self.grant_user(user=self.users[1], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_WRITE) tdLog.printNoPrefix("==========step 1.3: grant all, can write and read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_ALL, user_name=self.__user_list[2]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[2], passwd=self.__passwd_list[2], priv=PRIVILEGES_ALL) + self.grant_user(user=self.users[2]) + self.__user_check(user=self.users[2], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.4: change grant read to write, can write , can not read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_WRITE, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_WRITE) + tdLog.printNoPrefix("==========step 1.4: grant read to write = all ") + self.grant_user(user=self.users[0], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.5: change grant write to read, can not write , can read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_READ, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_READ) + tdLog.printNoPrefix("==========step 1.4: revoke write from all = read ") + self.revoke_user(user=self.users[0], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_READ) - tdLog.printNoPrefix("==========step 1.6: change grant read to all, can write , can read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_ALL, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_ALL) + tdLog.printNoPrefix("==========step 1.5: grant write to read = all") + self.grant_user(user=self.users[1], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.7: change grant all to write, can write , can not read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_WRITE, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_WRITE) + tdLog.printNoPrefix("==========step 1.4: revoke read from all = write ") + self.revoke_user(user=self.users[1], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_WRITE) - tdLog.printNoPrefix("==========step 1.8: change grant write to all, can write , can read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_ALL, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_ALL) + tdLog.printNoPrefix("==========step 1.5: grant read to all = all") + self.grant_user(user=self.users[0], priv=PRIVILEGES_ALL) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.9: change grant all to read, can not write , can read") - sql = self.__grant_user_privileges(privilege=PRIVILEGES_READ, user_name=self.__user_list[0]) - tdLog.info(sql) - tdSql.query(sql) - self.grant_check(user=self.__user_list[0], passwd=self.__passwd_list[0], priv=PRIVILEGES_READ) + tdLog.printNoPrefix("==========step 1.5: grant write to all = all") + self.grant_user(user=self.users[1], priv=PRIVILEGES_ALL) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_ALL) + + tdLog.printNoPrefix("==========step 1.5: grant all to read = all") + self.grant_user(user=self.users[0], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_ALL) + + tdLog.printNoPrefix("==========step 1.5: grant all to write = all") + self.grant_user(user=self.users[1], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_ALL) + + ### init user + self.revoke_user(user=self.users[0], priv=PRIVILEGES_WRITE) + self.revoke_user(user=self.users[1], priv=PRIVILEGES_READ) + + tdLog.printNoPrefix("==========step 1.5: revoke read from write = no change") + self.revoke_user(user=self.users[1], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[1], check_priv=PRIVILEGES_WRITE) + + tdLog.printNoPrefix("==========step 1.5: revoke write from read = no change") + self.revoke_user(user=self.users[0], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[0], check_priv=PRIVILEGES_READ) + + tdLog.printNoPrefix("==========step 1.5: revoke read from read = nothing") + self.revoke_user(user=self.users[0], priv=PRIVILEGES_READ) + self.__user_check(user=self.users[0], check_priv=None) + + tdLog.printNoPrefix("==========step 1.5: revoke write from write = nothing") + self.revoke_user(user=self.users[1], priv=PRIVILEGES_WRITE) + self.__user_check(user=self.users[1], check_priv=None) + + ### init user + self.grant_user(user=self.users[0], priv=PRIVILEGES_READ) + self.revoke_user(user=self.users[1], priv=PRIVILEGES_WRITE) + + tdLog.printNoPrefix("==========step 1.5: revoke all from write = nothing") + self.revoke_user(user=self.users[1], priv=PRIVILEGES_ALL) + self.__user_check(user=self.users[1], check_priv=None) + + tdLog.printNoPrefix("==========step 1.5: revoke all from read = nothing") + self.revoke_user(user=self.users[0], priv=PRIVILEGES_ALL) + self.__user_check(user=self.users[0], check_priv=None) + + tdLog.printNoPrefix("==========step 1.5: revoke all from all = nothing") + self.revoke_user(user=self.users[2], priv=PRIVILEGES_ALL) + self.__user_check(user=self.users[2], check_priv=None) def __grant_err(self): return [ @@ -288,13 +409,30 @@ class TDTestCase: f"GRANT {self.__privilege[0]} ON db.t1 TO {self.__user_list[0]}" , ] + def __revoke_err(self): + return [ + self.__revoke_user_privileges(privilege=self.__privilege[0], user_name="") , + self.__revoke_user_privileges(privilege=self.__privilege[0], user_name="*") , + self.__revoke_user_privileges(privilege=self.__privilege[1], dbname="not_exist_db", user_name=self.__user_list[0]), + self.__revoke_user_privileges(privilege="any_priv", user_name=self.__user_list[0]), + self.__revoke_user_privileges(privilege="", dbname="db", user_name=self.__user_list[0]) , + self.__revoke_user_privileges(privilege=" ".join(self.__privilege), user_name=self.__user_list[0]) , + f"REVOKE {self.__privilege[0]} ON * FROM {self.__user_list[0]}" , + f"REVOKE {self.__privilege[0]} ON db.t1 FROM {self.__user_list[0]}" , + ] + def test_grant_err(self): for sql in self.__grant_err(): tdSql.error(sql) - def test_grant(self): + def test_revoke_err(self): + for sql in self.__revoke_err(): + tdSql.error(sql) + + def test_change_priv(self): self.test_grant_err() - self.test_grant_current() + self.test_revoke_err() + self.test_priv_change_current() def test_user_create(self): self.create_user_current() @@ -482,7 +620,7 @@ class TDTestCase: self.login_err(self.__user_list[0], f"new{self.__passwd_list[0]}") # 用户权限设置 - self.test_grant() + self.test_change_priv() # 修改密码 tdLog.printNoPrefix("==========step3: alter user pass test") From d20bd3e4b63df4a67def75ee980405b38012e5b0 Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:22:43 +0800 Subject: [PATCH 24/73] fix case --- tests/system-test/0-others/user_control.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index d781b24cbb..25170c6a32 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -123,7 +123,7 @@ def taos_connect( class User: name : str = None passwd : str = None - db_set : set = set() + db_set : list = [] priv : str = None priv_weight : int = 0 @@ -142,7 +142,7 @@ class TDTestCase: self.root_user = User() self.root_user.name = "root" self.root_user.passwd = "passwd" - self.root_user.db_set = set("*") + self.root_user.db_set = ["*"] self.root_user.priv = PRIVILEGES_ALL self.root_user.priv_weight = WEIGHT_ALL for i in range(self.users_count): @@ -297,7 +297,7 @@ class TDTestCase: tdSql.error(sql) tdSql.query(sql) self.__change_user_priv(user=user, pre_priv=priv) - user.db_set.add(dbname) + user.db_set = list(set(user.db_set.append(dbname))) time.sleep(2) def revoke_user(self, user: User = None, priv=PRIVILEGES_ALL, dbname=None): From d31287840273562ce74fc1d934e4f67d419c172a Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:24:07 +0800 Subject: [PATCH 25/73] fix case --- tests/system-test/0-others/user_control.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index 25170c6a32..8843eba5a5 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -123,7 +123,7 @@ def taos_connect( class User: name : str = None passwd : str = None - db_set : list = [] + db_set : set = {} priv : str = None priv_weight : int = 0 @@ -142,7 +142,7 @@ class TDTestCase: self.root_user = User() self.root_user.name = "root" self.root_user.passwd = "passwd" - self.root_user.db_set = ["*"] + self.root_user.db_set = set("*") self.root_user.priv = PRIVILEGES_ALL self.root_user.priv_weight = WEIGHT_ALL for i in range(self.users_count): @@ -297,7 +297,7 @@ class TDTestCase: tdSql.error(sql) tdSql.query(sql) self.__change_user_priv(user=user, pre_priv=priv) - user.db_set = list(set(user.db_set.append(dbname))) + user.db_set.add(dbname) time.sleep(2) def revoke_user(self, user: User = None, priv=PRIVILEGES_ALL, dbname=None): From 8fdaa1c12ff1c9101eac4da00c40b8333ec5e71e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 21 May 2022 17:29:27 +0800 Subject: [PATCH 26/73] fea: add null tag support in schemaless --- source/common/src/tglobal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index bd7a04c991..b8dd33b915 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -327,6 +327,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; + if (cfgAddString(pCfg, "smlTagName", "", 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 4; From 4d18fa0875720159f115d8e509b74719ffe41b53 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 21 May 2022 17:29:50 +0800 Subject: [PATCH 27/73] enh: refator index/transport code --- source/libs/executor/src/indexoperator.c | 44 +++++++----------------- source/libs/index/inc/indexInt.h | 12 +++---- source/libs/index/src/index.c | 7 ++-- source/libs/index/src/indexCache.c | 2 +- source/libs/index/src/indexTfile.c | 9 +++-- source/libs/transport/inc/transComm.h | 31 ----------------- source/libs/transport/inc/transportInt.h | 6 ++-- source/libs/transport/src/transSrv.c | 4 +-- 8 files changed, 31 insertions(+), 84 deletions(-) diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/executor/src/indexoperator.c index 2c204e9356..123c77ce9b 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/executor/src/indexoperator.c @@ -19,38 +19,11 @@ #include "nodes.h" #include "tdatablock.h" -typedef struct SIFCtx { - int32_t code; - SHashObj *pRes; /* element is SScalarParam */ - bool noExec; // true: just iterate condition tree, and add hint to executor plan - // SIdxFltStatus st; -} SIFCtx; - -#define SIF_ERR_RET(c) \ - do { \ - int32_t _code = c; \ - if (_code != TSDB_CODE_SUCCESS) { \ - terrno = _code; \ - return _code; \ - } \ - } while (0) -#define SIF_RET(c) \ - do { \ - int32_t _code = c; \ - if (_code != TSDB_CODE_SUCCESS) { \ - terrno = _code; \ - } \ - return _code; \ - } while (0) -#define SIF_ERR_JRET(c) \ - do { \ - code = c; \ - if (code != TSDB_CODE_SUCCESS) { \ - terrno = code; \ - goto _return; \ - } \ - } while (0) - +// clang-format off +#define SIF_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define SIF_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define SIF_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) +// clang-format on typedef struct SIFParam { SHashObj *pFilter; @@ -65,6 +38,13 @@ typedef struct SIFParam { char colName[TSDB_COL_NAME_LEN]; } SIFParam; +typedef struct SIFCtx { + int32_t code; + SHashObj *pRes; /* element is SIFParam */ + bool noExec; // true: just iterate condition tree, and add hint to executor plan + // SIdxFltStatus st; +} SIFCtx; + static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { if (src == OP_TYPE_GREATER_THAN) { *dst = QUERY_GREATER_THAN; diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 3c05f38d05..5cb60abe7d 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -132,12 +132,12 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf); // int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); // clang-format off -#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while (0) -#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while (0) -#define indexWarn(...) do { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while (0) -#define indexInfo(...) do { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", DEBUG_INFO, 255, __VA_ARGS__); } } while (0) -#define indexDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__);} } while (0) -#define indexTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__);} } while (0) +#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("INDEX FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while (0) +#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("INDEX ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while (0) +#define indexWarn(...) do { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("INDEX WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while (0) +#define indexInfo(...) do { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("INDEX ", DEBUG_INFO, 255, __VA_ARGS__); } } while (0) +#define indexDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("INDEX ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__);} } while (0) +#define indexTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("INDEX ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__);} } while (0) // clang-format on #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 162d64c41c..92159dae7a 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -563,10 +563,11 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(key->colType, TSDB_DATA_TYPE_JSON); char* p = buf; - SERIALIZE_MEM_TO_BUF(buf, key, suid); + char tbuf[65] = {0}; + indexInt2str((int64_t)key->suid, tbuf, 0); + + SERIALIZE_STR_VAR_TO_BUF(buf, tbuf, strlen(tbuf)); SERIALIZE_VAR_TO_BUF(buf, '_', char); - // SERIALIZE_MEM_TO_BUF(buf, key, colType); - // SERIALIZE_VAR_TO_BUF(buf, '_', char); if (hasJson) { SERIALIZE_STR_VAR_TO_BUF(buf, JSON_COLUMN, strlen(JSON_COLUMN)); } else { diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 65986199d4..5f7cc10b99 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -385,7 +385,7 @@ void indexCacheDebug(IndexCache* cache) { void indexCacheDestroySkiplist(SSkipList* slt) { SSkipListIterator* iter = tSkipListCreateIter(slt); - while (tSkipListIterNext(iter)) { + while (iter != NULL && tSkipListIterNext(iter)) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); if (ct != NULL) { diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index dd6117ed2a..1cc91fe880 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -172,7 +172,6 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { oldReader->remove = true; tfileReaderUnRef(oldReader); } - taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); tfileReaderRef(reader); return; @@ -500,15 +499,15 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) { SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; - + int ret = 0; if (INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON)) { - return tfSearch[1][qtype](reader, term, tr); + ret = tfSearch[1][qtype](reader, term, tr); } else { - return tfSearch[0][qtype](reader, term, tr); + ret = tfSearch[0][qtype](reader, term, tr); } tfileReaderUnRef(reader); - return 0; + return ret; } TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t colType) { diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 3f82d6e2d8..e71e19edce 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -20,22 +20,12 @@ extern "C" { #endif #include -#include "lz4.h" #include "os.h" -#include "osSocket.h" #include "taoserror.h" -#include "tglobal.h" -#include "thash.h" #include "theap.h" -#include "tidpool.h" -#include "tmd5.h" -#include "tmempool.h" -#include "tmsg.h" #include "transLog.h" #include "transportInt.h" -#include "tref.h" #include "trpc.h" -#include "ttimer.h" #include "tutil.h" typedef void* queue[2]; @@ -108,27 +98,6 @@ typedef void* queue[2]; #define TRANS_RETRY_INTERVAL 15 // ms retry interval #define TRANS_CONN_TIMEOUT 3 // connect timeout -typedef struct { - SRpcInfo* pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app - // struct SRpcConn* pConn; // pConn allocated - tmsg_t msgType; // message type - uint8_t* pCont; // content provided by app - int32_t contLen; // content length - // int32_t code; // error code - // int16_t numOfTry; // number of try for different servers - // int8_t oldInUse; // server EP inUse passed by app - // int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type - int64_t rid; // refId returned by taosAddRef - SRpcMsg* pRsp; // for synchronous API - tsem_t* pSem; // for synchronous API - char* ip; - uint32_t port; - // SEpSet* pSet; // for synchronous API -} SRpcReqContext; - typedef SRpcMsg STransMsg; typedef SRpcCtx STransCtx; typedef SRpcCtxVal STransCtxVal; diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index a498571f33..8aeae1b5ad 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -22,15 +22,13 @@ #include "lz4.h" #include "os.h" #include "taoserror.h" -#include "tglobal.h" #include "thash.h" -#include "tidpool.h" +#include "tref.h" #include "tmsg.h" #include "transLog.h" -#include "tref.h" #include "trpc.h" -#include "ttimer.h" #include "tutil.h" +#include "tglobal.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index da83a6f37f..36f5cf9815 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -295,14 +295,14 @@ static void uvHandleReq(SSrvConn* pConn) { // no ref here } - // if pHead->noResp = 1, + // pHead->noResp = 1, // 1. server application should not send resp on handle // 2. once send out data, cli conn released to conn pool immediately // 3. not mixed with persist transMsg.info.handle = (void*)uvAcquireExHandle(pConn->refId); - tTrace("server handle %p conn: %p translated to app, refId: %" PRIu64 "", transMsg.info.handle, pConn, pConn->refId); transMsg.info.refId = pConn->refId; + tTrace("server handle %p conn: %p translated to app, refId: %" PRIu64 "", transMsg.info.handle, pConn, pConn->refId); assert(transMsg.info.handle != NULL); if (pHead->noResp == 1) { transMsg.info.refId = -1; From 1146196fd30333d32a39cfb3efdbdeb990b1ad1b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 17:29:58 +0800 Subject: [PATCH 28/73] fix: santizer error in sim --- tests/tsim/src/simSystem.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 2171c8f701..1c751f290a 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -92,12 +92,14 @@ SScript *simProcessCallOver(SScript *script) { simScriptPos--; simFreeScript(script); - if (simScriptPos == -1 || simExecSuccess) { + if (simScriptPos == -1 && simExecSuccess) { simInfo("----------------------------------------------------------------------"); simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced); return NULL; } + if (simScriptPos == -1) return NULL; + return simScriptList[simScriptPos]; } else { simDebug("script:%s, is stopped", script->fileName); From 70c3df86b84b59d9ce44467675135c5f1fb65498 Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:32:46 +0800 Subject: [PATCH 29/73] fix case --- tests/system-test/0-others/user_control.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index 8843eba5a5..c437568ea2 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -123,7 +123,7 @@ def taos_connect( class User: name : str = None passwd : str = None - db_set : set = {} + db_set : set = None priv : str = None priv_weight : int = 0 @@ -141,7 +141,7 @@ class TDTestCase: self.users = [] self.root_user = User() self.root_user.name = "root" - self.root_user.passwd = "passwd" + self.root_user.passwd = "taosdata" self.root_user.db_set = set("*") self.root_user.priv = PRIVILEGES_ALL self.root_user.priv_weight = WEIGHT_ALL @@ -149,6 +149,7 @@ class TDTestCase: user = User() user.name = f"user_test{i}" user.passwd = f"taosdata{i}" + user.db_set = {} self.users.append(user) return self.users @@ -314,7 +315,7 @@ class TDTestCase: def test_priv_change_current(self): tdLog.printNoPrefix("==========step 1.0: if do not grant, can not read/write") - self.__user_check() + self.__user_check(user=self.root_user) self.__user_check(user=self.users[0], check_priv=None) tdLog.printNoPrefix("==========step 1.1: grant read, can read, can not write") @@ -593,6 +594,7 @@ class TDTestCase: tdSql.prepare() self.__create_tb() self.rows = 10 + self.users_count = 5 self.__insert_data(self.rows) tdDnodes.stop(1) @@ -607,7 +609,6 @@ class TDTestCase: # root用户权限 # 创建用户测试 tdLog.printNoPrefix("==========step1: create user test") - self.users_count = 5 self.test_user_create() # 查看用户 From 8d6156f662b663fab95fba8624dddcd1313b8019 Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:32:53 +0800 Subject: [PATCH 30/73] fix case --- tests/system-test/0-others/user_control.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index c437568ea2..3c71f13dbc 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -596,6 +596,7 @@ class TDTestCase: self.rows = 10 self.users_count = 5 self.__insert_data(self.rows) + self.users = self.__users() tdDnodes.stop(1) tdDnodes.start(1) From 1f42c36e98994cf06eefbe7bbb6ffee3a6d45acd Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:34:42 +0800 Subject: [PATCH 31/73] fix case --- tests/system-test/0-others/user_control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index 3c71f13dbc..c72d6df259 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -294,7 +294,7 @@ class TDTestCase: user = self.root_user sql = self.__grant_user_privileges(privilege=priv, dbname=dbname, user_name=user.name) tdLog.info(sql) - if user not in self.users or user.name != "root" or priv not in (PRIVILEGES_ALL, PRIVILEGES_READ, PRIVILEGES_WRITE): + if (user not in self.users and user.name != "root") or priv not in (PRIVILEGES_ALL, PRIVILEGES_READ, PRIVILEGES_WRITE): tdSql.error(sql) tdSql.query(sql) self.__change_user_priv(user=user, pre_priv=priv) From fc9c22b89c4f08f2b496f854efbd41423a524ebb Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 21 May 2022 17:35:14 +0800 Subject: [PATCH 32/73] fix(stream): array init error --- source/libs/stream/src/tstreamUpdate.c | 44 ++++++++++++-------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/source/libs/stream/src/tstreamUpdate.c b/source/libs/stream/src/tstreamUpdate.c index 1197b6100a..7921193fa2 100644 --- a/source/libs/stream/src/tstreamUpdate.c +++ b/source/libs/stream/src/tstreamUpdate.c @@ -17,21 +17,20 @@ #include "ttime.h" #define DEFAULT_FALSE_POSITIVE 0.01 -#define DEFAULT_BUCKET_SIZE 1024 -#define ROWS_PER_MILLISECOND 1 -#define MAX_NUM_SCALABLE_BF 120 -#define MIN_NUM_SCALABLE_BF 10 -#define DEFAULT_PREADD_BUCKET 1 -#define MAX_INTERVAL MILLISECOND_PER_MINUTE -#define MIN_INTERVAL (MILLISECOND_PER_SECOND * 10) +#define DEFAULT_BUCKET_SIZE 1024 +#define ROWS_PER_MILLISECOND 1 +#define MAX_NUM_SCALABLE_BF 120 +#define MIN_NUM_SCALABLE_BF 10 +#define DEFAULT_PREADD_BUCKET 1 +#define MAX_INTERVAL MILLISECOND_PER_MINUTE +#define MIN_INTERVAL (MILLISECOND_PER_SECOND * 10) static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { - if (pInfo->numSBFs < count ) { + if (pInfo->numSBFs < count) { count = pInfo->numSBFs; } for (uint64_t i = 0; i < count; ++i) { - SScalableBf *tsSBF = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, - DEFAULT_FALSE_POSITIVE); + SScalableBf *tsSBF = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, DEFAULT_FALSE_POSITIVE); taosArrayPush(pInfo->pTsSBFs, &tsSBF); } } @@ -76,7 +75,7 @@ static int64_t adjustWatermark(int64_t interval, int32_t watermark) { return watermark; } -SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark) { +SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark) { return updateInfoInit(pInterval->interval, pInterval->precision, watermark); } @@ -93,7 +92,7 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma uint64_t bfSize = (uint64_t)(pInfo->watermark / pInfo->interval); - pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(SScalableBf)); + pInfo->pTsSBFs = taosArrayInit(bfSize, sizeof(void *)); if (pInfo->pTsSBFs == NULL) { updateInfoDestroy(pInfo); return NULL; @@ -108,14 +107,14 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } TSKEY dumy = 0; - for(uint64_t i=0; i < DEFAULT_BUCKET_SIZE; ++i) { + for (uint64_t i = 0; i < DEFAULT_BUCKET_SIZE; ++i) { taosArrayPush(pInfo->pTsBuckets, &dumy); } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; return pInfo; } -static SScalableBf* getSBf(SUpdateInfo *pInfo, TSKEY ts) { +static SScalableBf *getSBf(SUpdateInfo *pInfo, TSKEY ts) { if (ts <= 0) { return NULL; } @@ -131,24 +130,23 @@ static SScalableBf* getSBf(SUpdateInfo *pInfo, TSKEY ts) { } SScalableBf *res = taosArrayGetP(pInfo->pTsSBFs, index); if (res == NULL) { - res = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, - DEFAULT_FALSE_POSITIVE); + res = tScalableBfInit(pInfo->interval * ROWS_PER_MILLISECOND, DEFAULT_FALSE_POSITIVE); taosArrayPush(pInfo->pTsSBFs, &res); } return res; } bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) { - int32_t res = TSDB_CODE_FAILED; - uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; - SScalableBf* pSBf = getSBf(pInfo, ts); + int32_t res = TSDB_CODE_FAILED; + uint64_t index = ((uint64_t)tableId) % pInfo->numBuckets; + SScalableBf *pSBf = getSBf(pInfo, ts); // pSBf may be a null pointer if (pSBf) { res = tScalableBfPut(pSBf, &ts, sizeof(TSKEY)); } TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index); - if (maxTs < ts ) { + if (maxTs < ts) { taosArraySet(pInfo->pTsBuckets, index, &ts); return false; } @@ -159,7 +157,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) { return false; } - //check from tsdb api + // check from tsdb api return true; } @@ -174,7 +172,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { SScalableBf *pSBF = taosArrayGetP(pInfo->pTsSBFs, i); tScalableBfDestroy(pSBF); } - + taosArrayDestroy(pInfo->pTsSBFs); taosMemoryFree(pInfo); -} \ No newline at end of file +} From d5fabd53757c84216faf12c5694db97c3c80b4a3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 17:38:09 +0800 Subject: [PATCH 33/73] refactor: rpc msg handler --- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 20 +-------------- source/dnode/mnode/impl/src/mndQuery.c | 27 +++++++++++++++------ 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index c6233a4f18..43ee7cd80a 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -56,24 +56,6 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { taosFreeQitem(pMsg); } -static void mmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SMnodeMgmt *pMgmt = pInfo->ahandle; - int32_t code = -1; - dTrace("msg:%p, get from mnode-query queue", pMsg); - - pMsg->info.node = pMgmt->pMnode; - code = mndProcessMsg(pMsg); - - if (IsReq(pMsg) && pMsg->info.handle != NULL && code != 0) { - if (terrno != 0) code = terrno; - mmSendRsp(pMsg, code); - } - - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); @@ -131,7 +113,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = tsNumOfMnodeQueryThreads, .max = tsNumOfMnodeQueryThreads, .name = "mnode-query", - .fp = (FItem)mmProcessQueryQueue, + .fp = (FItem)mmProcessQueue, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) { diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 5c7089e1aa..0c0d771a24 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -19,36 +19,49 @@ #include "qworker.h" int32_t mndProcessQueryMsg(SRpcMsg *pReq) { + int32_t code = -1; SMnode *pMnode = pReq->info.node; SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb}; mTrace("msg:%p, in query queue is processing", pReq); switch (pReq->msgType) { case TDMT_VND_QUERY: - return qWorkerProcessQueryMsg(&handle, pMnode->pQuery, pReq); + code = qWorkerProcessQueryMsg(&handle, pMnode->pQuery, pReq); + break; case TDMT_VND_QUERY_CONTINUE: - return qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pReq); + code = qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pReq); + break; default: + terrno = TSDB_CODE_VND_APP_ERROR; mError("unknown msg type:%d in query queue", pReq->msgType); - return TSDB_CODE_VND_APP_ERROR; } + + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + return code; } int32_t mndProcessFetchMsg(SRpcMsg *pMsg) { + int32_t code = -1; SMnode *pMnode = pMsg->info.node; mTrace("msg:%p, in fetch queue is processing", pMsg); switch (pMsg->msgType) { case TDMT_VND_FETCH: - return qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, pMsg); + code = qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, pMsg); + break; case TDMT_VND_DROP_TASK: - return qWorkerProcessDropMsg(pMnode, pMnode->pQuery, pMsg); + code = qWorkerProcessDropMsg(pMnode, pMnode->pQuery, pMsg); + break; case TDMT_VND_QUERY_HEARTBEAT: - return qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg); + code = qWorkerProcessHbMsg(pMnode, pMnode->pQuery, pMsg); + break; default: + terrno = TSDB_CODE_VND_APP_ERROR; mError("unknown msg type:%d in fetch queue", pMsg->msgType); - return TSDB_CODE_VND_APP_ERROR; } + + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + return code; } int32_t mndInitQuery(SMnode *pMnode) { From 8e5884b51d72e21eddd7b79f32e137a3b48bc67c Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:38:33 +0800 Subject: [PATCH 34/73] fix case --- tests/system-test/0-others/user_control.py | 32 +++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index c72d6df259..17c29bf4a8 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -149,7 +149,7 @@ class TDTestCase: user = User() user.name = f"user_test{i}" user.passwd = f"taosdata{i}" - user.db_set = {} + user.db_set = set() self.users.append(user) return self.users @@ -304,7 +304,7 @@ class TDTestCase: def revoke_user(self, user: User = None, priv=PRIVILEGES_ALL, dbname=None): sql = self.__revoke_user_privileges(privilege=priv, dbname=dbname, user_name=user.name) tdLog.info(sql) - if not user or priv not in(): + if not user or priv not in (PRIVILEGES_ALL, PRIVILEGES_READ, PRIVILEGES_WRITE): tdSql.error(sql) tdSql.query(sql) if user.name == "root": @@ -334,31 +334,31 @@ class TDTestCase: self.grant_user(user=self.users[0], priv=PRIVILEGES_WRITE) self.__user_check(user=self.users[0], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.4: revoke write from all = read ") + tdLog.printNoPrefix("==========step 1.5: revoke write from all = read ") self.revoke_user(user=self.users[0], priv=PRIVILEGES_WRITE) self.__user_check(user=self.users[0], check_priv=PRIVILEGES_READ) - tdLog.printNoPrefix("==========step 1.5: grant write to read = all") + tdLog.printNoPrefix("==========step 1.6: grant write to read = all") self.grant_user(user=self.users[1], priv=PRIVILEGES_READ) self.__user_check(user=self.users[1], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.4: revoke read from all = write ") + tdLog.printNoPrefix("==========step 1.7: revoke read from all = write ") self.revoke_user(user=self.users[1], priv=PRIVILEGES_READ) self.__user_check(user=self.users[1], check_priv=PRIVILEGES_WRITE) - tdLog.printNoPrefix("==========step 1.5: grant read to all = all") + tdLog.printNoPrefix("==========step 1.8: grant read to all = all") self.grant_user(user=self.users[0], priv=PRIVILEGES_ALL) self.__user_check(user=self.users[0], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.5: grant write to all = all") + tdLog.printNoPrefix("==========step 1.9: grant write to all = all") self.grant_user(user=self.users[1], priv=PRIVILEGES_ALL) self.__user_check(user=self.users[1], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.5: grant all to read = all") + tdLog.printNoPrefix("==========step 1.10: grant all to read = all") self.grant_user(user=self.users[0], priv=PRIVILEGES_READ) self.__user_check(user=self.users[0], check_priv=PRIVILEGES_ALL) - tdLog.printNoPrefix("==========step 1.5: grant all to write = all") + tdLog.printNoPrefix("==========step 1.11: grant all to write = all") self.grant_user(user=self.users[1], priv=PRIVILEGES_WRITE) self.__user_check(user=self.users[1], check_priv=PRIVILEGES_ALL) @@ -366,19 +366,19 @@ class TDTestCase: self.revoke_user(user=self.users[0], priv=PRIVILEGES_WRITE) self.revoke_user(user=self.users[1], priv=PRIVILEGES_READ) - tdLog.printNoPrefix("==========step 1.5: revoke read from write = no change") + tdLog.printNoPrefix("==========step 1.12: revoke read from write = no change") self.revoke_user(user=self.users[1], priv=PRIVILEGES_READ) self.__user_check(user=self.users[1], check_priv=PRIVILEGES_WRITE) - tdLog.printNoPrefix("==========step 1.5: revoke write from read = no change") + tdLog.printNoPrefix("==========step 1.13: revoke write from read = no change") self.revoke_user(user=self.users[0], priv=PRIVILEGES_WRITE) self.__user_check(user=self.users[0], check_priv=PRIVILEGES_READ) - tdLog.printNoPrefix("==========step 1.5: revoke read from read = nothing") + tdLog.printNoPrefix("==========step 1.14: revoke read from read = nothing") self.revoke_user(user=self.users[0], priv=PRIVILEGES_READ) self.__user_check(user=self.users[0], check_priv=None) - tdLog.printNoPrefix("==========step 1.5: revoke write from write = nothing") + tdLog.printNoPrefix("==========step 1.15: revoke write from write = nothing") self.revoke_user(user=self.users[1], priv=PRIVILEGES_WRITE) self.__user_check(user=self.users[1], check_priv=None) @@ -386,15 +386,15 @@ class TDTestCase: self.grant_user(user=self.users[0], priv=PRIVILEGES_READ) self.revoke_user(user=self.users[1], priv=PRIVILEGES_WRITE) - tdLog.printNoPrefix("==========step 1.5: revoke all from write = nothing") + tdLog.printNoPrefix("==========step 1.16: revoke all from write = nothing") self.revoke_user(user=self.users[1], priv=PRIVILEGES_ALL) self.__user_check(user=self.users[1], check_priv=None) - tdLog.printNoPrefix("==========step 1.5: revoke all from read = nothing") + tdLog.printNoPrefix("==========step 1.17: revoke all from read = nothing") self.revoke_user(user=self.users[0], priv=PRIVILEGES_ALL) self.__user_check(user=self.users[0], check_priv=None) - tdLog.printNoPrefix("==========step 1.5: revoke all from all = nothing") + tdLog.printNoPrefix("==========step 1.18: revoke all from all = nothing") self.revoke_user(user=self.users[2], priv=PRIVILEGES_ALL) self.__user_check(user=self.users[2], check_priv=None) From 3a7347586e1fb3b2f4c0e86ce165008fad16c96d Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:39:12 +0800 Subject: [PATCH 35/73] fix case --- tests/system-test/0-others/user_control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index 17c29bf4a8..69e9753040 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -304,7 +304,7 @@ class TDTestCase: def revoke_user(self, user: User = None, priv=PRIVILEGES_ALL, dbname=None): sql = self.__revoke_user_privileges(privilege=priv, dbname=dbname, user_name=user.name) tdLog.info(sql) - if not user or priv not in (PRIVILEGES_ALL, PRIVILEGES_READ, PRIVILEGES_WRITE): + if user is None or priv not in (PRIVILEGES_ALL, PRIVILEGES_READ, PRIVILEGES_WRITE): tdSql.error(sql) tdSql.query(sql) if user.name == "root": From 1a8917cb5ffe64b7afa612d9bdcc162721ac2d9f Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:47:26 +0800 Subject: [PATCH 36/73] fix case --- tests/system-test/0-others/user_control.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index 69e9753040..a785060d52 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -262,6 +262,9 @@ class TDTestCase: def __change_user_priv(self, user: User, pre_priv, invoke=False): if user.priv == pre_priv and invoke : return + if user.name == "root": + return + if pre_priv.upper() == PRIVILEGES_ALL: pre_weight = -5 if invoke else 5 elif pre_priv.upper() == PRIVILEGES_READ: @@ -299,7 +302,7 @@ class TDTestCase: tdSql.query(sql) self.__change_user_priv(user=user, pre_priv=priv) user.db_set.add(dbname) - time.sleep(2) + time.sleep(1) def revoke_user(self, user: User = None, priv=PRIVILEGES_ALL, dbname=None): sql = self.__revoke_user_privileges(privilege=priv, dbname=dbname, user_name=user.name) @@ -307,11 +310,9 @@ class TDTestCase: if user is None or priv not in (PRIVILEGES_ALL, PRIVILEGES_READ, PRIVILEGES_WRITE): tdSql.error(sql) tdSql.query(sql) - if user.name == "root": - return self.__change_user_priv(user=user, pre_priv=priv, invoke=True) user.db_set.remove(dbname) - time.sleep(2) + time.sleep(1) def test_priv_change_current(self): tdLog.printNoPrefix("==========step 1.0: if do not grant, can not read/write") From 5ae72c15d4d7df8ffbd4900159d5f8d655ba8e03 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 17:47:29 +0800 Subject: [PATCH 37/73] refactor: rpc msg handler --- source/dnode/mgmt/mgmt_qnode/src/qmWorker.c | 71 ++++++--------------- source/dnode/qnode/src/qnode.c | 51 ++++++++++----- 2 files changed, 56 insertions(+), 66 deletions(-) diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 444c42717a..e30b683d19 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -19,70 +19,43 @@ static inline void qmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } -static void qmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +static void qmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SQnodeMgmt *pMgmt = pInfo->ahandle; + int32_t code = -1; + dTrace("msg:%p, get from qnode queue", pMsg); - dTrace("msg:%p, get from qnode-monitor queue", pMsg); - SRpcMsg *pRpc = pMsg; - int32_t code = -1; - - if (pMsg->msgType == TDMT_MON_QM_INFO) { - code = qmProcessGetMonitorInfoReq(pMgmt, pMsg); - } else { - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + switch (pMsg->msgType) { + case TDMT_MON_QM_INFO: + code = qmProcessGetMonitorInfoReq(pMgmt, pMsg); + break; + case TDMT_VND_QUERY: + case TDMT_VND_QUERY_CONTINUE: + code = qndProcessQueryMsg(pMgmt->pQnode, pMsg); + break; + default: + code = qndProcessFetchMsg(pMgmt->pQnode, pMsg); + break; } - if (pRpc->msgType & 1U) { + if (IsReq(pMsg) && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && terrno != 0) code = terrno; qmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pRpc->pCont); - taosFreeQitem(pMsg); -} - -static void qmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SQnodeMgmt *pMgmt = pInfo->ahandle; - - dTrace("msg:%p, get from qnode-query queue", pMsg); - SRpcMsg *pRpc = pMsg; - int32_t code = qndProcessQueryMsg(pMgmt->pQnode, pRpc); - - if (pRpc->msgType & 1U && code != 0) { - qmSendRsp(pMsg, code); - } - - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void qmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SQnodeMgmt *pMgmt = pInfo->ahandle; - - dTrace("msg:%p, get from qnode-fetch queue", pMsg); - SRpcMsg *pRpc = pMsg; - int32_t code = qndProcessFetchMsg(pMgmt->pQnode, pRpc); - - if (pRpc->msgType & 1U && code != 0) { - qmSendRsp(pMsg, code); - } - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } static int32_t qmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { - dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); return 0; } @@ -101,9 +74,7 @@ int32_t qmPutNodeMsgToMonitorQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) { SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - if (pMsg == NULL) { - return -1; - } + if (pMsg == NULL) return -1; dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); memcpy(pMsg, pRpc, sizeof(SRpcMsg)); @@ -141,7 +112,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { .min = tsNumOfVnodeQueryThreads, .max = tsNumOfVnodeQueryThreads, .name = "qnode-query", - .fp = (FItem)qmProcessQueryQueue, + .fp = (FItem)qmProcessQueue, .param = pMgmt, }; @@ -154,7 +125,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { .min = tsNumOfQnodeFetchThreads, .max = tsNumOfQnodeFetchThreads, .name = "qnode-fetch", - .fp = (FItem)qmProcessFetchQueue, + .fp = (FItem)qmProcessQueue, .param = pMgmt, }; @@ -167,7 +138,7 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "qnode-monitor", - .fp = (FItem)qmProcessMonitorQueue, + .fp = (FItem)qmProcessQueue, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 1259363f94..af25be8cdf 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -43,44 +43,63 @@ void qndClose(SQnode *pQnode) { int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { - qTrace("message in qnode query queue is processing"); + int32_t code = -1; SReadHandle handle = {.pMsgCb = &pQnode->msgCb}; + qTrace("message in qnode query queue is processing"); switch (pMsg->msgType) { - case TDMT_VND_QUERY: { - return qWorkerProcessQueryMsg(&handle, pQnode->pQuery, pMsg); - } + case TDMT_VND_QUERY: + code = qWorkerProcessQueryMsg(&handle, pQnode->pQuery, pMsg); + break; case TDMT_VND_QUERY_CONTINUE: - return qWorkerProcessCQueryMsg(&handle, pQnode->pQuery, pMsg); + code = qWorkerProcessCQueryMsg(&handle, pQnode->pQuery, pMsg); + break; default: qError("unknown msg type:%d in query queue", pMsg->msgType); - return TSDB_CODE_VND_APP_ERROR; + terrno = TSDB_CODE_VND_APP_ERROR; } + + if (code == 0) return TSDB_CODE_ACTION_IN_PROGRESS; + return code; } int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { + int32_t code = -1; qTrace("message in fetch queue is processing"); + switch (pMsg->msgType) { case TDMT_VND_FETCH: - return qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_FETCH_RSP: - return qWorkerProcessFetchRsp(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessFetchRsp(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_RES_READY: - return qWorkerProcessReadyMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessReadyMsg(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_TASKS_STATUS: - return qWorkerProcessStatusMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessStatusMsg(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_CANCEL_TASK: - return qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_DROP_TASK: - return qWorkerProcessDropMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessDropMsg(pQnode, pQnode->pQuery, pMsg); + break; case TDMT_VND_TABLE_META: - // return vnodeGetTableMeta(pQnode, pMsg); + // code = vnodeGetTableMeta(pQnode, pMsg); + // break; case TDMT_VND_CONSUME: - // return tqProcessConsumeReq(pQnode->pTq, pMsg); + // code = tqProcessConsumeReq(pQnode->pTq, pMsg); + // break; case TDMT_VND_QUERY_HEARTBEAT: - return qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg); + code = qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg); + break; default: qError("unknown msg type:%d in fetch queue", pMsg->msgType); - return TSDB_CODE_VND_APP_ERROR; + terrno = TSDB_CODE_VND_APP_ERROR; } + + if (code == 0) return TSDB_CODE_ACTION_IN_PROGRESS; + return code; } From 0288faac7f156b7fd7f2d574b9212749db9ed407 Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:50:10 +0800 Subject: [PATCH 38/73] fix case --- tests/system-test/0-others/user_control.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index a785060d52..327b1531d9 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -311,7 +311,8 @@ class TDTestCase: tdSql.error(sql) tdSql.query(sql) self.__change_user_priv(user=user, pre_priv=priv, invoke=True) - user.db_set.remove(dbname) + if user.name != "root": + user.db_set.remove(dbname) if dbname else user.db_set.clear() time.sleep(1) def test_priv_change_current(self): From e63af60d5b2e556074cd4b1466ef1cbd3ddf2100 Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:53:22 +0800 Subject: [PATCH 39/73] fix case --- tests/system-test/0-others/user_control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index 327b1531d9..4b5be79a70 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -312,7 +312,7 @@ class TDTestCase: tdSql.query(sql) self.__change_user_priv(user=user, pre_priv=priv, invoke=True) if user.name != "root": - user.db_set.remove(dbname) if dbname else user.db_set.clear() + user.db_set.discard(dbname) if dbname else user.db_set.clear() time.sleep(1) def test_priv_change_current(self): From 365d80890a71675cc783d8086872212d80514877 Mon Sep 17 00:00:00 2001 From: cpwu Date: Sat, 21 May 2022 17:54:59 +0800 Subject: [PATCH 40/73] add case to CI --- tests/system-test/fulltest.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 527cc1f8a3..835fd45226 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -9,8 +9,7 @@ python3 ./test.py -f 0-others/telemetry.py python3 ./test.py -f 0-others/taosdMonitor.py python3 ./test.py -f 0-others/udfTest.py -# TODO privilege has error -# python3 ./test.py -f 0-others/user_control.py +python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py #python3 ./test.py -f 2-query/between.py From 12d27bfdd3c8b4d678dfda49e610e9be60772e21 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 17:57:39 +0800 Subject: [PATCH 41/73] refactor: rpc msg handler --- include/dnode/qnode/qnode.h | 1 - source/dnode/mgmt/mgmt_qnode/src/qmWorker.c | 6 +--- source/dnode/mnode/impl/src/mndQuery.c | 35 ++++++--------------- source/dnode/qnode/src/qnode.c | 18 ++--------- 4 files changed, 13 insertions(+), 47 deletions(-) diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 89553f978b..1ab101f705 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -72,7 +72,6 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad); * @param pMsg The request message */ int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg); -int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index e30b683d19..35c94b7fbe 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -35,12 +35,8 @@ static void qmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_MON_QM_INFO: code = qmProcessGetMonitorInfoReq(pMgmt, pMsg); break; - case TDMT_VND_QUERY: - case TDMT_VND_QUERY_CONTINUE: - code = qndProcessQueryMsg(pMgmt->pQnode, pMsg); - break; default: - code = qndProcessFetchMsg(pMgmt->pQnode, pMsg); + code = qndProcessQueryMsg(pMgmt->pQnode, pMsg); break; } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 0c0d771a24..78b70c9a74 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -18,34 +18,19 @@ #include "mndMnode.h" #include "qworker.h" -int32_t mndProcessQueryMsg(SRpcMsg *pReq) { +int32_t mndProcessQueryMsg(SRpcMsg *pMsg) { int32_t code = -1; - SMnode *pMnode = pReq->info.node; + SMnode *pMnode = pMsg->info.node; SReadHandle handle = {.mnd = pMnode, .pMsgCb = &pMnode->msgCb}; - mTrace("msg:%p, in query queue is processing", pReq); - switch (pReq->msgType) { + mTrace("msg:%p, in query queue is processing", pMsg); + switch (pMsg->msgType) { case TDMT_VND_QUERY: - code = qWorkerProcessQueryMsg(&handle, pMnode->pQuery, pReq); + code = qWorkerProcessQueryMsg(&handle, pMnode->pQuery, pMsg); break; case TDMT_VND_QUERY_CONTINUE: - code = qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pReq); + code = qWorkerProcessCQueryMsg(&handle, pMnode->pQuery, pMsg); break; - default: - terrno = TSDB_CODE_VND_APP_ERROR; - mError("unknown msg type:%d in query queue", pReq->msgType); - } - - if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - return code; -} - -int32_t mndProcessFetchMsg(SRpcMsg *pMsg) { - int32_t code = -1; - SMnode *pMnode = pMsg->info.node; - mTrace("msg:%p, in fetch queue is processing", pMsg); - - switch (pMsg->msgType) { case TDMT_VND_FETCH: code = qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, pMsg); break; @@ -57,7 +42,7 @@ int32_t mndProcessFetchMsg(SRpcMsg *pMsg) { break; default: terrno = TSDB_CODE_VND_APP_ERROR; - mError("unknown msg type:%d in fetch queue", pMsg->msgType); + mError("unknown msg type:%d in query queue", pMsg->msgType); } if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -72,9 +57,9 @@ int32_t mndInitQuery(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_QUERY, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_VND_QUERY_CONTINUE, mndProcessQueryMsg); - mndSetMsgHandle(pMnode, TDMT_VND_FETCH, mndProcessFetchMsg); - mndSetMsgHandle(pMnode, TDMT_VND_DROP_TASK, mndProcessFetchMsg); - mndSetMsgHandle(pMnode, TDMT_VND_QUERY_HEARTBEAT, mndProcessFetchMsg); + mndSetMsgHandle(pMnode, TDMT_VND_FETCH, mndProcessQueryMsg); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_TASK, mndProcessQueryMsg); + mndSetMsgHandle(pMnode, TDMT_VND_QUERY_HEARTBEAT, mndProcessQueryMsg); return 0; } diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index af25be8cdf..929643fcdf 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -45,7 +45,7 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { int32_t code = -1; SReadHandle handle = {.pMsgCb = &pQnode->msgCb}; - qTrace("message in qnode query queue is processing"); + qTrace("message in qnode queue is processing"); switch (pMsg->msgType) { case TDMT_VND_QUERY: @@ -54,20 +54,6 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { case TDMT_VND_QUERY_CONTINUE: code = qWorkerProcessCQueryMsg(&handle, pQnode->pQuery, pMsg); break; - default: - qError("unknown msg type:%d in query queue", pMsg->msgType); - terrno = TSDB_CODE_VND_APP_ERROR; - } - - if (code == 0) return TSDB_CODE_ACTION_IN_PROGRESS; - return code; -} - -int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { - int32_t code = -1; - qTrace("message in fetch queue is processing"); - - switch (pMsg->msgType) { case TDMT_VND_FETCH: code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg); break; @@ -96,7 +82,7 @@ int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { code = qWorkerProcessHbMsg(pQnode, pQnode->pQuery, pMsg); break; default: - qError("unknown msg type:%d in fetch queue", pMsg->msgType); + qError("unknown msg type:%d in qnode queue", pMsg->msgType); terrno = TSDB_CODE_VND_APP_ERROR; } From f4c3141821ec4d23e64ca4993d95ae561741ce11 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 18:10:37 +0800 Subject: [PATCH 42/73] refactor: rpc msg handler --- source/dnode/mgmt/mgmt_bnode/src/bmWorker.c | 8 +++---- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 10 +++----- source/dnode/mgmt/mgmt_snode/src/smWorker.c | 8 +++---- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 23 +++++++++---------- source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 1 - source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 5 ---- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 3 ++- source/dnode/vnode/src/vnd/vnodeSvr.c | 3 --- 8 files changed, 22 insertions(+), 39 deletions(-) diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c index e019924268..08c9edd854 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c @@ -47,10 +47,8 @@ static inline void bmSendRsp(SRpcMsg *pMsg, int32_t code) { static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SBnodeMgmt *pMgmt = pInfo->ahandle; - + int32_t code = -1; dTrace("msg:%p, get from bnode-monitor queue", pMsg); - SRpcMsg *pRpc = pMsg; - int32_t code = -1; if (pMsg->msgType == TDMT_MON_BM_INFO) { code = bmProcessGetMonBmInfoReq(pMgmt, pMsg); @@ -58,13 +56,13 @@ static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; } - if (pRpc->msgType & 1U) { + if (IsReq(pMsg)) { if (code != 0 && terrno != 0) code = terrno; bmSendRsp(pMsg, code); } dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pRpc->pCont); + rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 6a7e0ad322..bd78fe46ef 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -19,7 +19,6 @@ static void *dmStatusThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); - setThreadName("dnode-status"); while (1) { @@ -40,7 +39,6 @@ static void *dmStatusThreadFp(void *param) { static void *dmMonitorThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); - setThreadName("dnode-monitor"); while (1) { @@ -103,11 +101,9 @@ void dmStopMonitorThread(SDnodeMgmt *pMgmt) { static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SDnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; - tmsg_t msgType = pMsg->msgType; - bool isRequest = msgType & 1u; - dTrace("msg:%p, will be processed in dnode-mgmt queue, type:%s", pMsg, TMSG_INFO(msgType)); + dTrace("msg:%p, will be processed in dnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - switch (msgType) { + switch (pMsg->msgType) { case TDMT_DND_CONFIG_DNODE: code = dmProcessConfigReq(pMgmt, pMsg); break; @@ -149,7 +145,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { break; } - if (isRequest) { + if (IsReq(pMsg)) { if (code != 0 && terrno != 0) code = terrno; SRpcMsg rsp = { .code = code, diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index fcfc4f4cee..34a205232e 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -28,10 +28,8 @@ static inline void smSendRsp(SRpcMsg *pMsg, int32_t code) { static void smProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SSnodeMgmt *pMgmt = pInfo->ahandle; - + int32_t code = -1; dTrace("msg:%p, get from snode-monitor queue", pMsg); - SRpcMsg *pRpc = pMsg; - int32_t code = -1; if (pMsg->msgType == TDMT_MON_SM_INFO) { code = smProcessGetMonitorInfoReq(pMgmt, pMsg); @@ -39,13 +37,13 @@ static void smProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; } - if (pRpc->msgType & 1U) { + if (IsReq(pMsg)) { if (code != 0 && terrno != 0) code = terrno; smSendRsp(pMsg, code); } dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pRpc->pCont); + rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 3a5e8a671c..e7bfbdae58 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -29,7 +29,7 @@ static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) { tmsgSendRsp(&rsp); } -static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +static void vmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SVnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; dTrace("msg:%p, get from vnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); @@ -92,7 +92,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SArray * pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); if (pArray == NULL) { dError("failed to process %d msgs in write-queue since %s", numOfMsgs, terrstr()); return; @@ -222,8 +222,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { - SRpcMsg * pRpc = pMsg; - SMsgHead *pHead = pRpc->pCont; + SMsgHead *pHead = pMsg->pCont; int32_t code = 0; pHead->contLen = ntohl(pHead->contLen); @@ -237,23 +236,23 @@ static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType switch (qtype) { case QUERY_QUEUE: - dTrace("msg:%p, put into vnode-query worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-query worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pVnode->pQueryQ, pMsg); break; case FETCH_QUEUE: - dTrace("msg:%p, put into vnode-fetch worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-fetch worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - dTrace("msg:%p, put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pVnode->pWriteQ, pMsg); break; case SYNC_QUEUE: - dTrace("msg:%p, put into vnode-sync worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-sync worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pVnode->pSyncQ, pMsg); break; case MERGE_QUEUE: - dTrace("msg:%p, put into vnode-merge worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, put into vnode-merge worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pVnode->pMergeQ, pMsg); break; default: @@ -301,7 +300,7 @@ int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType qtype) { - SMsgHead * pHead = pRpc->pCont; + SMsgHead *pHead = pRpc->pCont; SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) return -1; @@ -469,7 +468,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "vnode-mgmt", - .fp = (FItem)vmProcessMgmtMonitorQueue, + .fp = (FItem)vmProcessQueue, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) { @@ -481,7 +480,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "vnode-monitor", - .fp = (FItem)vmProcessMgmtMonitorQueue, + .fp = (FItem)vmProcessQueue, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 2b3ac7ae73..27f1140f23 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -137,7 +137,6 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper); void dmSetStatus(SDnode *pDnode, EDndRunStatus stype); void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg); void dmProcessNetTestReq(SDnode *pDnode, SRpcMsg *pMsg); -void dmProcessFetchRsp(SRpcMsg *pMsg); // dmNodes.c int32_t dmOpenNode(SMgmtWrapper *pWrapper); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index c83550c7b1..787f5e5019 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -314,8 +314,3 @@ void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg) { rpcSendResponse(&rsp); rpcFreeCont(pMsg->pCont); } - -void dmProcessFetchRsp(SRpcMsg *pMsg) { - qWorkerProcessFetchRsp(NULL, NULL, pMsg); - // rpcFreeCont(pMsg->pCont); -} \ No newline at end of file diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 8dfcb798ac..6fbfae8b41 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "dmMgmt.h" +#include "qworker.h" static void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet); static void dmSendRsp(SRpcMsg *pMsg); @@ -61,7 +62,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { dmProcessNetTestReq(pDnode, pRpc); return; } else if (pRpc->msgType == TDMT_MND_SYSTABLE_RETRIEVE_RSP || pRpc->msgType == TDMT_VND_FETCH_RSP) { - dmProcessFetchRsp(pRpc); + qWorkerProcessFetchRsp(NULL, NULL, pRpc); return; } else { } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 297b518ac7..88cc97ddd5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -147,9 +147,6 @@ _err: int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); -#if 0 - SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; -#endif SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; switch (pMsg->msgType) { case TDMT_VND_QUERY: From 3def04a54a30b91da9767a3d9bbe8b6d19c81432 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 20:10:37 +0800 Subject: [PATCH 43/73] test: case for valgrind --- tests/script/tsim/valgrind/basic.sim | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/script/tsim/valgrind/basic.sim diff --git a/tests/script/tsim/valgrind/basic.sim b/tests/script/tsim/valgrind/basic.sim new file mode 100644 index 0000000000..0f11ae0313 --- /dev/null +++ b/tests/script/tsim/valgrind/basic.sim @@ -0,0 +1,8 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database d0 vgroups 1; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 7fc2e8806734f54bb2fcd722b5c5222d110662b9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 21 May 2022 20:22:27 +0800 Subject: [PATCH 44/73] fea:add tag default name in schemaless --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index b8dd33b915..1a113a67d1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -327,7 +327,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; - if (cfgAddString(pCfg, "smlTagName", "", 1) != 0) return -1; + if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 4; From 6253ed7db5b48e625808b416cad76a6475296693 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 20:32:45 +0800 Subject: [PATCH 45/73] fix: memory leak --- source/dnode/mnode/impl/src/mndDb.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 633dda454b..6921235f8b 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -525,7 +525,6 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; dbObj.cfg.pRetensions = pCreate->pRetensions; - pCreate->pRetensions = NULL; mndSetDefaultDbCfg(&dbObj.cfg); From f3d527589f9c45ba4a27d848c418c5134db0d984 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 21 May 2022 12:37:04 +0000 Subject: [PATCH 46/73] fix: memory leak --- source/dnode/vnode/src/meta/metaEntry.c | 6 +++--- source/dnode/vnode/src/meta/metaTable.c | 17 ++++++++++------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 1 + 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index b91622619f..8a4db3100d 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -56,8 +56,8 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeCStr(pCoder, &pME->name) < 0) return -1; if (pME->type == TSDB_SUPER_TABLE) { - if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schema) < 0) return -1; - if (tDecodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; + if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schema) < 0) return -1; + if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; } else if (pME->type == TSDB_CHILD_TABLE) { if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; @@ -67,7 +67,7 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1; - if (tDecodeSSchemaWrapper(pCoder, &pME->ntbEntry.schema) < 0) return -1; + if (tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schema) < 0) return -1; } else if (pME->type == TSDB_TSMA_TABLE) { pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma)); if (!pME->smaEntry.tsma) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index ee4dfc6531..f548904a7a 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -562,7 +562,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA // search table.db TBC *pTbDbc = NULL; - SDecoder dc = {0}; + SDecoder dc1 = {0}; + SDecoder dc2 = {0}; /* get ctbEntry */ tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); @@ -572,18 +573,16 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ctbEntry.pBuf = taosMemoryMalloc(nData); memcpy(ctbEntry.pBuf, pData, nData); - tDecoderInit(&dc, ctbEntry.pBuf, nData); - metaDecodeEntry(&dc, &ctbEntry); - tDecoderClear(&dc); + tDecoderInit(&dc1, ctbEntry.pBuf, nData); + metaDecodeEntry(&dc1, &ctbEntry); /* get stbEntry*/ tdbTbGet(pMeta->pUidIdx, &ctbEntry.ctbEntry.suid, sizeof(tb_uid_t), &pVal, &nVal); tdbTbGet(pMeta->pTbDb, &((STbDbKey){.uid = ctbEntry.ctbEntry.suid, .version = *(int64_t *)pVal}), sizeof(STbDbKey), (void **)&stbEntry.pBuf, &nVal); tdbFree(pVal); - tDecoderInit(&dc, stbEntry.pBuf, nVal); - metaDecodeEntry(&dc, &stbEntry); - tDecoderClear(&dc); + tDecoderInit(&dc2, stbEntry.pBuf, nVal); + metaDecodeEntry(&dc2, &stbEntry); SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; SSchema *pColumn = NULL; @@ -638,6 +637,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA // save to uid.idx tdbTbUpsert(pMeta->pUidIdx, &ctbEntry.uid, sizeof(tb_uid_t), &version, sizeof(version), &pMeta->txn); + tDecoderClear(&dc1); + tDecoderClear(&dc2); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); tdbTbcClose(pTbDbc); @@ -645,6 +646,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA return 0; _err: + tDecoderClear(&dc1); + tDecoderClear(&dc2); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); tdbTbcClose(pTbDbc); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 76d5c3cb3a..28b0ae042d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -477,6 +477,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) { pCommitIter->pTable->pSchema = pTSchema; // metaGetTbTSchema(REPO_META(pRepo), pTbData->uid, 0); } } + tSkipListDestroyIter(pSlIter); return 0; } From 3b07f7121901ecc97e43d351f3eeeeb70b390907 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 21 May 2022 20:37:07 +0800 Subject: [PATCH 47/73] fix: free the memory leaked by sma open --- source/dnode/vnode/src/sma/smaOpen.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 1c7db28e18..2a74fe78cb 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -132,6 +132,7 @@ int32_t smaClose(SSma *pSma) { if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma)); if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma)); if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma)); + taosMemoryFree(pSma); } return 0; } \ No newline at end of file From 7a215a238f19c2f751e7331edbea8f872e4fb6ae Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 21 May 2022 20:58:12 +0800 Subject: [PATCH 48/73] fix: some prblems of parser and planner --- include/util/taoserror.h | 1 + source/libs/parser/src/parTranslater.c | 9 +++++++-- source/libs/parser/src/parUtil.c | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index e28618d940..1d95ca7ca3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -637,6 +637,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_COMMENT_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x264E) #define TSDB_CODE_PAR_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x264F) #define TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY TAOS_DEF_ERROR_CODE(0, 0x2650) +#define TSDB_CODE_PAR_INVALID_DROP_COL TAOS_DEF_ERROR_CODE(0, 0x2651) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8e18c267d6..99e1135599 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4454,8 +4454,10 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type); pReq->nTagVal = pStmt->pVal->node.resType.bytes; - char* pVal = nodesGetValueFromNode(pStmt->pVal); - pReq->pTagVal = IS_VAR_DATA_TYPE(pStmt->pVal->node.resType.type) ? pVal + VARSTR_HEADER_SIZE : pVal; + if (TSDB_DATA_TYPE_NCHAR == pStmt->pVal->node.resType.type) { + pReq->nTagVal = pReq->nTagVal * TSDB_NCHAR_SIZE; + } + pReq->pTagVal = nodesGetValueFromNode(pStmt->pVal); return TSDB_CODE_SUCCESS; } @@ -4479,6 +4481,9 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, SVAlterTbReq* pReq) { + if (2 == getNumOfColumns(pTableMeta)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DROP_COL); + } SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index fe21915b1a..3df9c8abf7 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -169,6 +169,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { "And, cannot be mixed with other non scalar functions or columns."; case TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY: return "Window query not supported, since the result of subquery not include valid timestamp column"; + case TSDB_CODE_PAR_INVALID_DROP_COL: + return "No columns can be dropped"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: From b09750fe7a474404e8f7c6f1e236f713545d81f1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 21 May 2022 21:05:33 +0800 Subject: [PATCH 49/73] fea:add tag default name in schemaless --- source/client/src/clientSml.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8b2f47b295..68c47c2d13 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -726,9 +726,6 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { if(value + len != endPtr){ return -1; } - if(tsInt64 == 0){ - return taosGetTimestampNs(); - } double ts = tsInt64; switch (type) { case TSDB_TIME_PRECISION_HOURS: @@ -794,8 +791,8 @@ static int8_t smlGetTsTypeByPrecision(int8_t precision) { } static int64_t smlParseInfluxTime(SSmlHandle* info, const char* data, int32_t len){ - if(len == 0){ - return taosGetTimestamp(TSDB_TIME_PRECISION_NANO); + if(len == 0 || (len == 1 && data[0] == '0')){ + return taosGetTimestampNs(); } int8_t tsType = smlGetTsTypeByPrecision(info->precision); @@ -817,6 +814,9 @@ static int64_t smlParseOpenTsdbTime(SSmlHandle* info, const char* data, int32_t smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); return -1; } + if(len == 1 && data[0] == '0'){ + return taosGetTimestampNs(); + } int8_t tsType = smlGetTsTypeByLen(len); if (tsType == -1) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); From 648fa307f2be16c2b3e3d25dd9a1c729f529cf6b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 21 May 2022 21:16:35 +0800 Subject: [PATCH 50/73] fea:add tag default name in schemaless --- source/common/src/tglobal.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 1a113a67d1..1b61a0bc60 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -78,7 +78,7 @@ char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com"; uint16_t tsTelemPort = 80; // schemaless -char tsSmlTagName[TSDB_COL_NAME_LEN] = "__tag"; +char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; //user defined child table name can be specified in tag value. //If set to empty system will generate table name using MD5 hash. bool tsSmlDataFormat = true; // true means that the name and order of cols in each line are the same(only for influx protocol) @@ -327,7 +327,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; - if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; + if (cfgAddString(pCfg, "smlTagNullName", tsSmlTagName, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 4; @@ -524,7 +524,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { } tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); - tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); + tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagNullName")->str, TSDB_COL_NAME_LEN); tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; From 133a963c7c263511db02b3e3fd2d98a85fbd3a49 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 May 2022 21:29:32 +0800 Subject: [PATCH 51/73] fix: memory leak while record history in shell --- tools/shell/src/shellEngine.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d6c295a222..a866488d3a 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -742,6 +742,7 @@ void shellReadHistory() { int32_t read_size = 0; while ((read_size = taosGetLineFile(pFile, &line)) != -1) { line[read_size - 1] = '\0'; + taosMemoryFree(pHistory->hist[pHistory->hend]); pHistory->hist[pHistory->hend] = strdup(line); pHistory->hend = (pHistory->hend + 1) % SHELL_MAX_HISTORY_SIZE; @@ -763,7 +764,8 @@ void shellWriteHistory() { for (int32_t i = pHistory->hstart; i != pHistory->hend;) { if (pHistory->hist[i] != NULL) { taosFprintfFile(pFile, "%s\n", pHistory->hist[i]); - taosMemoryFreeClear(pHistory->hist[i]); + taosMemoryFree(pHistory->hist[i]); + pHistory->hist[i] = NULL; } i = (i + 1) % SHELL_MAX_HISTORY_SIZE; } @@ -771,6 +773,16 @@ void shellWriteHistory() { taosCloseFile(&pFile); } +void shellCleanupHistory() { + SShellHistory *pHistory = &shell.history; + for (int32_t i = 0; i < SHELL_MAX_HISTORY_SIZE; ++i) { + if (pHistory->hist[i] != NULL) { + taosMemoryFree(pHistory->hist[i]); + pHistory->hist[i] = NULL; + } + } +} + void shellPrintError(TAOS_RES *tres, int64_t st) { int64_t et = taosGetTimestampUs(); fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6); @@ -971,6 +983,7 @@ int32_t shellExecute() { taos_close(shell.conn); shellWriteHistory(); + shellCleanupHistory(); return 0; } @@ -996,5 +1009,6 @@ int32_t shellExecute() { taosThreadClear(&shell.pid); } + shellCleanupHistory(); return 0; } From 8e3701df01866122f0f9c14211f0a5d40ae94972 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 21 May 2022 21:36:26 +0800 Subject: [PATCH 52/73] fix(tmq): memory leak --- example/src/tmq.c | 1 + include/common/tmsg.h | 12 ++++++++---- source/client/src/clientEnv.c | 4 ++-- source/client/src/clientImpl.c | 2 +- source/client/src/clientMain.c | 4 ++-- source/client/src/tmq.c | 35 ++++++++++++++++++++++++---------- source/dnode/vnode/src/tq/tq.c | 17 ++++++++++------- 7 files changed, 49 insertions(+), 26 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index 338399232c..913096ee90 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -171,6 +171,7 @@ tmq_t* build_consumer() { tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); + tmq_conf_destroy(conf); return tmq; } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bd3ca6ae8d..df66174c9b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2523,11 +2523,9 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); buf = taosDecodeFixedI32(buf, &pRsp->blockNum); - pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); - pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); if (pRsp->blockNum != 0) { + pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); buf = taosDecodeFixedI8(buf, &pRsp->withTbName); buf = taosDecodeFixedI8(buf, &pRsp->withSchema); buf = taosDecodeFixedI8(buf, &pRsp->withTag); @@ -2540,14 +2538,20 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p taosArrayPush(pRsp->blockDataLen, &bLen); taosArrayPush(pRsp->blockData, &data); if (pRsp->withSchema) { + pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); buf = taosDecodeSSchemaWrapper(buf, pSW); taosArrayPush(pRsp->blockSchema, &pSW); + } else { + pRsp->blockSchema = NULL; } if (pRsp->withTbName) { + pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); char* name = NULL; buf = taosDecodeString(buf, &name); taosArrayPush(pRsp->blockTbName, &name); + } else { + pRsp->blockTbName = NULL; } } } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index c4dc98354e..669b2bc97e 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -60,7 +60,7 @@ static void registerRequest(SRequestObj *pRequest) { static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj * pTscObj = pRequest->pTscObj; + STscObj *pTscObj = pRequest->pTscObj; SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); @@ -313,7 +313,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return 0; } - SConfig * pCfg = taosGetCfg(); + SConfig *pCfg = taosGetCfg(); SConfigItem *pItem = NULL; switch (option) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 386283b5b5..d1a3511252 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -390,7 +390,7 @@ void freeRequestRes(SRequestObj* pRequest, void* res) { if (TDMT_VND_SUBMIT == pRequest->type) { tFreeSSubmitRsp((SSubmitRsp*)res); } else if (TDMT_VND_QUERY == pRequest->type) { - taosArrayDestroy((SArray *)res); + taosArrayDestroy((SArray*)res); } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 7360b054e2..eec8cab7ad 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -146,10 +146,10 @@ void taos_free_result(TAOS_RES *res) { SMqRspObj *pRsp = (SMqRspObj *)res; if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen); - if (pRsp->rsp.blockSchema) taosArrayDestroy(pRsp->rsp.blockSchema); - if (pRsp->rsp.blockTbName) taosArrayDestroy(pRsp->rsp.blockTbName); if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags); if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema); + if (pRsp->rsp.withTbName) taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree); + if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); pRsp->resInfo.pRspMsg = NULL; doFreeReqResultInfo(&pRsp->resInfo); } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 36c0d8156c..dfa56f80c4 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -202,7 +202,12 @@ tmq_conf_t* tmq_conf_new() { } void tmq_conf_destroy(tmq_conf_t* conf) { - if (conf) taosMemoryFree(conf); + if (conf) { + if (conf->ip) taosMemoryFree(conf->ip); + if (conf->user) taosMemoryFree(conf->user); + if (conf->pass) taosMemoryFree(conf->pass); + taosMemoryFree(conf); + } } tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { @@ -497,6 +502,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { } else { ASSERT(0); } + taosFreeQitem(pTaskType); } taosFreeQall(qall); return 0; @@ -954,8 +960,12 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqClientVg* pVg = pParam->pVg; SMqClientTopic* pTopic = pParam->pTopic; tmq_t* tmq = pParam->tmq; + int32_t vgId = pParam->vgId; + int32_t epoch = pParam->epoch; + taosMemoryFree(pParam); if (code != 0) { - tscWarn("msg discard from vg %d, epoch %d, code:%x", pParam->vgId, pParam->epoch, code); + tscWarn("msg discard from vg %d, epoch %d, code:%x", vgId, epoch, code); + if (pMsg->pData) taosMemoryFree(pMsg->pData); goto CREATE_MSG_FAIL; } @@ -963,19 +973,21 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { // do not write into queue since updating epoch reset - tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch, + tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch); tsem_post(&tmq->rspSem); + taosMemoryFree(pMsg->pData); return 0; } if (msgEpoch != tmqEpoch) { - tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); + tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch); } SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { - tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch); + taosMemoryFree(pMsg->pData); + tscWarn("msg discard from vg %d, epoch %d since out of memory", vgId, epoch); goto CREATE_MSG_FAIL; } @@ -986,6 +998,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg); + taosMemoryFree(pMsg->pData); tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId, pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset); @@ -995,7 +1008,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { return 0; CREATE_MSG_FAIL: - if (pParam->epoch == tmq->epoch) { + if (epoch == tmq->epoch) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } tsem_post(&tmq->rspSem); @@ -1088,6 +1101,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; + int8_t async = pParam->async; pParam->code = code; if (code != 0) { tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async); @@ -1104,7 +1118,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { goto END; } - if (!pParam->async) { + if (!async) { SMqAskEpRsp rsp; tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ @@ -1125,13 +1139,14 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { taosWriteQitem(tmq->mqueue, pWrapper); tsem_post(&tmq->rspSem); - taosMemoryFree(pParam); } END: /*atomic_store_8(&tmq->epStatus, 0);*/ - if (!pParam->async) { + if (!async) { tsem_post(&pParam->rspSem); + } else { + taosMemoryFree(pParam); } return code; } @@ -1279,7 +1294,6 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); } - taosFreeQitem(pWrapper); return pRspObj; } @@ -1401,6 +1415,7 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t waitTime, bool pollIfReset) { } // build rsp SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper); + taosFreeQitem(pollRspWrapper); return pRsp; } else { /*printf("epoch mismatch\n");*/ diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 1f35ec2650..0671044bad 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -57,6 +57,9 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { void tqClose(STQ* pTq) { if (pTq) { taosMemoryFreeClear(pTq->path); + taosHashCleanup(pTq->execs); + taosHashCleanup(pTq->pStreamTasks); + taosHashCleanup(pTq->pushMgr); taosMemoryFree(pTq); } // TODO @@ -409,9 +412,9 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu pTopic->buffer.output[j].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { - .reader = pReadHandle, - .meta = pTq->pVnode->pMeta, - .pMsgCb = &pTq->pVnode->msgCb, + .reader = pReadHandle, + .meta = pTq->pVnode->pMeta, + .pMsgCb = &pTq->pVnode->msgCb, }; pTopic->buffer.output[j].pReadHandle = pReadHandle; pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); @@ -1000,10 +1003,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { for (int32_t i = 0; i < parallel; i++) { STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { - .reader = pStreamReader, - .meta = pTq->pVnode->pMeta, - .pMsgCb = &pTq->pVnode->msgCb, - .vnode = pTq->pVnode, + .reader = pStreamReader, + .meta = pTq->pVnode->pMeta, + .pMsgCb = &pTq->pVnode->msgCb, + .vnode = pTq->pVnode, }; pTask->exec.runners[i].inputHandle = pStreamReader; pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); From 93642189636060d6cec2b3b9990725d9636edc99 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 21 May 2022 21:59:04 +0800 Subject: [PATCH 53/73] fix mem leak --- include/common/tmsg.h | 10 +++--- source/client/src/clientHb.c | 45 ++++++++++++------------ source/client/src/clientImpl.c | 7 ++-- source/client/src/clientStmt.c | 9 +++-- source/dnode/mnode/impl/src/mndProfile.c | 18 +++------- source/dnode/mnode/impl/src/mndStb.c | 3 ++ source/libs/parser/src/parInsert.c | 4 +++ source/libs/parser/src/parTokenizer.c | 1 + 8 files changed, 50 insertions(+), 47 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bd3ca6ae8d..6811b29d47 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1987,19 +1987,16 @@ static FORCE_INLINE void tFreeClientHbReq(void* pReq) { if (req->info) { tFreeReqKvHash(req->info); taosHashCleanup(req->info); + req->info = NULL; } } int32_t tSerializeSClientHbBatchReq(void* buf, int32_t bufLen, const SClientHbBatchReq* pReq); int32_t tDeserializeSClientHbBatchReq(void* buf, int32_t bufLen, SClientHbBatchReq* pReq); -static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { +static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) { SClientHbBatchReq* req = (SClientHbBatchReq*)pReq; - if (deep) { - taosArrayDestroyEx(req->reqs, tFreeClientHbReq); - } else { - taosArrayDestroy(req->reqs); - } + taosArrayDestroyEx(req->reqs, tFreeClientHbReq); taosMemoryFree(pReq); } @@ -2023,6 +2020,7 @@ static FORCE_INLINE void tFreeClientHbBatchRsp(void* pRsp) { int32_t tSerializeSClientHbBatchRsp(void* buf, int32_t bufLen, const SClientHbBatchRsp* pBatchRsp); int32_t tDeserializeSClientHbBatchRsp(void* buf, int32_t bufLen, SClientHbBatchRsp* pBatchRsp); +void tFreeSClientHbBatchRsp(SClientHbBatchRsp *pBatchRsp); static FORCE_INLINE int32_t tEncodeSKv(SEncoder* pEncoder, const SKv* pKv) { if (tEncodeI32(pEncoder, pKv->key) < 0) return -1; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 13ac43bc39..b068e13b7d 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -394,6 +394,10 @@ int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S tscDebug("hb got %d expired users, valueLen:%d", userNum, kv.valueLen); + if (NULL == req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + } + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); return TSDB_CODE_SUCCESS; @@ -429,6 +433,10 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen); + if (NULL == req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + } + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); return TSDB_CODE_SUCCESS; @@ -463,6 +471,10 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen); + if (NULL == req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + } + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); return TSDB_CODE_SUCCESS; @@ -511,16 +523,6 @@ static FORCE_INLINE void hbMgrInitHandle() { hbMgrInitMqHbHandle(); } -void hbFreeReq(void *req) { - SClientHbReq *pReq = (SClientHbReq *)req; - tFreeReqKvHash(pReq->info); -} - -void hbClearClientHbReq(SClientHbReq *pReq) { - pReq->query = NULL; - pReq->info = NULL; -} - SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { SClientHbBatchReq *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { @@ -535,6 +537,8 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { while (pIter != NULL) { SClientHbReq *pOneReq = pIter; + pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); + SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pOneReq->connKey, sizeof(SClientHbKey)); if (info) { code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, info->param, pOneReq); @@ -544,7 +548,6 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { } } - taosArrayPush(pBatchReq->reqs, pOneReq); //hbClearClientHbReq(pOneReq); pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); @@ -601,8 +604,8 @@ static void *hbThreadFunc(void *param) { void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tFreeClientHbBatchReq(pReq, false); - hbClearReqInfo(pAppHbMgr); + tFreeClientHbBatchReq(pReq); + //hbClearReqInfo(pAppHbMgr); break; } @@ -611,8 +614,8 @@ static void *hbThreadFunc(void *param) { if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tFreeClientHbBatchReq(pReq, false); - hbClearReqInfo(pAppHbMgr); + tFreeClientHbBatchReq(pReq); + //hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); break; } @@ -628,8 +631,8 @@ static void *hbThreadFunc(void *param) { int64_t transporterId = 0; SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); - tFreeClientHbBatchReq(pReq, false); - hbClearReqInfo(pAppHbMgr); + tFreeClientHbBatchReq(pReq); + //hbClearReqInfo(pAppHbMgr); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } @@ -721,8 +724,7 @@ void appHbMgrCleanup(void) { void *pIter = taosHashIterate(pTarget->activeInfo, NULL); while (pIter != NULL) { SClientHbReq *pOneReq = pIter; - hbFreeReq(pOneReq); - taosHashCleanup(pOneReq->info); + tFreeClientHbReq(pOneReq); pIter = taosHashIterate(pTarget->activeInfo, pIter); } taosHashCleanup(pTarget->activeInfo); @@ -782,7 +784,7 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo * } SClientHbReq hbReq = {0}; hbReq.connKey = connKey; - hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + //hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq)); @@ -823,8 +825,7 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (pReq) { - hbFreeReq(pReq); - taosHashCleanup(pReq->info); + tFreeClientHbReq(pReq); taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8a03043a99..8c7e7cef98 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -387,7 +387,7 @@ _return: } void freeRequestRes(SRequestObj* pRequest, void* res) { - if (NULL == res) { + if (NULL == pRequest || NULL == res) { return; } @@ -435,12 +435,13 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; - freeRequestRes(pRequest, pRes); - pRes = NULL; } if (res) { *res = pRes; + } else { + freeRequestRes(pRequest, pRes); + pRes = NULL; } return pRequest; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index a9b4e234e8..1746858482 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -800,6 +800,7 @@ int stmtExec(TAOS_STMT* stmt) { if (code) { pStmt->exec.pRequest->code = code; } else { + tFreeSSubmitRsp(pRsp); STMT_ERR_RET(stmtResetStmt(pStmt)); STMT_ERR_RET(TSDB_CODE_NEED_RETRY); } @@ -817,11 +818,13 @@ _return: if (TSDB_CODE_SUCCESS == code && autoCreateTbl) { if (NULL == pRsp) { tscError("no submit resp got for auto create table"); - STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + code = TSDB_CODE_TSC_APP_ERROR; + } else { + code = stmtUpdateTableUid(pStmt, pRsp); } - - STMT_ERR_RET(stmtUpdateTableUid(pStmt, pRsp)); } + + tFreeSSubmitRsp(pRsp); ++pStmt->sql.runTimes; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index cd57e76b2c..b9ac82d890 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -128,7 +128,8 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType } static void mndFreeConn(SConnObj *pConn) { - taosMemoryFreeClear(pConn->pQueries); + taosArrayDestroyEx(pConn->pQueries, tFreeClientHbQueryDesc); + mTrace("conn:%u, is destroyed, data:%p", pConn->id, pConn); } @@ -396,6 +397,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb if (NULL == hbRsp.info) { mError("taosArrayInit %d rsp kv failed", kvNum); terrno = TSDB_CODE_OUT_OF_MEMORY; + tFreeClientHbRsp(&hbRsp); return -1; } @@ -453,6 +455,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { SClientHbBatchReq batchReq = {0}; if (tDeserializeSClientHbBatchReq(pReq->pCont, pReq->contLen, &batchReq) != 0) { + taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq); terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -479,18 +482,7 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { void *buf = rpcMallocCont(tlen); tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp); - int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps); - for (int32_t i = 0; i < rspNum; ++i) { - SClientHbRsp *rsp = taosArrayGet(batchRsp.rsps, i); - int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info) : 0; - for (int32_t n = 0; n < kvNum; ++n) { - SKv *kv = taosArrayGet(rsp->info, n); - taosMemoryFreeClear(kv->value); - } - taosArrayDestroy(rsp->info); - } - - taosArrayDestroy(batchRsp.rsps); + tFreeClientHbBatchRsp(&batchRsp); pReq->info.rspLen = tlen; pReq->info.rsp = buf; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 7485510bc6..4ad92a5416 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1642,6 +1642,8 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int if (pStbVersion->sversion != metaRsp.sversion) { taosArrayPush(batchMetaRsp.pArray, &metaRsp); + } else { + tFreeSTableMetaRsp(&metaRsp); } } @@ -1660,6 +1662,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int } tSerializeSTableMetaBatchRsp(pRsp, rspLen, &batchMetaRsp); + tFreeSTableMetaBatchRsp(&batchMetaRsp); *ppRsp = pRsp; *pRspLen = rspLen; return 0; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index b452950624..6847d680d2 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1085,6 +1085,10 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // no data in the sql string anymore. if (sToken.n == 0) { + if (sToken.type && pCxt->pSql[0]) { + return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", sToken.z); + } + if (0 == pCxt->totalNum && (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { return buildInvalidOperationMsg(&pCxt->msg, "no data in sql"); } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index abc7ddb17f..8fb9780f8a 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -704,6 +704,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) { if (t0.type == TK_NK_SEMI) { t0.n = 0; + t0.type = 0; return t0; } From fe394b358b0bd280dfa4f8c2605ead41cc889dcb Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 21 May 2022 22:37:40 +0800 Subject: [PATCH 54/73] feat: add update case to ci --- tests/script/jenkins/basic.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 18fe3b9afe..bb96e3642b 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -28,6 +28,7 @@ ./test.sh -f tsim/insert/basic1.sim ./test.sh -f tsim/insert/backquote.sim ./test.sh -f tsim/insert/null.sim +./test.sh -f tsim/insert/update0.sim # ---- parser ./test.sh -f tsim/parser/groupby-basic.sim From 8f75b96ed4e1b44f8079143540d667a04736d579 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 21 May 2022 23:33:39 +0800 Subject: [PATCH 55/73] enh: refator index/transport code --- source/libs/index/inc/indexCache.h | 1 + source/libs/index/src/index.c | 30 ++++++++++++++++++++-------- source/libs/index/src/indexCache.c | 20 +++++++++++++++---- source/libs/index/test/indexTests.cc | 21 +++++++++++++++++-- 4 files changed, 58 insertions(+), 14 deletions(-) diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index d474d87409..96bfb2cd7f 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -63,6 +63,7 @@ typedef struct CacheTerm { IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type); +void indexCacheForceToMerge(void* cache); void indexCacheDestroy(void* cache); Iterate* indexCacheIteratorCreate(IndexCache* cache); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 92159dae7a..d1f50f1979 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -124,15 +124,21 @@ END: void indexDestroy(void* handle) { SIndex* sIdx = handle; - void* iter = taosHashIterate(sIdx->colObj, NULL); - while (iter) { - IndexCache** pCache = iter; - if (*pCache) { + // indexAcquireRef(sIdx->refId); + if (sIdx->colObj != NULL) { + void* iter = taosHashIterate(sIdx->colObj, NULL); + while (iter) { + IndexCache** pCache = iter; + indexCacheForceToMerge((void*)(*pCache)); + iter = taosHashIterate(sIdx->colObj, iter); + indexCacheUnRef(*pCache); } - iter = taosHashIterate(sIdx->colObj, iter); - } - taosHashCleanup(sIdx->colObj); + taosHashCleanup(sIdx->colObj); + sIdx->colObj = NULL; + return; + } // indexReleaseRef(sIdx->refId); + taosThreadMutexDestroy(&sIdx->mtx); indexTFileDestroy(sIdx->tindex); taosMemoryFree(sIdx->path); @@ -177,7 +183,6 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*)); } } - taosThreadMutexUnlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); @@ -193,6 +198,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { return ret; } } + taosThreadMutexUnlock(&index->mtx); return 0; } int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { @@ -451,6 +457,14 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { } // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); + if (cacheIter == NULL) { + indexError("%p immtable is empty, ignore merge opera", pCache); + indexCacheDestroyImm(pCache); + tfileReaderUnRef(pReader); + indexReleaseRef(sIdx->refId); + return 0; + } + Iterate* tfileIter = tfileIteratorCreate(pReader); if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 5f7cc10b99..ef01094b5d 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -429,11 +429,13 @@ void indexCacheDestroy(void* cache) { } Iterate* indexCacheIteratorCreate(IndexCache* cache) { + if (cache->imm == NULL) { + return NULL; + } Iterate* iiter = taosMemoryCalloc(1, sizeof(Iterate)); if (iiter == NULL) { return NULL; } - taosThreadMutexLock(&cache->mtx); indexMemRef(cache->imm); @@ -463,12 +465,9 @@ int indexCacheSchedToMerge(IndexCache* pCache) { schedMsg.fp = doMergeWork; schedMsg.ahandle = pCache; schedMsg.thandle = NULL; - // schedMsg.thandle = taosMemoryCalloc(1, sizeof(int64_t)); - // memcpy((char*)(schedMsg.thandle), (char*)&(pCache->index->refId), sizeof(int64_t)); schedMsg.msg = NULL; indexAcquireRef(pCache->index->refId); taosScheduleTask(indexQhandle, &schedMsg); - return 0; } @@ -533,6 +532,19 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { return 0; // encode end } +void indexCacheForceToMerge(void* cache) { + IndexCache* pCache = cache; + indexCacheRef(pCache); + taosThreadMutexLock(&pCache->mtx); + + indexInfo("%p is forced to merge into tfile", pCache); + pCache->occupiedMem += MEM_THRESHOLD * 5; + indexCacheMakeRoomForWrite(pCache); + + taosThreadMutexUnlock(&pCache->mtx); + indexCacheUnRef(pCache); + return; +} int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { IndexCache* pCache = cache; return 0; diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 733f1b4ed1..262bbfe55a 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -272,9 +272,26 @@ void validateFst() { } delete m; } +static std::string logDir = "/tmp/log"; + +static void initLog() { + const char* defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + sDebugFlag = 143; + strcpy(tsLogDir, logDir.c_str()); + taosRemoveDir(tsLogDir); + taosMkDir(tsLogDir); + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} class IndexEnv : public ::testing::Test { protected: virtual void SetUp() { + initLog(); taosRemoveDir(path); opts = indexOptsCreate(); int ret = indexOpen(opts, path, &index); @@ -804,7 +821,7 @@ class IndexObj { } ~IndexObj() { - indexCleanUp(); + // indexCleanUp(); indexClose(idx); } @@ -884,7 +901,7 @@ TEST_F(IndexEnv2, testIndexOpen) { SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); index->Search(mq, result); std::cout << "target size: " << taosArrayGetSize(result) << std::endl; - assert(taosArrayGetSize(result) == 400); + EXPECT_EQ(400, taosArrayGetSize(result)); taosArrayDestroy(result); indexMultiTermQueryDestroy(mq); } From 5acf419019848900ad7671f7e0590f9cca328092 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 21 May 2022 23:47:39 +0800 Subject: [PATCH 56/73] fix(os): win not close file ctest error --- include/os/osDir.h | 6 +++ source/dnode/mgmt/test/bnode/dbnode.cpp | 2 +- source/dnode/mgmt/test/mnode/dmnode.cpp | 2 +- source/dnode/mgmt/test/qnode/dqnode.cpp | 2 +- source/dnode/mgmt/test/snode/dsnode.cpp | 2 +- source/dnode/mgmt/test/sut/src/sut.cpp | 2 +- source/dnode/mgmt/test/vnode/vnode.cpp | 2 +- source/dnode/mnode/impl/test/acct/acct.cpp | 2 +- source/dnode/mnode/impl/test/bnode/mbnode.cpp | 4 +- source/dnode/mnode/impl/test/db/db.cpp | 2 +- source/dnode/mnode/impl/test/dnode/mdnode.cpp | 10 ++-- source/dnode/mnode/impl/test/func/func.cpp | 2 +- source/dnode/mnode/impl/test/mnode/mnode.cpp | 4 +- .../dnode/mnode/impl/test/profile/profile.cpp | 2 +- source/dnode/mnode/impl/test/qnode/qnode.cpp | 4 +- source/dnode/mnode/impl/test/sdb/sdbTest.cpp | 6 +-- source/dnode/mnode/impl/test/show/show.cpp | 2 +- source/dnode/mnode/impl/test/sma/sma.cpp | 2 +- source/dnode/mnode/impl/test/snode/snode.cpp | 4 +- source/dnode/mnode/impl/test/stb/stb.cpp | 2 +- source/dnode/mnode/impl/test/topic/topic.cpp | 2 +- source/dnode/mnode/impl/test/trans/trans1.cpp | 6 +-- source/dnode/mnode/impl/test/trans/trans2.cpp | 4 +- source/dnode/mnode/impl/test/user/user.cpp | 2 +- source/dnode/vnode/test/tqMetaTest.cpp | 2 +- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/groupoperator.c | 2 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/tlinearhash.c | 2 +- source/libs/executor/src/tsort.c | 4 +- .../executor/test/index_executor_tests.cpp | 2 +- source/libs/function/src/tpercentile.c | 2 +- source/libs/function/src/udfd.c | 2 +- source/libs/index/src/indexTfile.c | 2 +- source/libs/index/test/fstTest.cc | 4 +- source/libs/index/test/fstUT.cc | 2 +- source/libs/index/test/indexTests.cc | 46 +++++++++---------- source/libs/index/test/jsonUT.cc | 4 +- source/libs/planner/test/planTestMain.cpp | 2 +- source/libs/tfs/test/tfsTest.cpp | 20 ++++---- source/libs/transport/test/rclient.c | 2 +- source/libs/transport/test/rserver.c | 2 +- source/libs/transport/test/transUT.cpp | 6 +-- source/libs/wal/src/walWrite.c | 2 + source/libs/wal/test/walMetaTest.cpp | 9 ++-- source/util/test/cfgTest.cpp | 4 +- source/util/test/pageBufferTest.cpp | 6 +-- source/util/test/procTest.cpp | 6 +-- 48 files changed, 112 insertions(+), 103 deletions(-) diff --git a/include/os/osDir.h b/include/os/osDir.h index b549acde37..a4c686e280 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -31,6 +31,12 @@ extern "C" { #endif +#ifdef WINDOWS +#define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\" +#else +#define TD_TMP_DIR_PATH "/tmp/" +#endif + typedef struct TdDir *TdDirPtr; typedef struct TdDirEntry *TdDirEntryPtr; diff --git a/source/dnode/mgmt/test/bnode/dbnode.cpp b/source/dnode/mgmt/test/bnode/dbnode.cpp index 0568b30245..c2a9873e5b 100644 --- a/source/dnode/mgmt/test/bnode/dbnode.cpp +++ b/source/dnode/mgmt/test/bnode/dbnode.cpp @@ -14,7 +14,7 @@ class DndTestBnode : public ::testing::Test { protected: static void SetUpTestSuite() { - test.Init("/tmp/dbnodeTest", 9112); + test.Init(TD_TMP_DIR_PATH "dbnodeTest", 9112); taosMsleep(1100); } static void TearDownTestSuite() { test.Cleanup(); } diff --git a/source/dnode/mgmt/test/mnode/dmnode.cpp b/source/dnode/mgmt/test/mnode/dmnode.cpp index 98b50e96cf..8c945b50ac 100644 --- a/source/dnode/mgmt/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/test/mnode/dmnode.cpp @@ -13,7 +13,7 @@ class DndTestMnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dmnodeTest", 9114); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dmnodeTest", 9114); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mgmt/test/qnode/dqnode.cpp b/source/dnode/mgmt/test/qnode/dqnode.cpp index 2430419bef..ef51be47a6 100644 --- a/source/dnode/mgmt/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/test/qnode/dqnode.cpp @@ -13,7 +13,7 @@ class DndTestQnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dqnodeTest", 9111); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dqnodeTest", 9111); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mgmt/test/snode/dsnode.cpp b/source/dnode/mgmt/test/snode/dsnode.cpp index 9ade616f19..9ae0fbdc54 100644 --- a/source/dnode/mgmt/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/test/snode/dsnode.cpp @@ -13,7 +13,7 @@ class DndTestSnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dsnodeTest", 9113); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dsnodeTest", 9113); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index 7bfa0417af..6ef94481ea 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -48,7 +48,7 @@ void Testbase::Init(const char* path, int16_t port) { strcpy(tsDataDir, path); taosRemoveDir(path); taosMkDir(path); - InitLog("/tmp/td"); + InitLog(TD_TMP_DIR_PATH "td"); server.Start(); client.Init("root", "taosdata"); diff --git a/source/dnode/mgmt/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp index bddf951819..8aba4f81b5 100644 --- a/source/dnode/mgmt/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/test/vnode/vnode.cpp @@ -13,7 +13,7 @@ class DndTestVnode : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dvnodeTest", 9115); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "dvnodeTest", 9115); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp index 6dcb931ed5..46a9a465eb 100644 --- a/source/dnode/mnode/impl/test/acct/acct.cpp +++ b/source/dnode/mnode/impl/test/acct/acct.cpp @@ -13,7 +13,7 @@ class MndTestAcct : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/acctTest", 9012); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "acctTest", 9012); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/bnode/mbnode.cpp b/source/dnode/mnode/impl/test/bnode/mbnode.cpp index 316ac8cc36..c93e2142d0 100644 --- a/source/dnode/mnode/impl/test/bnode/mbnode.cpp +++ b/source/dnode/mnode/impl/test/bnode/mbnode.cpp @@ -18,11 +18,11 @@ class MndTestBnode : public ::testing::Test { public: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_bnode1", 9018); + test.Init(TD_TMP_DIR_PATH "mnode_test_bnode1", 9018); const char* fqdn = "localhost"; const char* firstEp = "localhost:9018"; - server2.Start("/tmp/mnode_test_bnode2", 9019); + server2.Start(TD_TMP_DIR_PATH "mnode_test_bnode2", 9019); taosMsleep(300); } diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 545f9f22bb..a1bab5d1d4 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -13,7 +13,7 @@ class MndTestDb : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_db", 9030); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_db", 9030); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/dnode/mdnode.cpp b/source/dnode/mnode/impl/test/dnode/mdnode.cpp index e63536d494..0b42b28219 100644 --- a/source/dnode/mnode/impl/test/dnode/mdnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/mdnode.cpp @@ -18,14 +18,14 @@ class MndTestDnode : public ::testing::Test { public: static void SetUpTestSuite() { - test.Init("/tmp/dnode_test_dnode1", 9023); + test.Init(TD_TMP_DIR_PATH "dnode_test_dnode1", 9023); const char* fqdn = "localhost"; const char* firstEp = "localhost:9023"; - // server2.Start("/tmp/dnode_test_dnode2", fqdn, 9024, firstEp); - // server3.Start("/tmp/dnode_test_dnode3", fqdn, 9025, firstEp); - // server4.Start("/tmp/dnode_test_dnode4", fqdn, 9026, firstEp); - // server5.Start("/tmp/dnode_test_dnode5", fqdn, 9027, firstEp); + // server2.Start(TD_TMP_DIR_PATH "dnode_test_dnode2", fqdn, 9024, firstEp); + // server3.Start(TD_TMP_DIR_PATH "dnode_test_dnode3", fqdn, 9025, firstEp); + // server4.Start(TD_TMP_DIR_PATH "dnode_test_dnode4", fqdn, 9026, firstEp); + // server5.Start(TD_TMP_DIR_PATH "dnode_test_dnode5", fqdn, 9027, firstEp); taosMsleep(300); } diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index c8f832160b..2bebe7ef19 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -13,7 +13,7 @@ class MndTestFunc : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_func", 9038); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_func", 9038); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/mnode/mnode.cpp b/source/dnode/mnode/impl/test/mnode/mnode.cpp index d953bdfdcb..1ed613c723 100644 --- a/source/dnode/mnode/impl/test/mnode/mnode.cpp +++ b/source/dnode/mnode/impl/test/mnode/mnode.cpp @@ -18,11 +18,11 @@ class MndTestMnode : public ::testing::Test { public: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_mnode1", 9028); + test.Init(TD_TMP_DIR_PATH "mnode_test_mnode1", 9028); const char* fqdn = "localhost"; const char* firstEp = "localhost:9028"; - // server2.Start("/tmp/mnode_test_mnode2", fqdn, 9029, firstEp); + // server2.Start(TD_TMP_DIR_PATH "mnode_test_mnode2", fqdn, 9029, firstEp); taosMsleep(300); } diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index 9c8e0298aa..794374a91d 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -13,7 +13,7 @@ class MndTestProfile : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_profile", 9031); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_profile", 9031); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index 87ba7caa4e..57b38e55c1 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -18,11 +18,11 @@ class MndTestQnode : public ::testing::Test { public: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_qnode1", 9014); + test.Init(TD_TMP_DIR_PATH "mnode_test_qnode1", 9014); const char* fqdn = "localhost"; const char* firstEp = "localhost:9014"; - // server2.Start("/tmp/mnode_test_qnode2", fqdn, 9015, firstEp); + // server2.Start(TD_TMP_DIR_PATH "mnode_test_qnode2", fqdn, 9015, firstEp); taosMsleep(300); } diff --git a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp index b93adf9930..f67ca2fb91 100644 --- a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp +++ b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp @@ -31,7 +31,7 @@ class MndTestSdb : public ::testing::Test { tsLogEmbedded = 1; tsAsyncLog = 0; - const char *path = "/tmp/td"; + const char *path = TD_TMP_DIR_PATH "td"; taosRemoveDir(path); taosMkDir(path); tstrncpy(tsLogDir, path, PATH_MAX); @@ -385,7 +385,7 @@ TEST_F(MndTestSdb, 01_Write_Str) { mnode.v100 = 100; mnode.v200 = 200; opt.pMnode = &mnode; - opt.path = "/tmp/mnode_test_sdb"; + opt.path = TD_TMP_DIR_PATH "mnode_test_sdb"; taosRemoveDir(opt.path); SSdbTable strTable1; @@ -730,7 +730,7 @@ TEST_F(MndTestSdb, 01_Read_Str) { mnode.v100 = 100; mnode.v200 = 200; opt.pMnode = &mnode; - opt.path = "/tmp/mnode_test_sdb"; + opt.path = TD_TMP_DIR_PATH "mnode_test_sdb"; SSdbTable strTable1; memset(&strTable1, 0, sizeof(SSdbTable)); diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index 5c431f65d3..0de8c9dca8 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -13,7 +13,7 @@ class MndTestShow : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_show", 9021); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_show", 9021); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/sma/sma.cpp b/source/dnode/mnode/impl/test/sma/sma.cpp index 4dc4e04779..d795816f57 100644 --- a/source/dnode/mnode/impl/test/sma/sma.cpp +++ b/source/dnode/mnode/impl/test/sma/sma.cpp @@ -13,7 +13,7 @@ class MndTestSma : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_sma", 9035); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_sma", 9035); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/snode/snode.cpp b/source/dnode/mnode/impl/test/snode/snode.cpp index 0b1d3c38b2..1828fbd570 100644 --- a/source/dnode/mnode/impl/test/snode/snode.cpp +++ b/source/dnode/mnode/impl/test/snode/snode.cpp @@ -18,11 +18,11 @@ class MndTestSnode : public ::testing::Test { public: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_snode1", 9016); + test.Init(TD_TMP_DIR_PATH "mnode_test_snode1", 9016); const char* fqdn = "localhost"; const char* firstEp = "localhost:9016"; - // server2.Start("/tmp/mnode_test_snode2", fqdn, 9017, firstEp); + // server2.Start(TD_TMP_DIR_PATH "mnode_test_snode2", fqdn, 9017, firstEp); taosMsleep(300); } diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index b8873210ab..56f1b8240d 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -13,7 +13,7 @@ class MndTestStb : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_stb", 9034); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_stb", 9034); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/topic/topic.cpp b/source/dnode/mnode/impl/test/topic/topic.cpp index eccc1b99d3..433a0ab5cc 100644 --- a/source/dnode/mnode/impl/test/topic/topic.cpp +++ b/source/dnode/mnode/impl/test/topic/topic.cpp @@ -13,7 +13,7 @@ class MndTestTopic : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_topic", 9039); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_topic", 9039); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/mnode/impl/test/trans/trans1.cpp b/source/dnode/mnode/impl/test/trans/trans1.cpp index 80109a39b2..5a470fc900 100644 --- a/source/dnode/mnode/impl/test/trans/trans1.cpp +++ b/source/dnode/mnode/impl/test/trans/trans1.cpp @@ -14,10 +14,10 @@ class MndTestTrans1 : public ::testing::Test { protected: static void SetUpTestSuite() { - test.Init("/tmp/mnode_test_trans1", 9013); + test.Init(TD_TMP_DIR_PATH "mnode_test_trans1", 9013); const char* fqdn = "localhost"; const char* firstEp = "localhost:9013"; - // server2.Start("/tmp/mnode_test_trans2", fqdn, 9020, firstEp); + // server2.Start(TD_TMP_DIR_PATH "mnode_test_trans2", fqdn, 9020, firstEp); } static void TearDownTestSuite() { @@ -26,7 +26,7 @@ class MndTestTrans1 : public ::testing::Test { } static void KillThenRestartServer() { - char file[PATH_MAX] = "/tmp/mnode_test_trans1/mnode/data/sdb.data"; + char file[PATH_MAX] = TD_TMP_DIR_PATH "mnode_test_trans1/mnode/data/sdb.data"; TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; void* buffer = taosMemoryMalloc(size); diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index c4ed48fe60..ec972cf784 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -41,7 +41,7 @@ class MndTestTrans2 : public ::testing::Test { tsLogEmbedded = 1; tsAsyncLog = 0; - const char *logpath = "/tmp/td"; + const char *logpath = TD_TMP_DIR_PATH "td"; taosRemoveDir(logpath); taosMkDir(logpath); tstrncpy(tsLogDir, logpath, PATH_MAX); @@ -68,7 +68,7 @@ class MndTestTrans2 : public ::testing::Test { tsTransPullupInterval = 1; - const char *mnodepath = "/tmp/mnode_test_trans"; + const char *mnodepath = TD_TMP_DIR_PATH "mnode_test_trans"; taosRemoveDir(mnodepath); pMnode = mndOpen(mnodepath, &opt); mndStart(pMnode); diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 9e4bd79274..6aa28a9007 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -13,7 +13,7 @@ class MndTestUser : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_user", 9011); } + static void SetUpTestSuite() { test.Init(TD_TMP_DIR_PATH "mnode_test_user", 9011); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; diff --git a/source/dnode/vnode/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp index 627dbc6f18..168daeb19f 100644 --- a/source/dnode/vnode/test/tqMetaTest.cpp +++ b/source/dnode/vnode/test/tqMetaTest.cpp @@ -41,7 +41,7 @@ class TqMetaUpdateAppendTest : public ::testing::Test { void TearDown() override { tqStoreClose(pMeta); } STqMetaStore* pMeta; - const char* pathName = "/tmp/tq_test"; + const char* pathName = TD_TMP_DIR_PATH "tq_test"; }; TEST_F(TqMetaUpdateAppendTest, copyPutTest) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4475cb9e62..b1b0e69084 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4059,7 +4059,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n defaultBufsz = defaultPgsz * 4; } - int32_t code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, "/tmp/"); + int32_t code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, TD_TMP_DIR_PATH); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 7606374cdb..484d439dea 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -614,7 +614,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - int32_t code = createDiskbasedBuf(&pInfo->pBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/"); + int32_t code = createDiskbasedBuf(&pInfo->pBuf, 4096, 4096 * 256, pTaskInfo->id.str, TD_TMP_DIR_PATH); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d4225caa71..d0eb0ae8da 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -990,7 +990,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, void* pDataR size_t childKeyBufSize = sizeof(int64_t) + sizeof(int64_t) + sizeof(TSKEY); initCatchSupporter(&pInfo->childAggSup, 1024, childKeyBufSize, - "StreamFinalInterval", "/tmp/"); // TODO(liuyao) get row size from phy plan + "StreamFinalInterval", TD_TMP_DIR_PATH); // TODO(liuyao) get row size from phy plan pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 21fd54b620..00a9f3ae6c 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -247,7 +247,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ return NULL; } - int32_t code = createDiskbasedBuf(&pHashObj->pBuf, pageSize, inMemPages * pageSize, 0, "/tmp"); + int32_t code = createDiskbasedBuf(&pHashObj->pBuf, pageSize, inMemPages * pageSize, 0, TD_TMP_DIR_PATH); if (code != 0) { terrno = code; return NULL; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index d585988e5e..c826cb68bf 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -155,7 +155,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { int32_t start = 0; if (pHandle->pBuf == NULL) { - int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "doAddToBuf", "/tmp"); + int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "doAddToBuf", TD_TMP_DIR_PATH); dBufSetPrintInfo(pHandle->pBuf); if (code != TSDB_CODE_SUCCESS) { return code; @@ -217,7 +217,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int } else { // multi-pass internal merge sort is required if (pHandle->pBuf == NULL) { - code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "sortComparInit", "/tmp"); + code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "sortComparInit", TD_TMP_DIR_PATH); dBufSetPrintInfo(pHandle->pBuf); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/executor/test/index_executor_tests.cpp b/source/libs/executor/test/index_executor_tests.cpp index 2449bd1da1..b0c2a983d1 100644 --- a/source/libs/executor/test/index_executor_tests.cpp +++ b/source/libs/executor/test/index_executor_tests.cpp @@ -57,7 +57,7 @@ void sifInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - strcpy(tsLogDir, "/tmp/sif"); + strcpy(tsLogDir, TD_TMP_DIR_PATH "sif"); taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index dd57024624..90d0640f40 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -255,7 +255,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); - int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", "/tmp"); + int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", TD_TMP_DIR_PATH); if (ret != 0) { tMemBucketDestroy(pBucket); return NULL; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 9185f70711..e1ebfa8254 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -144,7 +144,7 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { udf->bufSize = pFuncInfo->bufSize; char path[PATH_MAX] = {0}; - snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", pFuncInfo->name); + snprintf(path, sizeof(path), "%s/lib%s.so", TD_TMP_DIR_PATH, pFuncInfo->name); TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); // TODO check for failure of flush to disk diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index dd6117ed2a..52f8886b40 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -141,7 +141,7 @@ void tfileCacheDestroy(TFileCache* tcache) { TFileReader* p = *reader; indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType); - + tfileReaderDestroy(p); tfileReaderUnRef(p); reader = taosHashIterate(tcache->tableCache, reader); } diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 679e24f1a7..a2d7adf1c7 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -15,7 +15,7 @@ #include "tutil.h" void* callback(void* s) { return s; } -static std::string fileName = "/tmp/tindex.tindex"; +static std::string fileName = TD_TMP_DIR_PATH "tindex.tindex"; class FstWriter { public: FstWriter() { @@ -48,7 +48,7 @@ class FstWriter { class FstReadMemory { public: - FstReadMemory(int32_t size, const std::string& fileName = "/tmp/tindex.tindex") { + FstReadMemory(int32_t size, const std::string& fileName = TD_TMP_DIR_PATH "tindex.tindex") { _wc = writerCtxCreate(TFile, fileName.c_str(), true, 64 * 1024); _w = fstCountingWriterCreate(_wc); _size = size; diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index ab6c1a4704..136c4dafec 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -17,7 +17,7 @@ #include "tskiplist.h" #include "tutil.h" -static std::string dir = "/tmp/index"; +static std::string dir = TD_TMP_DIR_PATH "index"; static char indexlog[PATH_MAX] = {0}; static char tindex[PATH_MAX] = {0}; diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 733f1b4ed1..cff513b861 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -51,7 +51,7 @@ class DebugInfo { class FstWriter { public: FstWriter() { - _wc = writerCtxCreate(TFile, "/tmp/tindex", false, 64 * 1024 * 1024); + _wc = writerCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", false, 64 * 1024 * 1024); _b = fstBuilderCreate(NULL, 0); } bool Put(const std::string& key, uint64_t val) { @@ -75,7 +75,7 @@ class FstWriter { class FstReadMemory { public: FstReadMemory(size_t size) { - _wc = writerCtxCreate(TFile, "/tmp/tindex", true, 64 * 1024); + _wc = writerCtxCreate(TFile, TD_TMP_DIR_PATH "tindex", true, 64 * 1024); _w = fstCountingWriterCreate(_wc); _size = size; memset((void*)&_s, 0, sizeof(_s)); @@ -285,7 +285,7 @@ class IndexEnv : public ::testing::Test { indexOptsDestroy(opts); } - const char* path = "/tmp/tindex"; + const char* path = TD_TMP_DIR_PATH "tindex"; SIndexOpts* opts; SIndex* index; }; @@ -342,7 +342,7 @@ class IndexEnv : public ::testing::Test { class TFileObj { public: - TFileObj(const std::string& path = "/tmp/tindex", const std::string& colName = "voltage") + TFileObj(const std::string& path = TD_TMP_DIR_PATH "tindex", const std::string& colName = "voltage") : path_(path), colName_(colName) { colId_ = 10; reader_ = NULL; @@ -437,7 +437,7 @@ class IndexTFileEnv : public ::testing::Test { // tfileWriterDestroy(twrite); } TFileObj* fObj; - std::string dir = "/tmp/tindex"; + std::string dir = TD_TMP_DIR_PATH "tindex"; std::string colName = "voltage"; int coldId = 2; @@ -822,7 +822,7 @@ class IndexEnv2 : public ::testing::Test { IndexObj* index; }; TEST_F(IndexEnv2, testIndexOpen) { - std::string path = "/tmp/test"; + std::string path = TD_TMP_DIR_PATH "test"; if (index->Init(path) != 0) { std::cout << "failed to init index" << std::endl; exit(1); @@ -891,7 +891,7 @@ TEST_F(IndexEnv2, testIndexOpen) { } TEST_F(IndexEnv2, testIndex_TrigeFlush) { - std::string path = "/tmp/testxxx"; + std::string path = TD_TMP_DIR_PATH "testxxx"; if (index->Init(path) != 0) { // r std::cout << "failed to init" << std::endl; @@ -914,7 +914,7 @@ static void multi_write_and_search(IndexObj* idx) { idx->WriteMultiMillonData("tag2", "world test nothing", 100 * 10); } TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { // opt } @@ -934,7 +934,7 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { } } TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } @@ -950,14 +950,14 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { } // TEST_F(IndexEnv2, testIndex_restart) { -// std::string path = "/tmp/cache_and_tfile"; +// std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; // if (index->Init(path) != 0) { // } // index->SearchOneTarget("tag1", "Hello", 10); // index->SearchOneTarget("tag2", "Test", 10); //} // TEST_F(IndexEnv2, testIndex_restart1) { -// std::string path = "/tmp/cache_and_tfile"; +// std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; // if (index->Init(path) != 0) { // } // index->ReadMultiMillonData("tag1", "coding"); @@ -966,7 +966,7 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { //} // TEST_F(IndexEnv2, testIndex_read_performance) { -// std::string path = "/tmp/cache_and_tfile"; +// std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; // if (index->Init(path) != 0) { // } // index->PutOneTarge("tag1", "Hello", 12); @@ -976,7 +976,7 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { // assert(3 == index->SearchOne("tag1", "Hello")); //} // TEST_F(IndexEnv2, testIndexMultiTag) { -// std::string path = "/tmp/multi_tag"; +// std::string path = TD_TMP_DIR_PATH "multi_tag"; // if (index->Init(path) != 0) { // } // int64_t st = taosGetTimestampUs(); @@ -986,7 +986,7 @@ TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { // // index->WriteMultiMillonData("tag2", "xxxxxxxxxxxxxxxxxxxxxxxxx", 100 * 10000); //} TEST_F(IndexEnv2, testLongComVal1) { - std::string path = "/tmp/long_colVal"; + std::string path = TD_TMP_DIR_PATH "long_colVal"; if (index->Init(path) != 0) { } // gen colVal by randstr @@ -995,7 +995,7 @@ TEST_F(IndexEnv2, testLongComVal1) { } TEST_F(IndexEnv2, testLongComVal2) { - std::string path = "/tmp/long_colVal"; + std::string path = TD_TMP_DIR_PATH "long_colVal"; if (index->Init(path) != 0) { } // gen colVal by randstr @@ -1003,7 +1003,7 @@ TEST_F(IndexEnv2, testLongComVal2) { index->WriteMultiMillonData("tag1", randstr, 100 * 1000); } TEST_F(IndexEnv2, testLongComVal3) { - std::string path = "/tmp/long_colVal"; + std::string path = TD_TMP_DIR_PATH "long_colVal"; if (index->Init(path) != 0) { } // gen colVal by randstr @@ -1011,7 +1011,7 @@ TEST_F(IndexEnv2, testLongComVal3) { index->WriteMultiMillonData("tag1", randstr, 100 * 1000); } TEST_F(IndexEnv2, testLongComVal4) { - std::string path = "/tmp/long_colVal"; + std::string path = TD_TMP_DIR_PATH "long_colVal"; if (index->Init(path) != 0) { } // gen colVal by randstr @@ -1019,7 +1019,7 @@ TEST_F(IndexEnv2, testLongComVal4) { index->WriteMultiMillonData("tag1", randstr, 100 * 100); } TEST_F(IndexEnv2, testIndex_read_performance1) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } index->PutOneTarge("tag1", "Hello", 12); @@ -1029,7 +1029,7 @@ TEST_F(IndexEnv2, testIndex_read_performance1) { EXPECT_EQ(2, index->SearchOne("tag1", "Hello")); } TEST_F(IndexEnv2, testIndex_read_performance2) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } index->PutOneTarge("tag1", "Hello", 12); @@ -1039,7 +1039,7 @@ TEST_F(IndexEnv2, testIndex_read_performance2) { EXPECT_EQ(2, index->SearchOne("tag1", "Hello")); } TEST_F(IndexEnv2, testIndex_read_performance3) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } index->PutOneTarge("tag1", "Hello", 12); @@ -1049,7 +1049,7 @@ TEST_F(IndexEnv2, testIndex_read_performance3) { EXPECT_EQ(2, index->SearchOne("tag1", "Hello")); } TEST_F(IndexEnv2, testIndex_read_performance4) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } index->PutOneTarge("tag10", "Hello", 12); @@ -1059,7 +1059,7 @@ TEST_F(IndexEnv2, testIndex_read_performance4) { EXPECT_EQ(1, index->SearchOne("tag10", "Hello")); } TEST_F(IndexEnv2, testIndex_cache_del) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } for (int i = 0; i < 100; i++) { @@ -1098,7 +1098,7 @@ TEST_F(IndexEnv2, testIndex_cache_del) { } TEST_F(IndexEnv2, testIndex_del) { - std::string path = "/tmp/cache_and_tfile"; + std::string path = TD_TMP_DIR_PATH "cache_and_tfile"; if (index->Init(path) != 0) { } for (int i = 0; i < 100; i++) { diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index e827d1763f..8a837c5700 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -16,8 +16,8 @@ #include "tskiplist.h" #include "tutil.h" -static std::string dir = "/tmp/json"; -static std::string logDir = "/tmp/log"; +static std::string dir = TD_TMP_DIR_PATH "json"; +static std::string logDir = TD_TMP_DIR_PATH "log"; static void initLog() { const char* defaultLogFileNamePrefix = "taoslog"; diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 0373ab38d3..42c8558239 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -25,7 +25,7 @@ class PlannerEnv : public testing::Environment { virtual void SetUp() { initMetaDataEnv(); generateMetaData(); - initLog("/tmp/td"); + initLog(TD_TMP_DIR_PATH "td"); } virtual void TearDown() { destroyMetaDataEnv(); } diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 58c3a83aff..d53c4a49ba 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -16,7 +16,7 @@ class TfsTest : public ::testing::Test { protected: - static void SetUpTestSuite() { root = "/tmp/tfsTest"; } + static void SetUpTestSuite() { root = TD_TMP_DIR_PATH "tfsTest"; } static void TearDownTestSuite() {} public: @@ -299,15 +299,15 @@ TEST_F(TfsTest, 04_File) { TEST_F(TfsTest, 05_MultiDisk) { int32_t code = 0; - const char *root00 = "/tmp/tfsTest00"; - const char *root01 = "/tmp/tfsTest01"; - const char *root10 = "/tmp/tfsTest10"; - const char *root11 = "/tmp/tfsTest11"; - const char *root12 = "/tmp/tfsTest12"; - const char *root20 = "/tmp/tfsTest20"; - const char *root21 = "/tmp/tfsTest21"; - const char *root22 = "/tmp/tfsTest22"; - const char *root23 = "/tmp/tfsTest23"; + const char *root00 = TD_TMP_DIR_PATH "tfsTest00"; + const char *root01 = TD_TMP_DIR_PATH "tfsTest01"; + const char *root10 = TD_TMP_DIR_PATH "tfsTest10"; + const char *root11 = TD_TMP_DIR_PATH "tfsTest11"; + const char *root12 = TD_TMP_DIR_PATH "tfsTest12"; + const char *root20 = TD_TMP_DIR_PATH "tfsTest20"; + const char *root21 = TD_TMP_DIR_PATH "tfsTest21"; + const char *root22 = TD_TMP_DIR_PATH "tfsTest22"; + const char *root23 = TD_TMP_DIR_PATH "tfsTest23"; SDiskCfg dCfg[9] = {0}; tstrncpy(dCfg[0].dir, root01, TSDB_FILENAME_LEN); diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index eea76096ff..55e6dd000a 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -161,7 +161,7 @@ int main(int argc, char *argv[]) { } } - const char *path = "/tmp/transport/client"; + const char *path = TD_TMP_DIR_PATH "transport/client"; taosRemoveDir(path); taosMkDir(path); tstrncpy(tsLogDir, path, PATH_MAX); diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 6262b3ae48..1fd78be77d 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -160,7 +160,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - const char *path = "/tmp/transport/server"; + const char *path = TD_TMP_DIR_PATH "transport/server"; taosRemoveDir(path); taosMkDir(path); tstrncpy(tsLogDir, path, PATH_MAX); diff --git a/source/libs/transport/test/transUT.cpp b/source/libs/transport/test/transUT.cpp index 3f5ef1fb53..25b04e769c 100644 --- a/source/libs/transport/test/transUT.cpp +++ b/source/libs/transport/test/transUT.cpp @@ -43,7 +43,7 @@ static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); class Client { public: void Init(int nThread) { - memcpy(tsTempDir, "/tmp", strlen("/tmp")); + memcpy(tsTempDir, TD_TMP_DIR_PATH, strlen(TD_TMP_DIR_PATH)); memset(&rpcInit_, 0, sizeof(rpcInit_)); rpcInit_.localPort = 0; rpcInit_.label = (char *)label; @@ -105,7 +105,7 @@ class Client { class Server { public: Server() { - memcpy(tsTempDir, "/tmp", strlen("/tmp")); + memcpy(tsTempDir, TD_TMP_DIR_PATH, strlen(TD_TMP_DIR_PATH)); memset(&rpcInit_, 0, sizeof(rpcInit_)); memcpy(rpcInit_.localFqdn, "localhost", strlen("localhost")); @@ -219,7 +219,7 @@ static void initEnv() { tsLogEmbedded = 1; tsAsyncLog = 0; - std::string path = "/tmp/transport"; + std::string path = TD_TMP_DIR_PATH "transport"; // taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 2ddcb27835..d2a43c4107 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -121,6 +121,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { pWal->vers.lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; + taosCloseFile(&pIdxTFile); + taosCloseFile(&pLogTFile); // unlock taosThreadMutexUnlock(&pWal->mutex); diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 18345699b2..b1c673e87b 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -37,7 +37,7 @@ class WalCleanEnv : public ::testing::Test { } SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; }; class WalCleanDeleteEnv : public ::testing::Test { @@ -67,7 +67,7 @@ class WalCleanDeleteEnv : public ::testing::Test { } SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; }; class WalKeepEnv : public ::testing::Test { @@ -104,7 +104,7 @@ class WalKeepEnv : public ::testing::Test { } SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; }; class WalRetentionEnv : public ::testing::Test { @@ -141,7 +141,7 @@ class WalRetentionEnv : public ::testing::Test { } SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + const char* pathName = TD_TMP_DIR_PATH "wal_test"; }; TEST_F(WalCleanEnv, createNew) { @@ -325,6 +325,7 @@ TEST_F(WalKeepEnv, readHandleRead) { EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); } } + walCloseReadHandle(pRead); } TEST_F(WalRetentionEnv, repairMeta1) { diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp index cd13ebe8ae..c5d9e830d2 100644 --- a/source/util/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -59,7 +59,7 @@ TEST_F(CfgTest, 02_Basic) { EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0), 0); EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0), 0); EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0), 0); - EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp", 0), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0), 0); EXPECT_EQ(cfgGetSize(pConfig), 6); @@ -126,7 +126,7 @@ TEST_F(CfgTest, 02_Basic) { EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR); EXPECT_STREQ(pItem->name, "test_dir"); - EXPECT_STREQ(pItem->str, "/tmp"); + EXPECT_STREQ(pItem->str, TD_TMP_DIR_PATH); cfgCleanup(pConfig); } diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 5ad3cb42aa..eaf198a483 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -13,7 +13,7 @@ namespace { // simple test void simpleTest() { SDiskbasedBuf* pBuf = NULL; - int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4096, "", "/tmp/"); + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4096, "", TD_TMP_DIR_PATH); int32_t pageId = 0; int32_t groupId = 0; @@ -57,7 +57,7 @@ void simpleTest() { void writeDownTest() { SDiskbasedBuf* pBuf = NULL; - int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/"); + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", TD_TMP_DIR_PATH); int32_t pageId = 0; int32_t writePageId = 0; @@ -106,7 +106,7 @@ void writeDownTest() { void recyclePageTest() { SDiskbasedBuf* pBuf = NULL; - int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/"); + int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", TD_TMP_DIR_PATH); int32_t pageId = 0; int32_t writePageId = 0; diff --git a/source/util/test/procTest.cpp b/source/util/test/procTest.cpp index af53ddcea5..53d3fa2c4b 100644 --- a/source/util/test/procTest.cpp +++ b/source/util/test/procTest.cpp @@ -38,9 +38,9 @@ class UtilTesProc : public ::testing::Test { head.noResp = 3; head.persistHandle = 4; - taosRemoveDir("/tmp/td"); - taosMkDir("/tmp/td"); - tstrncpy(tsLogDir, "/tmp/td", PATH_MAX); + taosRemoveDir(TD_TMP_DIR_PATH "td"); + taosMkDir(TD_TMP_DIR_PATH "td"); + tstrncpy(tsLogDir, TD_TMP_DIR_PATH "td", PATH_MAX); if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } From 548838d9c35e8b2258f694ba1365b328cb2baa40 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 22 May 2022 16:03:46 +0800 Subject: [PATCH 57/73] enhance: udf code refactoring --- source/libs/function/src/tudf.c | 624 ++++++++++++++++++-------------- 1 file changed, 350 insertions(+), 274 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 5f20d2e50a..cba5af2850 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -46,6 +46,13 @@ typedef struct SUdfdData { SUdfdData udfdGlobal = {0}; static int32_t udfSpawnUdfd(SUdfdData *pData); +void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal); +static int32_t udfSpawnUdfd(SUdfdData* pData); +static void udfUdfdCloseWalkCb(uv_handle_t* handle, void* arg); +static void udfUdfdStopAsyncCb(uv_async_t *async); +static void udfWatchUdfd(void *args); +int32_t udfStartUdfd(int32_t startDnodeId); +int32_t udfStopUdfd(); void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { fnInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); @@ -413,6 +420,34 @@ enum { UDFC_STATE_STOPPING, // stopping after udfcClose }; +int32_t getUdfdPipeName(char* pipeName, int32_t size); +int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup); +void* decodeUdfSetupRequest(const void* buf, SUdfSetupRequest *request); +int32_t encodeUdfInterBuf(void **buf, const SUdfInterBuf* state); +void* decodeUdfInterBuf(const void* buf, SUdfInterBuf* state); +int32_t encodeUdfCallRequest(void **buf, const SUdfCallRequest *call); +void* decodeUdfCallRequest(const void* buf, SUdfCallRequest* call); +int32_t encodeUdfTeardownRequest(void **buf, const SUdfTeardownRequest *teardown); +void* decodeUdfTeardownRequest(const void* buf, SUdfTeardownRequest *teardown); +int32_t encodeUdfRequest(void** buf, const SUdfRequest* request); +void* decodeUdfRequest(const void* buf, SUdfRequest* request); +int32_t encodeUdfSetupResponse(void **buf, const SUdfSetupResponse *setupRsp); +void* decodeUdfSetupResponse(const void* buf, SUdfSetupResponse* setupRsp); +int32_t encodeUdfCallResponse(void **buf, const SUdfCallResponse *callRsp); +void* decodeUdfCallResponse(const void* buf, SUdfCallResponse* callRsp); +int32_t encodeUdfTeardownResponse(void** buf, const SUdfTeardownResponse* teardownRsp); +void* decodeUdfTeardownResponse(const void* buf, SUdfTeardownResponse* teardownResponse); +int32_t encodeUdfResponse(void** buf, const SUdfResponse* rsp); +void* decodeUdfResponse(const void* buf, SUdfResponse* rsp); +void freeUdfColumnData(SUdfColumnData *data, SUdfColumnMeta *meta); +void freeUdfColumn(SUdfColumn* col); +void freeUdfDataDataBlock(SUdfDataBlock *block); +void freeUdfInterBuf(SUdfInterBuf *buf); +int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock); +int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block); +int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output); +int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output); + int32_t getUdfdPipeName(char* pipeName, int32_t size) { char dnodeId[8] = {0}; size_t dnodeIdSize = sizeof(dnodeId); @@ -650,7 +685,7 @@ int32_t encodeUdfResponse(void** buf, const SUdfResponse* rsp) { len += encodeUdfTeardownResponse(buf, &rsp->teardownRsp); break; default: - //TODO: log error + fnError("encode udf response, invalid udf response type %d", rsp->type); break; } return len; @@ -676,7 +711,7 @@ void* decodeUdfResponse(const void* buf, SUdfResponse* rsp) { buf = decodeUdfTeardownResponse(buf, &rsp->teardownRsp); break; default: - //TODO: log error + fnError("decode udf response, invalid udf response type %d", rsp->type); break; } return (void*)buf; @@ -817,6 +852,319 @@ int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { return 0; } +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +//memory layout |---SUdfAggRes----|-----final result-----|---inter result----| +typedef struct SUdfAggRes { + int8_t finalResNum; + int8_t interResNum; + char* finalResBuf; + char* interResBuf; +} SUdfAggRes; +void onUdfcPipeClose(uv_handle_t *handle); +int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *uvTask); +void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf); +bool isUdfcUvMsgComplete(SClientConnBuf *connBuf); +void udfcUvHandleRsp(SClientUvConn *conn); +void udfcUvHandleError(SClientUvConn *conn); +void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); +void onUdfcPipetWrite(uv_write_t *write, int status); +void onUdfcPipeConnect(uv_connect_t *connect, int status); +int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask); +int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask); +int32_t udfcStartUvTask(SClientUvTaskNode *uvTask); +void udfcAsyncTaskCb(uv_async_t *async); +void cleanUpUvTasks(SUdfcProxy *udfc); +void udfStopAsyncCb(uv_async_t *async); +void constructUdfService(void *argsThread); +int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType); +int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle); +int compareUdfcFuncSub(const void* elem1, const void* elem2); +int32_t doTeardownUdf(UdfcFuncHandle handle); + +int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, + SSDataBlock* output, SUdfInterBuf *newState); +int32_t doCallUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf); +int32_t doCallUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState); +int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf); +int32_t doCallUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); +int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam* output); +int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output); + +int32_t udfcOpen(); +int32_t udfcClose(); + +int32_t acquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle); +void releaseUdfFuncHandle(char* udfName); +int32_t cleanUpUdfs(); + +bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); +int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); +int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); + +int compareUdfcFuncSub(const void* elem1, const void* elem2) { + SUdfcFuncStub *stub1 = (SUdfcFuncStub *)elem1; + SUdfcFuncStub *stub2 = (SUdfcFuncStub *)elem2; + return strcmp(stub1->udfName, stub2->udfName); +} + +int32_t acquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) { + int32_t code = 0; + uv_mutex_lock(&gUdfdProxy.udfStubsMutex); + SUdfcFuncStub key = {0}; + strcpy(key.udfName, udfName); + int32_t stubIndex = taosArraySearchIdx(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); + if (stubIndex != -1) { + SUdfcFuncStub *foundStub = taosArrayGet(gUdfdProxy.udfStubs, stubIndex); + UdfcFuncHandle handle = foundStub->handle; + if (handle != NULL && ((SUdfcUvSession*)handle)->udfUvPipe != NULL) { + *pHandle = foundStub->handle; + ++foundStub->refCount; + foundStub->lastRefTime = taosGetTimestampUs(); + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); + return 0; + } else { + fnInfo("invalid handle for %s, refCount: %d, last ref time: %"PRId64". remove it from cache", + udfName, foundStub->refCount, foundStub->lastRefTime); + taosArrayRemove(gUdfdProxy.udfStubs, stubIndex); + } + } + *pHandle = NULL; + code = doSetupUdf(udfName, pHandle); + if (code == TSDB_CODE_SUCCESS) { + SUdfcFuncStub stub = {0}; + strcpy(stub.udfName, udfName); + stub.handle = *pHandle; + ++stub.refCount; + stub.lastRefTime = taosGetTimestampUs(); + taosArrayPush(gUdfdProxy.udfStubs, &stub); + taosArraySort(gUdfdProxy.udfStubs, compareUdfcFuncSub); + } else { + *pHandle = NULL; + } + + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); + return code; +} + +void releaseUdfFuncHandle(char* udfName) { + uv_mutex_lock(&gUdfdProxy.udfStubsMutex); + SUdfcFuncStub key = {0}; + strcpy(key.udfName, udfName); + SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); + if (!foundStub) { + return; + } + if (foundStub->refCount > 0) { + --foundStub->refCount; + } + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); +} + +int32_t cleanUpUdfs() { + uv_mutex_lock(&gUdfdProxy.udfStubsMutex); + int32_t i = 0; + SArray* udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); + while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) { + SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i); + if (stub->refCount == 0) { + fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); + doTeardownUdf(stub->handle); + } else { + fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %"PRId64", handle: %p", + stub->udfName, stub->refCount, stub->lastRefTime, stub->handle); + UdfcFuncHandle handle = stub->handle; + if (handle != NULL && ((SUdfcUvSession*)handle)->udfUvPipe != NULL) { + taosArrayPush(udfStubs, stub); + } else { + fnInfo("udf invalid handle for %s, refCount: %d, last ref time: %"PRId64". remove it from cache", + stub->udfName, stub->refCount, stub->lastRefTime); + } + } + ++i; + } + taosArrayDestroy(gUdfdProxy.udfStubs); + gUdfdProxy.udfStubs = udfStubs; + uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); + return 0; +} + +int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output) { + UdfcFuncHandle handle = NULL; + int32_t code = acquireUdfFuncHandle(udfName, &handle); + if (code != 0) { + return code; + } + SUdfcUvSession *session = handle; + code = doCallUdfScalarFunc(handle, input, numOfCols, output); + if (output->columnData == NULL) { + fnError("udfc scalar function calculate error. no column data"); + code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } else { + if (session->outputType != output->columnData->info.type || session->outputLen != output->columnData->info.bytes) { + fnError("udfc scalar function calculate error. type mismatch. session type: %d(%d), output type: %d(%d)", session->outputType, + session->outputLen, output->columnData->info.type, output->columnData->info.bytes); + code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } + } + releaseUdfFuncHandle(udfName); + return code; +} + +bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + if (fmIsScalarFunc(pFunc->funcId)) { + return false; + } + pEnv->calcMemSize = sizeof(SUdfAggRes) + pFunc->node.resType.bytes + pFunc->udfBufSize; + return true; +} + +bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo) { + if (functionSetup(pCtx, pResultCellInfo) != true) { + return false; + } + UdfcFuncHandle handle; + int32_t udfCode = 0; + if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { + fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode); + return false; + } + SUdfcUvSession *session = (SUdfcUvSession *)handle; + SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo); + int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize; + memset(udfRes, 0, envSize); + + udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); + udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + + SUdfInterBuf buf = {0}; + if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { + fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); + releaseUdfFuncHandle(pCtx->udfName); + return false; + } + udfRes->interResNum = buf.numOfResult; + if (buf.bufLen <= session->bufSize) { + memcpy(udfRes->interResBuf, buf.buf, buf.bufLen); + } else { + fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); + releaseUdfFuncHandle(pCtx->udfName); + return false; + } + releaseUdfFuncHandle(pCtx->udfName); + freeUdfInterBuf(&buf); + return true; +} + +int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { + int32_t udfCode = 0; + UdfcFuncHandle handle = 0; + if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { + fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); + return udfCode; + } + + SUdfcUvSession *session = handle; + SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); + udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + + SInputColumnInfoData* pInput = &pCtx->input; + int32_t numOfCols = pInput->numOfInputCols; + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + + SSDataBlock tempBlock = {0}; + tempBlock.info.numOfCols = numOfCols; + tempBlock.info.rows = pInput->totalRows; + tempBlock.info.uid = pInput->uid; + bool hasVarCol = false; + tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData *col = pInput->pData[i]; + if (IS_VAR_DATA_TYPE(col->info.type)) { + hasVarCol = true; + } + taosArrayPush(tempBlock.pDataBlock, col); + } + tempBlock.info.hasVarCol = hasVarCol; + + SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows); + + SUdfInterBuf state = {.buf = udfRes->interResBuf, + .bufLen = session->bufSize, + .numOfResult = udfRes->interResNum}; + SUdfInterBuf newState = {0}; + + udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState); + if (udfCode != 0) { + fnError("udfAggProcess error. code: %d", udfCode); + newState.numOfResult = 0; + } else { + udfRes->interResNum = newState.numOfResult; + if (newState.bufLen <= session->bufSize) { + memcpy(udfRes->interResBuf, newState.buf, newState.bufLen); + } else { + fnError("udfc inter buf size %d is greater than function bufSize %d", newState.bufLen, session->bufSize); + udfCode = TSDB_CODE_UDF_INVALID_BUFSIZE; + } + } + if (newState.numOfResult == 1 || state.numOfResult == 1) { + GET_RES_INFO(pCtx)->numOfRes = 1; + } + + blockDataDestroy(inputBlock); + taosArrayDestroy(tempBlock.pDataBlock); + + releaseUdfFuncHandle(pCtx->udfName); + freeUdfInterBuf(&newState); + return udfCode; +} + +int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { + int32_t udfCode = 0; + UdfcFuncHandle handle = 0; + if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { + fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); + return udfCode; + } + + SUdfcUvSession *session = handle; + SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); + udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + + + SUdfInterBuf resultBuf = {0}; + SUdfInterBuf state = {.buf = udfRes->interResBuf, + .bufLen = session->bufSize, + .numOfResult = udfRes->interResNum}; + int32_t udfCallCode= 0; + udfCallCode= doCallUdfAggFinalize(session, &state, &resultBuf); + if (udfCallCode != 0) { + fnError("udfAggFinalize error. doCallUdfAggFinalize step. udf code:%d", udfCallCode); + GET_RES_INFO(pCtx)->numOfRes = 0; + } else { + if (resultBuf.bufLen <= session->outputLen) { + memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen); + udfRes->finalResNum = resultBuf.numOfResult; + GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum; + } else { + fnError("udfc inter buf size %d is greater than function output size %d", resultBuf.bufLen, session->outputLen); + GET_RES_INFO(pCtx)->numOfRes = 0; + udfCallCode = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } + } + + freeUdfInterBuf(&resultBuf); + + int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); + releaseUdfFuncHandle(pCtx->udfName); + return udfCallCode == 0 ? numOfResults : udfCallCode; +} + void onUdfcPipeClose(uv_handle_t *handle) { SClientUvConn *conn = handle->data; if (!QUEUE_EMPTY(&conn->taskQueue)) { @@ -1314,93 +1662,6 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { return err; } -int compareUdfcFuncSub(const void* elem1, const void* elem2) { - SUdfcFuncStub *stub1 = (SUdfcFuncStub *)elem1; - SUdfcFuncStub *stub2 = (SUdfcFuncStub *)elem2; - return strcmp(stub1->udfName, stub2->udfName); -} - -int32_t acquireUdfFuncHandle(char* udfName, UdfcFuncHandle* pHandle) { - int32_t code = 0; - uv_mutex_lock(&gUdfdProxy.udfStubsMutex); - SUdfcFuncStub key = {0}; - strcpy(key.udfName, udfName); - int32_t stubIndex = taosArraySearchIdx(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); - if (stubIndex != -1) { - SUdfcFuncStub *foundStub = taosArrayGet(gUdfdProxy.udfStubs, stubIndex); - UdfcFuncHandle handle = foundStub->handle; - if (handle != NULL && ((SUdfcUvSession*)handle)->udfUvPipe != NULL) { - *pHandle = foundStub->handle; - ++foundStub->refCount; - foundStub->lastRefTime = taosGetTimestampUs(); - uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); - return 0; - } else { - fnInfo("invalid handle for %s, refCount: %d, last ref time: %"PRId64". remove it from cache", - udfName, foundStub->refCount, foundStub->lastRefTime); - taosArrayRemove(gUdfdProxy.udfStubs, stubIndex); - } - } - *pHandle = NULL; - code = doSetupUdf(udfName, pHandle); - if (code == TSDB_CODE_SUCCESS) { - SUdfcFuncStub stub = {0}; - strcpy(stub.udfName, udfName); - stub.handle = *pHandle; - ++stub.refCount; - stub.lastRefTime = taosGetTimestampUs(); - taosArrayPush(gUdfdProxy.udfStubs, &stub); - taosArraySort(gUdfdProxy.udfStubs, compareUdfcFuncSub); - } else { - *pHandle = NULL; - } - - uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); - return code; -} - -void releaseUdfFuncHandle(char* udfName) { - uv_mutex_lock(&gUdfdProxy.udfStubsMutex); - SUdfcFuncStub key = {0}; - strcpy(key.udfName, udfName); - SUdfcFuncStub *foundStub = taosArraySearch(gUdfdProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); - if (!foundStub) { - return; - } - if (foundStub->refCount > 0) { - --foundStub->refCount; - } - uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); -} - -int32_t cleanUpUdfs() { - uv_mutex_lock(&gUdfdProxy.udfStubsMutex); - int32_t i = 0; - SArray* udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); - while (i < taosArrayGetSize(gUdfdProxy.udfStubs)) { - SUdfcFuncStub *stub = taosArrayGet(gUdfdProxy.udfStubs, i); - if (stub->refCount == 0) { - fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); - doTeardownUdf(stub->handle); - } else { - fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %"PRId64", handle: %p", - stub->udfName, stub->refCount, stub->lastRefTime, stub->handle); - UdfcFuncHandle handle = stub->handle; - if (handle != NULL && ((SUdfcUvSession*)handle)->udfUvPipe != NULL) { - taosArrayPush(udfStubs, stub); - } else { - fnInfo("udf invalid handle for %s, refCount: %d, last ref time: %"PRId64". remove it from cache", - stub->udfName, stub->refCount, stub->lastRefTime); - } - } - ++i; - } - taosArrayDestroy(gUdfdProxy.udfStubs); - gUdfdProxy.udfStubs = udfStubs; - uv_mutex_unlock(&gUdfdProxy.udfStubsMutex); - return 0; -} - int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, SSDataBlock* output, SUdfInterBuf *newState) { fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle); @@ -1524,29 +1785,6 @@ int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t return err; } - -int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output) { - UdfcFuncHandle handle = NULL; - int32_t code = acquireUdfFuncHandle(udfName, &handle); - if (code != 0) { - return code; - } - SUdfcUvSession *session = handle; - code = doCallUdfScalarFunc(handle, input, numOfCols, output); - if (output->columnData == NULL) { - fnError("udfc scalar function calculate error. no column data"); - code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; - } else { - if (session->outputType != output->columnData->info.type || session->outputLen != output->columnData->info.bytes) { - fnError("udfc scalar function calculate error. type mismatch. session type: %d(%d), output type: %d(%d)", session->outputType, - session->outputLen, output->columnData->info.type, output->columnData->info.bytes); - code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; - } - } - releaseUdfFuncHandle(udfName); - return code; -} - int32_t doTeardownUdf(UdfcFuncHandle handle) { SUdfcUvSession *session = (SUdfcUvSession *) handle; @@ -1576,165 +1814,3 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { return err; } - -//memory layout |---SUdfAggRes----|-----final result-----|---inter result----| -typedef struct SUdfAggRes { - int8_t finalResNum; - int8_t interResNum; - char* finalResBuf; - char* interResBuf; -} SUdfAggRes; - -bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { - if (fmIsScalarFunc(pFunc->funcId)) { - return false; - } - pEnv->calcMemSize = sizeof(SUdfAggRes) + pFunc->node.resType.bytes + pFunc->udfBufSize; - return true; -} - -bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo) { - if (functionSetup(pCtx, pResultCellInfo) != true) { - return false; - } - UdfcFuncHandle handle; - int32_t udfCode = 0; - if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { - fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode); - return false; - } - SUdfcUvSession *session = (SUdfcUvSession *)handle; - SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo); - int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize; - memset(udfRes, 0, envSize); - - udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - - SUdfInterBuf buf = {0}; - if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { - fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); - releaseUdfFuncHandle(pCtx->udfName); - return false; - } - udfRes->interResNum = buf.numOfResult; - if (buf.bufLen <= session->bufSize) { - memcpy(udfRes->interResBuf, buf.buf, buf.bufLen); - } else { - fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); - releaseUdfFuncHandle(pCtx->udfName); - return false; - } - releaseUdfFuncHandle(pCtx->udfName); - freeUdfInterBuf(&buf); - return true; -} - -int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { - int32_t udfCode = 0; - UdfcFuncHandle handle = 0; - if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { - fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); - return udfCode; - } - - SUdfcUvSession *session = handle; - SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - - SInputColumnInfoData* pInput = &pCtx->input; - int32_t numOfCols = pInput->numOfInputCols; - int32_t start = pInput->startRowIndex; - int32_t numOfRows = pInput->numOfRows; - - - SSDataBlock tempBlock = {0}; - tempBlock.info.numOfCols = numOfCols; - tempBlock.info.rows = pInput->totalRows; - tempBlock.info.uid = pInput->uid; - bool hasVarCol = false; - tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData *col = pInput->pData[i]; - if (IS_VAR_DATA_TYPE(col->info.type)) { - hasVarCol = true; - } - taosArrayPush(tempBlock.pDataBlock, col); - } - tempBlock.info.hasVarCol = hasVarCol; - - SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows); - - SUdfInterBuf state = {.buf = udfRes->interResBuf, - .bufLen = session->bufSize, - .numOfResult = udfRes->interResNum}; - SUdfInterBuf newState = {0}; - - udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState); - if (udfCode != 0) { - fnError("udfAggProcess error. code: %d", udfCode); - newState.numOfResult = 0; - } else { - udfRes->interResNum = newState.numOfResult; - if (newState.bufLen <= session->bufSize) { - memcpy(udfRes->interResBuf, newState.buf, newState.bufLen); - } else { - fnError("udfc inter buf size %d is greater than function bufSize %d", newState.bufLen, session->bufSize); - udfCode = TSDB_CODE_UDF_INVALID_BUFSIZE; - } - } - if (newState.numOfResult == 1 || state.numOfResult == 1) { - GET_RES_INFO(pCtx)->numOfRes = 1; - } - - blockDataDestroy(inputBlock); - taosArrayDestroy(tempBlock.pDataBlock); - - releaseUdfFuncHandle(pCtx->udfName); - freeUdfInterBuf(&newState); - return udfCode; -} - -int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { - int32_t udfCode = 0; - UdfcFuncHandle handle = 0; - if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { - fnError("udfAggProcess error. step acquireUdfFuncHandle. udf code: %d", udfCode); - return udfCode; - } - - SUdfcUvSession *session = handle; - SUdfAggRes* udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - - - SUdfInterBuf resultBuf = {0}; - SUdfInterBuf state = {.buf = udfRes->interResBuf, - .bufLen = session->bufSize, - .numOfResult = udfRes->interResNum}; - int32_t udfCallCode= 0; - udfCallCode= doCallUdfAggFinalize(session, &state, &resultBuf); - if (udfCallCode != 0) { - fnError("udfAggFinalize error. doCallUdfAggFinalize step. udf code:%d", udfCallCode); - GET_RES_INFO(pCtx)->numOfRes = 0; - } else { - if (resultBuf.bufLen <= session->outputLen) { - memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen); - udfRes->finalResNum = resultBuf.numOfResult; - GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum; - } else { - fnError("udfc inter buf size %d is greater than function output size %d", resultBuf.bufLen, session->outputLen); - GET_RES_INFO(pCtx)->numOfRes = 0; - udfCallCode = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; - } - } - - freeUdfInterBuf(&resultBuf); - - int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); - releaseUdfFuncHandle(pCtx->udfName); - return udfCallCode == 0 ? numOfResults : udfCallCode; -} From 712217ab024c52dfe3808804cee6010a46fe8368 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 22 May 2022 16:08:39 +0800 Subject: [PATCH 58/73] enhance: udf code refactoring --- source/libs/function/src/tudf.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index cba5af2850..d5dc967fed 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -24,7 +24,6 @@ #include "builtinsimpl.h" #include "functionMgt.h" -//TODO: add unit test typedef struct SUdfdData { bool startCalled; bool needCleanUp; @@ -45,14 +44,15 @@ typedef struct SUdfdData { SUdfdData udfdGlobal = {0}; +int32_t udfStartUdfd(int32_t startDnodeId); +int32_t udfStopUdfd(); + static int32_t udfSpawnUdfd(SUdfdData *pData); void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal); static int32_t udfSpawnUdfd(SUdfdData* pData); static void udfUdfdCloseWalkCb(uv_handle_t* handle, void* arg); static void udfUdfdStopAsyncCb(uv_async_t *async); static void udfWatchUdfd(void *args); -int32_t udfStartUdfd(int32_t startDnodeId); -int32_t udfStopUdfd(); void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { fnInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); @@ -1191,18 +1191,15 @@ int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode * switch (task->type) { case UDF_TASK_SETUP: { - //TODO: copy or not task->_setup.rsp = rsp.setupRsp; break; } case UDF_TASK_CALL: { task->_call.rsp = rsp.callRsp; - //TODO: copy or not break; } case UDF_TASK_TEARDOWN: { task->_teardown.rsp = rsp.teardownRsp; - //TODO: copy or not? break; } default: { @@ -1398,7 +1395,7 @@ int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN request.teardown = task->_teardown.req; request.type = UDF_TASK_TEARDOWN; } else { - //TODO log and return error + fnError("udfc create uv task, invalid task type : %d", task->type); } int32_t bufLen = encodeUdfRequest(NULL, &request); request.msgLen = bufLen; From 7976b6abfdbf849b9902f26ef9c61974fad46903 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 22 May 2022 16:29:41 +0800 Subject: [PATCH 59/73] feat:udf refactoring --- source/libs/function/src/udfd.c | 277 ++++++++++++++++++-------------- 1 file changed, 157 insertions(+), 120 deletions(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 9185f70711..abb8cefed6 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -103,6 +103,42 @@ typedef struct SUdfdRpcSendRecvInfo { uv_sem_t resultSem; } SUdfdRpcSendRecvInfo; +static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); +static int32_t udfdConnectToMnode(); +static int32_t udfdLoadUdf(char *udfName, SUdf *udf); +static bool udfdRpcRfp(int32_t code); +static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); +static int32_t udfdOpenClientRpc(); +static int32_t udfdCloseClientRpc(); + +static void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request); +static void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request); +static void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request); +static void udfdProcessRequest(uv_work_t *req); +static void udfdOnWrite(uv_write_t *req, int status); +static void udfdSendResponse(uv_work_t *work, int status); +static void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf); +static bool isUdfdUvMsgComplete(SUdfdUvConn *pipe); +static void udfdHandleRequest(SUdfdUvConn *conn); +static void udfdPipeCloseCb(uv_handle_t *pipe); +static void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn->client, udfdPipeCloseCb); } +static void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); +static void udfdOnNewConnection(uv_stream_t *server, int status); + +static void udfdIntrSignalHandler(uv_signal_t *handle, int signum); +static int32_t removeListeningPipe(); + +static void udfdPrintVersion(); +static int32_t udfdParseArgs(int32_t argc, char *argv[]); +static int32_t udfdInitLog(); + +static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf); +static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf); +static int32_t udfdUvInit(); +static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); +static int32_t udfdRun(); + void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->info.ahandle; ASSERT(pMsg->info.ahandle != NULL); @@ -144,11 +180,14 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { udf->bufSize = pFuncInfo->bufSize; char path[PATH_MAX] = {0}; - snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", pFuncInfo->name); + snprintf(path, sizeof(path), "%s/lib%s.so", tsTempDir, pFuncInfo->name); TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); - // TODO check for failure of flush to disk - taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); + int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); + if (count != pFuncInfo->codeSize) { + fnError("udfd write udf shared library failed"); + msgInfo->code = TSDB_CODE_FILE_CORRUPTED; + } taosCloseFile(&file); strncpy(udf->path, path, strlen(path)); tFreeSFuncInfo(pFuncInfo); @@ -275,6 +314,104 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { } return 0; } +static bool udfdRpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT) { + return true; + } else { + return false; + } +} + +int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) { + pEpSet->version = 0; + + // init mnode ip set + SEpSet *mgmtEpSet = &(pEpSet->epSet); + mgmtEpSet->numOfEps = 0; + mgmtEpSet->inUse = 0; + + if (firstEp && firstEp[0] != 0) { + if (strlen(firstEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + if (code != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return terrno; + } + + mgmtEpSet->numOfEps++; + } + + if (secondEp && secondEp[0] != 0) { + if (strlen(secondEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); + mgmtEpSet->numOfEps++; + } + + if (mgmtEpSet->numOfEps == 0) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + return 0; +} + +int32_t udfdOpenClientRpc() { + SRpcInit rpcInit = {0}; + rpcInit.label = "UDFD"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = (RpcCfp)udfdProcessRpcRsp; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = TSDB_DEFAULT_USER; + rpcInit.parent = &global; + rpcInit.rfp = udfdRpcRfp; + + global.clientRpc = rpcOpen(&rpcInit); + if (global.clientRpc == NULL) { + fnError("failed to init dnode rpc client"); + return -1; + } + return 0; +} + +int32_t udfdCloseClientRpc() { + rpcClose(global.clientRpc); + return 0; +} + +void udfdProcessRequest(uv_work_t *req) { + SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); + SUdfRequest request = {0}; + decodeUdfRequest(uvUdf->input.base, &request); + + switch (request.type) { + case UDF_TASK_SETUP: { + udfdProcessSetupRequest(uvUdf, &request); + break; + } + + case UDF_TASK_CALL: { + udfdProcessCallRequest(uvUdf, &request); + break; + } + case UDF_TASK_TEARDOWN: { + udfdProcessTeardownRequest(uvUdf, &request); + break; + } + default: { + break; + } + } +} void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { // TODO: tracable id from client. connect, setup, call, teardown @@ -471,31 +608,6 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { return; } -void udfdProcessRequest(uv_work_t *req) { - SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); - SUdfRequest request = {0}; - decodeUdfRequest(uvUdf->input.base, &request); - - switch (request.type) { - case UDF_TASK_SETUP: { - udfdProcessSetupRequest(uvUdf, &request); - break; - } - - case UDF_TASK_CALL: { - udfdProcessCallRequest(uvUdf, &request); - break; - } - case UDF_TASK_TEARDOWN: { - udfdProcessTeardownRequest(uvUdf, &request); - break; - } - default: { - break; - } - } -} - void udfdOnWrite(uv_write_t *req, int status) { SUvUdfWork *work = (SUvUdfWork *)req->data; if (status < 0) { @@ -529,7 +641,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->base = ctx->inputBuf; buf->len = ctx->inputCap; } else { - // TODO: log error + fnError("udfd can not allocate enough memory") buf->base = NULL; buf->len = 0; } @@ -541,7 +653,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->base = ctx->inputBuf + ctx->inputLen; buf->len = ctx->inputCap - ctx->inputLen; } else { - // TODO: log error + fnError("udfd can not allocate enough memory") buf->base = NULL; buf->len = 0; } @@ -580,8 +692,6 @@ void udfdPipeCloseCb(uv_handle_t *pipe) { taosMemoryFree(conn); } -void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn->client, udfdPipeCloseCb); } - void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { fnDebug("udf read %zd bytes from client", nread); if (nread == 0) return; @@ -638,91 +748,6 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) { uv_stop(global.loop); } -static bool udfdRpcRfp(int32_t code) { - if (code == TSDB_CODE_RPC_REDIRECT) { - return true; - } else { - return false; - } -} - -int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) { - pEpSet->version = 0; - - // init mnode ip set - SEpSet *mgmtEpSet = &(pEpSet->epSet); - mgmtEpSet->numOfEps = 0; - mgmtEpSet->inUse = 0; - - if (firstEp && firstEp[0] != 0) { - if (strlen(firstEp) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - - int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); - if (code != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return terrno; - } - - mgmtEpSet->numOfEps++; - } - - if (secondEp && secondEp[0] != 0) { - if (strlen(secondEp) >= TSDB_EP_LEN) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - - taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); - mgmtEpSet->numOfEps++; - } - - if (mgmtEpSet->numOfEps == 0) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; - } - - return 0; -} - -int32_t udfdOpenClientRpc() { - SRpcInit rpcInit = {0}; - rpcInit.label = "UDFD"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = (RpcCfp)udfdProcessRpcRsp; - rpcInit.sessions = 1024; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.user = TSDB_DEFAULT_USER; - rpcInit.parent = &global; - rpcInit.rfp = udfdRpcRfp; - - global.clientRpc = rpcOpen(&rpcInit); - if (global.clientRpc == NULL) { - fnError("failed to init dnode rpc client"); - return -1; - } - return 0; -} - -int32_t udfdCloseClientRpc() { - rpcClose(global.clientRpc); - return 0; -} - -static void udfdPrintVersion() { -#ifdef TD_ENTERPRISE - char *releaseName = "enterprise"; -#else - char *releaseName = "community"; -#endif - printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); - printf("gitinfo: %s\n", gitinfo); - printf("buildInfo: %s\n", buildinfo); -} - static int32_t udfdParseArgs(int32_t argc, char *argv[]) { for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { @@ -745,6 +770,17 @@ static int32_t udfdParseArgs(int32_t argc, char *argv[]) { return 0; } +static void udfdPrintVersion() { +#ifdef TD_ENTERPRISE + char *releaseName = "enterprise"; +#else + char *releaseName = "community"; +#endif + printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); + printf("gitinfo: %s\n", gitinfo); + printf("buildInfo: %s\n", buildinfo); +} + static int32_t udfdInitLog() { char logName[12] = {0}; snprintf(logName, sizeof(logName), "%slog", "udfd"); @@ -868,8 +904,8 @@ int main(int argc, char *argv[]) { int32_t retryMnodeTimes = 0; int32_t code = 0; - while (retryMnodeTimes++ < TSDB_MAX_REPLICA) { - uv_sleep(500 * (1 << retryMnodeTimes)); + while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) { + uv_sleep(100 * (1 << retryMnodeTimes)); code = udfdConnectToMnode(); if (code == 0) { break; @@ -890,6 +926,7 @@ int main(int argc, char *argv[]) { udfdRun(); removeListeningPipe(); - udfdCloseClientRpc(); + + return 0; } From e56a650877fc5d495942379d22bc97e0982cef83 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 22 May 2022 16:31:37 +0800 Subject: [PATCH 60/73] udf: refactor code --- source/libs/function/src/udfd.c | 440 ++++++++++++++++---------------- 1 file changed, 220 insertions(+), 220 deletions(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index abb8cefed6..964385e663 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -139,6 +139,226 @@ static int32_t udfdUvInit(); static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); static int32_t udfdRun(); +void udfdProcessRequest(uv_work_t *req) { + SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); + SUdfRequest request = {0}; + decodeUdfRequest(uvUdf->input.base, &request); + + switch (request.type) { + case UDF_TASK_SETUP: { + udfdProcessSetupRequest(uvUdf, &request); + break; + } + + case UDF_TASK_CALL: { + udfdProcessCallRequest(uvUdf, &request); + break; + } + case UDF_TASK_TEARDOWN: { + udfdProcessTeardownRequest(uvUdf, &request); + break; + } + default: { + break; + } + } +} + +void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { + // TODO: tracable id from client. connect, setup, call, teardown + fnInfo("setup request. seq num: %" PRId64 ", udf name: %s", request->seqNum, request->setup.udfName); + SUdfSetupRequest *setup = &request->setup; + int32_t code = TSDB_CODE_SUCCESS; + SUdf * udf = NULL; + uv_mutex_lock(&global.udfsMutex); + SUdf **udfInHash = taosHashGet(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName)); + if (udfInHash) { + ++(*udfInHash)->refCount; + udf = *udfInHash; + uv_mutex_unlock(&global.udfsMutex); + } else { + SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); + udfNew->refCount = 1; + udfNew->state = UDF_STATE_INIT; + + uv_mutex_init(&udfNew->lock); + uv_cond_init(&udfNew->condReady); + udf = udfNew; + taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), &udfNew, sizeof(&udfNew)); + uv_mutex_unlock(&global.udfsMutex); + } + + uv_mutex_lock(&udf->lock); + if (udf->state == UDF_STATE_INIT) { + udf->state = UDF_STATE_LOADING; + code = udfdLoadUdf(setup->udfName, udf); + if (udf->initFunc) { + udf->initFunc(); + } + udf->state = UDF_STATE_READY; + uv_cond_broadcast(&udf->condReady); + uv_mutex_unlock(&udf->lock); + } else { + while (udf->state != UDF_STATE_READY) { + uv_cond_wait(&udf->condReady, &udf->lock); + } + uv_mutex_unlock(&udf->lock); + } + SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); + handle->udf = udf; + + SUdfResponse rsp; + rsp.seqNum = request->seqNum; + rsp.type = request->type; + rsp.code = code; + rsp.setupRsp.udfHandle = (int64_t)(handle); + rsp.setupRsp.outputType = udf->outputType; + rsp.setupRsp.outputLen = udf->outputLen; + rsp.setupRsp.bufSize = udf->bufSize; + + int32_t len = encodeUdfResponse(NULL, &rsp); + rsp.msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, &rsp); + + uvUdf->output = uv_buf_init(bufBegin, len); + + taosMemoryFree(uvUdf->input.base); + return; +} + +void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { + SUdfCallRequest *call = &request->call; + fnDebug("%" PRId64 "call request. call type %d, handle: %" PRIx64, request->seqNum, call->callType, call->udfHandle); + SUdfcFuncHandle * handle = (SUdfcFuncHandle *)(call->udfHandle); + SUdf * udf = handle->udf; + SUdfResponse response = {0}; + SUdfResponse * rsp = &response; + SUdfCallResponse *subRsp = &rsp->callRsp; + + int32_t code = TSDB_CODE_SUCCESS; + switch (call->callType) { + case TSDB_UDF_CALL_SCALA_PROC: { + SUdfColumn output = {0}; + + SUdfDataBlock input = {0}; + convertDataBlockToUdfDataBlock(&call->block, &input); + code = udf->scalarProcFunc(&input, &output); + freeUdfDataDataBlock(&input); + convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); + freeUdfColumn(&output); + break; + } + case TSDB_UDF_CALL_AGG_INIT: { + SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; + udf->aggStartFunc(&outBuf); + subRsp->resultBuf = outBuf; + break; + } + case TSDB_UDF_CALL_AGG_PROC: { + SUdfDataBlock input = {0}; + convertDataBlockToUdfDataBlock(&call->block, &input); + SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; + code = udf->aggProcFunc(&input, &call->interBuf, &outBuf); + freeUdfInterBuf(&call->interBuf); + freeUdfDataDataBlock(&input); + subRsp->resultBuf = outBuf; + + break; + } + case TSDB_UDF_CALL_AGG_FIN: { + SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; + code = udf->aggFinishFunc(&call->interBuf, &outBuf); + freeUdfInterBuf(&call->interBuf); + subRsp->resultBuf = outBuf; + break; + } + default: + break; + } + + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = code; + subRsp->callType = call->callType; + + int32_t len = encodeUdfResponse(NULL, rsp); + rsp->msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, rsp); + uvUdf->output = uv_buf_init(bufBegin, len); + + switch (call->callType) { + case TSDB_UDF_CALL_SCALA_PROC: { + tDeleteSSDataBlock(&call->block); + tDeleteSSDataBlock(&subRsp->resultData); + break; + } + case TSDB_UDF_CALL_AGG_INIT: { + freeUdfInterBuf(&subRsp->resultBuf); + break; + } + case TSDB_UDF_CALL_AGG_PROC: { + tDeleteSSDataBlock(&call->block); + freeUdfInterBuf(&subRsp->resultBuf); + break; + } + case TSDB_UDF_CALL_AGG_FIN: { + freeUdfInterBuf(&subRsp->resultBuf); + break; + } + default: + break; + } + + taosMemoryFree(uvUdf->input.base); + return; +} + +void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { + SUdfTeardownRequest *teardown = &request->teardown; + fnInfo("teardown. seq number: %" PRId64 ", handle:%" PRIx64, request->seqNum, teardown->udfHandle); + SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle); + SUdf * udf = handle->udf; + bool unloadUdf = false; + int32_t code = TSDB_CODE_SUCCESS; + + uv_mutex_lock(&global.udfsMutex); + udf->refCount--; + if (udf->refCount == 0) { + unloadUdf = true; + taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); + } + uv_mutex_unlock(&global.udfsMutex); + if (unloadUdf) { + uv_cond_destroy(&udf->condReady); + uv_mutex_destroy(&udf->lock); + if (udf->destroyFunc) { + (udf->destroyFunc)(); + } + uv_dlclose(&udf->lib); + taosMemoryFree(udf); + } + taosMemoryFree(handle); + + SUdfResponse response; + SUdfResponse *rsp = &response; + rsp->seqNum = request->seqNum; + rsp->type = request->type; + rsp->code = code; + int32_t len = encodeUdfResponse(NULL, rsp); + rsp->msgLen = len; + void *bufBegin = taosMemoryMalloc(len); + void *buf = bufBegin; + encodeUdfResponse(&buf, rsp); + uvUdf->output = uv_buf_init(bufBegin, len); + + taosMemoryFree(uvUdf->input.base); + return; +} + void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->info.ahandle; ASSERT(pMsg->info.ahandle != NULL); @@ -388,226 +608,6 @@ int32_t udfdCloseClientRpc() { return 0; } -void udfdProcessRequest(uv_work_t *req) { - SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); - SUdfRequest request = {0}; - decodeUdfRequest(uvUdf->input.base, &request); - - switch (request.type) { - case UDF_TASK_SETUP: { - udfdProcessSetupRequest(uvUdf, &request); - break; - } - - case UDF_TASK_CALL: { - udfdProcessCallRequest(uvUdf, &request); - break; - } - case UDF_TASK_TEARDOWN: { - udfdProcessTeardownRequest(uvUdf, &request); - break; - } - default: { - break; - } - } -} - -void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { - // TODO: tracable id from client. connect, setup, call, teardown - fnInfo("setup request. seq num: %" PRId64 ", udf name: %s", request->seqNum, request->setup.udfName); - SUdfSetupRequest *setup = &request->setup; - int32_t code = TSDB_CODE_SUCCESS; - SUdf * udf = NULL; - uv_mutex_lock(&global.udfsMutex); - SUdf **udfInHash = taosHashGet(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName)); - if (udfInHash) { - ++(*udfInHash)->refCount; - udf = *udfInHash; - uv_mutex_unlock(&global.udfsMutex); - } else { - SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); - udfNew->refCount = 1; - udfNew->state = UDF_STATE_INIT; - - uv_mutex_init(&udfNew->lock); - uv_cond_init(&udfNew->condReady); - udf = udfNew; - taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), &udfNew, sizeof(&udfNew)); - uv_mutex_unlock(&global.udfsMutex); - } - - uv_mutex_lock(&udf->lock); - if (udf->state == UDF_STATE_INIT) { - udf->state = UDF_STATE_LOADING; - code = udfdLoadUdf(setup->udfName, udf); - if (udf->initFunc) { - udf->initFunc(); - } - udf->state = UDF_STATE_READY; - uv_cond_broadcast(&udf->condReady); - uv_mutex_unlock(&udf->lock); - } else { - while (udf->state != UDF_STATE_READY) { - uv_cond_wait(&udf->condReady, &udf->lock); - } - uv_mutex_unlock(&udf->lock); - } - SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); - handle->udf = udf; - - SUdfResponse rsp; - rsp.seqNum = request->seqNum; - rsp.type = request->type; - rsp.code = code; - rsp.setupRsp.udfHandle = (int64_t)(handle); - rsp.setupRsp.outputType = udf->outputType; - rsp.setupRsp.outputLen = udf->outputLen; - rsp.setupRsp.bufSize = udf->bufSize; - - int32_t len = encodeUdfResponse(NULL, &rsp); - rsp.msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, &rsp); - - uvUdf->output = uv_buf_init(bufBegin, len); - - taosMemoryFree(uvUdf->input.base); - return; -} - -void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { - SUdfCallRequest *call = &request->call; - fnDebug("%" PRId64 "call request. call type %d, handle: %" PRIx64, request->seqNum, call->callType, call->udfHandle); - SUdfcFuncHandle * handle = (SUdfcFuncHandle *)(call->udfHandle); - SUdf * udf = handle->udf; - SUdfResponse response = {0}; - SUdfResponse * rsp = &response; - SUdfCallResponse *subRsp = &rsp->callRsp; - - int32_t code = TSDB_CODE_SUCCESS; - switch (call->callType) { - case TSDB_UDF_CALL_SCALA_PROC: { - SUdfColumn output = {0}; - - SUdfDataBlock input = {0}; - convertDataBlockToUdfDataBlock(&call->block, &input); - code = udf->scalarProcFunc(&input, &output); - freeUdfDataDataBlock(&input); - convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); - freeUdfColumn(&output); - break; - } - case TSDB_UDF_CALL_AGG_INIT: { - SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - udf->aggStartFunc(&outBuf); - subRsp->resultBuf = outBuf; - break; - } - case TSDB_UDF_CALL_AGG_PROC: { - SUdfDataBlock input = {0}; - convertDataBlockToUdfDataBlock(&call->block, &input); - SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - code = udf->aggProcFunc(&input, &call->interBuf, &outBuf); - freeUdfInterBuf(&call->interBuf); - freeUdfDataDataBlock(&input); - subRsp->resultBuf = outBuf; - - break; - } - case TSDB_UDF_CALL_AGG_FIN: { - SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - code = udf->aggFinishFunc(&call->interBuf, &outBuf); - freeUdfInterBuf(&call->interBuf); - subRsp->resultBuf = outBuf; - break; - } - default: - break; - } - - rsp->seqNum = request->seqNum; - rsp->type = request->type; - rsp->code = code; - subRsp->callType = call->callType; - - int32_t len = encodeUdfResponse(NULL, rsp); - rsp->msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, rsp); - uvUdf->output = uv_buf_init(bufBegin, len); - - switch (call->callType) { - case TSDB_UDF_CALL_SCALA_PROC: { - tDeleteSSDataBlock(&call->block); - tDeleteSSDataBlock(&subRsp->resultData); - break; - } - case TSDB_UDF_CALL_AGG_INIT: { - freeUdfInterBuf(&subRsp->resultBuf); - break; - } - case TSDB_UDF_CALL_AGG_PROC: { - tDeleteSSDataBlock(&call->block); - freeUdfInterBuf(&subRsp->resultBuf); - break; - } - case TSDB_UDF_CALL_AGG_FIN: { - freeUdfInterBuf(&subRsp->resultBuf); - break; - } - default: - break; - } - - taosMemoryFree(uvUdf->input.base); - return; -} - -void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { - SUdfTeardownRequest *teardown = &request->teardown; - fnInfo("teardown. seq number: %" PRId64 ", handle:%" PRIx64, request->seqNum, teardown->udfHandle); - SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle); - SUdf * udf = handle->udf; - bool unloadUdf = false; - int32_t code = TSDB_CODE_SUCCESS; - - uv_mutex_lock(&global.udfsMutex); - udf->refCount--; - if (udf->refCount == 0) { - unloadUdf = true; - taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); - } - uv_mutex_unlock(&global.udfsMutex); - if (unloadUdf) { - uv_cond_destroy(&udf->condReady); - uv_mutex_destroy(&udf->lock); - if (udf->destroyFunc) { - (udf->destroyFunc)(); - } - uv_dlclose(&udf->lib); - taosMemoryFree(udf); - } - taosMemoryFree(handle); - - SUdfResponse response; - SUdfResponse *rsp = &response; - rsp->seqNum = request->seqNum; - rsp->type = request->type; - rsp->code = code; - int32_t len = encodeUdfResponse(NULL, rsp); - rsp->msgLen = len; - void *bufBegin = taosMemoryMalloc(len); - void *buf = bufBegin; - encodeUdfResponse(&buf, rsp); - uvUdf->output = uv_buf_init(bufBegin, len); - - taosMemoryFree(uvUdf->input.base); - return; -} - void udfdOnWrite(uv_write_t *req, int status) { SUvUdfWork *work = (SUvUdfWork *)req->data; if (status < 0) { From 1e8a287662c6449a46d5a8ac919e09ae89ace1e7 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 22 May 2022 16:37:06 +0800 Subject: [PATCH 61/73] feat: udf code refactoring --- source/libs/function/src/udfd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 964385e663..2d48756a1e 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -709,7 +709,7 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } if (nread < 0) { - fnDebug("Receive error %s", uv_err_name(nread)); + fnError("Receive error %s", uv_err_name(nread)); if (nread == UV_EOF) { // TODO check more when close } else { From 7e9e0596b7cc6741538bdff51ba92871dae64d31 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sun, 22 May 2022 16:41:10 +0800 Subject: [PATCH 62/73] test:modify valgrind log director --- tests/pytest/util/dnodes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 952aca70cf..9190943dfd 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -264,7 +264,7 @@ class TDDnode: cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( binPath, self.cfgDir) else: - valgrindCmdline = "valgrind --log-file=\"valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" + valgrindCmdline = "valgrind --log-file=\"%s/../log/valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"%self.cfgDir cmd = "nohup %s %s -c %s 2>&1 & " % ( valgrindCmdline, binPath, self.cfgDir) @@ -325,7 +325,7 @@ class TDDnode: cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( binPath, self.cfgDir) else: - valgrindCmdline = "valgrind --log-file=\"valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" + valgrindCmdline = "valgrind --log-file=\"%s/../log/valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"%self.cfgDir cmd = "nohup %s %s -c %s 2>&1 & " % ( valgrindCmdline, binPath, self.cfgDir) From 8a0d6d6e998fd832944925297a5f7b0c1448b053 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sun, 22 May 2022 17:16:01 +0800 Subject: [PATCH 63/73] fix(os): win not close file ctest error --- source/libs/index/src/indexTfile.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 52f8886b40..9fb6cf216e 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -141,7 +141,6 @@ void tfileCacheDestroy(TFileCache* tcache) { TFileReader* p = *reader; indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType); - tfileReaderDestroy(p); tfileReaderUnRef(p); reader = taosHashIterate(tcache->tableCache, reader); } From f4aaf791fb6e57d2630f85ccbeffc071cea5ed6e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 22 May 2022 19:37:32 +0800 Subject: [PATCH 64/73] fix: check commit and merge and add cases --- include/common/taosdef.h | 5 + include/common/tdataformat.h | 5 +- source/common/src/tdataformat.c | 4 +- source/common/src/trow.c | 7 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 8 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 12 +- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 21 +- tests/script/tsim/insert/commit-merge0.sim | 262 +++++++++++++++++++++ tests/script/tsim/insert/update0.sim | 17 ++ 9 files changed, 322 insertions(+), 19 deletions(-) create mode 100644 tests/script/tsim/insert/commit-merge0.sim diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 72d2c142d2..1d32c9825f 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -86,6 +86,11 @@ typedef enum { TSDB_RETENTION_MAX = 3 } ERetentionLevel; +typedef enum { + TSDB_BITMODE_DEFAULT = 0, // 2 bits + TSDB_BITMODE_ONE_BIT = 1, // 1 bit +} EBitmapMode; + extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index e13705d403..f1f96bfedd 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -313,8 +313,9 @@ typedef struct { SDataCol *cols; } SDataCols; -static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != 0; } -static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = 1; } +static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != TSDB_BITMODE_DEFAULT; } +static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = TSDB_BITMODE_ONE_BIT; } +static FORCE_INLINE bool tdIsBitmapModeI(int8_t bitmapMode) { return bitmapMode != TSDB_BITMODE_DEFAULT; } #define keyCol(pCols) (&((pCols)->cols[0])) // Key column #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] // the idx row of column-wised data diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index fe112ba67e..f82df0d9bc 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -855,7 +855,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { pCols->maxCols = maxCols; pCols->numOfRows = 0; pCols->numOfCols = 0; - // pCols->bitmapMode = 0; // calloc already set 0 + pCols->bitmapMode = TSDB_BITMODE_DEFAULT; if (maxCols > 0) { pCols->cols = (SDataCol *)taosMemoryCalloc(maxCols, sizeof(SDataCol)); @@ -899,7 +899,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { #endif pCols->numOfRows = 0; - pCols->bitmapMode = 0; + pCols->bitmapMode = TSDB_BITMODE_DEFAULT; pCols->numOfCols = schemaNCols(pSchema); for (i = 0; i < schemaNCols(pSchema); ++i) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 22bdd960ea..ae41e5d234 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -341,18 +341,19 @@ int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8 bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode) { int32_t nBytes = (bitmapMode == 0 ? numOfBits / TD_VTYPE_PARTS : numOfBits / TD_VTYPE_PARTS_I); uint8_t vTypeByte = tdVTypeByte[bitmapMode][TD_VTYPE_NORM]; + uint8_t *qBitmap = (uint8_t*)pBitmap; for (int i = 0; i < nBytes; ++i) { - if (*((uint8_t *)pBitmap) != vTypeByte) { + if (*qBitmap != vTypeByte) { return false; } - pBitmap = POINTER_SHIFT(pBitmap, i); + qBitmap = (uint8_t *)POINTER_SHIFT(pBitmap, i); } int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I); for (int j = 0; j < nLeft; ++j) { uint8_t vType; - tdGetBitmapValType(pBitmap, j, &vType, bitmapMode); + tdGetBitmapValType(qBitmap, j, &vType, bitmapMode); if (vType != TD_VTYPE_NORM) { return false; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 28b0ae042d..93ec6028f8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1138,6 +1138,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF memcpy(tptr, pDataCol->pData, flen); if (tBitmaps > 0) { bptr = POINTER_SHIFT(pBlockData, lsize + flen); + if (isSuper && !tdDataColsIsBitmapI(pDataCols)) { + tdMergeBitmap((uint8_t *)pDataCol->pBitmap, rowsToWrite, (uint8_t *)pDataCol->pBitmap); + } memcpy(bptr, pDataCol->pBitmap, tBitmaps); tBitmapsLen = tBitmaps; flen += tBitmapsLen; @@ -1503,13 +1506,16 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt tSkipListIterNext(pCommitIter->pIter); } else { if (lastKey != key1) { + if (lastKey != TSKEY_INITIAL_VAL) { + ++pTarget->numOfRows; + } lastKey = key1; - ++pTarget->numOfRows; } // copy disk data for (int i = 0; i < pDataCols->numOfCols; ++i) { SCellVal sVal = {0}; + // no duplicated TS keys in pDataCols from file if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) { TASSERT(0); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d3b1f56a71..8ea4016fe3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1965,7 +1965,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID && cur->pos >= 0 && cur->pos < pBlock->numOfRows); - + // Even Multi-Version supported, the records with duplicated TSKEY would be merged inside of tsdbLoadData interface. TSKEY* tsArray = pCols->cols[0].pData; assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && tsArray[pBlock->numOfRows - 1] == pBlock->keyLast); @@ -1995,6 +1995,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf int32_t pos = cur->pos; cur->win = TSWINDOW_INITIALIZER; + bool adjustPos = false; // no data in buffer, load data from file directly if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) { @@ -2016,6 +2017,13 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf break; } + if (adjustPos) { + if (key == lastKeyAppend) { + pos -= step; + } + adjustPos = false; + } + if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ascScan) || ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ascScan)) { break; @@ -2107,7 +2115,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf moveToNextRowInMem(pCheckInfo); pos += step; + adjustPos = true; } else { + // discard the memory record moveToNextRowInMem(pCheckInfo); } } else if ((key > tsArray[pos] && ascScan) || (key < tsArray[pos] && !ascScan)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index c1a1e7570e..0773805684 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -20,11 +20,11 @@ static void tsdbResetReadTable(SReadH *pReadh); static void tsdbResetReadFile(SReadH *pReadh); static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock); -static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols); +static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int8_t bitmapMode); static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp, int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize); static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds, - int numOfColIds); + int numOfColIds, int8_t bitmapMode); static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol); int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo) { @@ -266,10 +266,11 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { } } - if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0]) < 0) return -1; + + if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0], TSDB_BITMODE_ONE_BIT) < 0) return -1; for (int i = 1; i < pBlock->numOfSubBlocks; i++) { iBlock++; - if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1; + if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1], TSDB_BITMODE_DEFAULT) < 0) return -1; // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) @@ -309,10 +310,10 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, } } - if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds) < 0) return -1; + if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds, TSDB_BITMODE_ONE_BIT) < 0) return -1; for (int i = 1; i < pBlock->numOfSubBlocks; i++) { iBlock++; - if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1; + if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds, TSDB_BITMODE_DEFAULT) < 0) return -1; // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL, TD_SUPPORT_UPDATE(update), TD_VER_MAX) < 0) @@ -543,14 +544,14 @@ static void tsdbResetReadFile(SReadH *pReadh) { tsdbCloseDFileSet(TSDB_READ_FSET(pReadh)); } -static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols) { +static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int8_t bitmapMode) { ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); tdResetDataCols(pDataCols); - if (tsdbIsSupBlock(pBlock)) { + if (tdIsBitmapModeI(bitmapMode)) { tdDataColsSetBitmapI(pDataCols); } @@ -730,7 +731,7 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32 } static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds, - int numOfColIds) { + int numOfColIds, int8_t bitmapMode) { ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); ASSERT(colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID); @@ -739,7 +740,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * tdResetDataCols(pDataCols); - if (tsdbIsSupBlock(pBlock)) { + if (tdIsBitmapModeI(bitmapMode)) { tdDataColsSetBitmapI(pDataCols); } diff --git a/tests/script/tsim/insert/commit-merge0.sim b/tests/script/tsim/insert/commit-merge0.sim new file mode 100644 index 0000000000..adbd1904b2 --- /dev/null +++ b/tests/script/tsim/insert/commit-merge0.sim @@ -0,0 +1,262 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database db days 300 keep 365000d,365000d,365000d +sql show databases +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use db +sql create table stb1(ts timestamp, c6 double) tags (t1 int); +sql create table ct1 using stb1 tags ( 1 ); +sql create table ct2 using stb1 tags ( 2 ); +sql create table ct3 using stb1 tags ( 3 ); +sql create table ct4 using stb1 tags ( 4 ); +sql insert into ct1 values ('2022-05-01 18:30:27.001', 0.0); +sql insert into ct4 values ('2022-04-28 18:30:27.002', 0.0); +sql insert into ct1 values ('2022-05-01 18:30:17.003', 11.11); +sql insert into ct4 values ('2022-02-01 18:30:27.004', 11.11); +sql insert into ct1 values ('2022-05-01 18:30:07.005', 22.22); +sql insert into ct4 values ('2021-11-01 18:30:27.006', 22.22); +sql insert into ct1 values ('2022-05-01 18:29:27.007', 33.33); +sql insert into ct4 values ('2022-08-01 18:30:27.008', 33.33); +sql insert into ct1 values ('2022-05-01 18:20:27.009', 44.44); +sql insert into ct4 values ('2021-05-01 18:30:27.010', 44.44); +sql insert into ct1 values ('2022-05-01 18:21:27.011', 55.55); +sql insert into ct4 values ('2021-01-01 18:30:27.012', 55.55); +sql insert into ct1 values ('2022-05-01 18:22:27.013', 66.66); +sql insert into ct4 values ('2020-06-01 18:30:27.014', 66.66); +sql insert into ct1 values ('2022-05-01 18:28:37.015', 77.77); +sql insert into ct4 values ('2020-05-01 18:30:27.016', 77.77); +sql insert into ct1 values ('2022-05-01 18:29:17.017', 88.88); +sql insert into ct4 values ('2019-05-01 18:30:27.018', 88.88); +sql insert into ct1 values ('2022-05-01 18:30:20.019', 0); +sql insert into ct1 values ('2022-05-01 18:30:47.020', -99.99); +sql insert into ct1 values ('2022-05-01 18:30:49.021', NULL); +sql insert into ct1 values ('2022-05-01 18:30:51.022', -99.99); +sql insert into ct4 values ('2018-05-01 18:30:27.023', NULL) ; +sql insert into ct4 values ('2021-03-01 18:30:27.024', NULL) ; +sql insert into ct4 values ('2022-08-01 18:30:27.025', NULL) ; + +print =============== select * from ct1 - memory +sql select * from stb1; +if $rows != 25 then + print rows = $rows != 25 + return -1 +endi + + +print =============== stop and restart taosd + +$reboot_max = 10; + +$reboot_cnt = 0 + +reboot_and_check: + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +print =============== insert duplicated records to memory - loop $reboot_max - $reboot_cnt +sql use db +sql insert into ct1 values ('2022-05-01 18:30:27.001', 0.0); +sql insert into ct4 values ('2022-04-28 18:30:27.002', 0.0); +sql insert into ct1 values ('2022-05-01 18:30:17.003', 11.11); +sql insert into ct4 values ('2022-02-01 18:30:27.004', 11.11); +sql insert into ct1 values ('2022-05-01 18:30:07.005', 22.22); +sql insert into ct4 values ('2021-11-01 18:30:27.006', 22.22); +sql insert into ct1 values ('2022-05-01 18:29:27.007', 33.33); +sql insert into ct4 values ('2022-08-01 18:30:27.008', 33.33); +sql insert into ct1 values ('2022-05-01 18:20:27.009', 44.44); +sql insert into ct4 values ('2021-05-01 18:30:27.010', 44.44); +sql insert into ct1 values ('2022-05-01 18:21:27.011', 55.55); +sql insert into ct4 values ('2021-01-01 18:30:27.012', 55.55); +sql insert into ct1 values ('2022-05-01 18:22:27.013', 66.66); +sql insert into ct4 values ('2020-06-01 18:30:27.014', 66.66); +sql insert into ct1 values ('2022-05-01 18:28:37.015', 77.77); +sql insert into ct4 values ('2020-05-01 18:30:27.016', 77.77); +sql insert into ct1 values ('2022-05-01 18:29:17.017', 88.88); +sql insert into ct4 values ('2019-05-01 18:30:27.018', 88.88); +sql insert into ct1 values ('2022-05-01 18:30:20.019', 0); +sql insert into ct1 values ('2022-05-01 18:30:47.020', -99.99); +sql insert into ct1 values ('2022-05-01 18:30:49.021', NULL); +sql insert into ct1 values ('2022-05-01 18:30:51.022', -99.99); +sql insert into ct4 values ('2018-05-01 18:30:27.023', NULL) ; +sql insert into ct4 values ('2021-03-01 18:30:27.024', NULL) ; +sql insert into ct4 values ('2022-08-01 18:30:27.025', NULL) ; + +print =============== select * from ct1 - merge memory and file - loop $reboot_max - $reboot_cnt +sql select * from ct1; +if $rows != 13 then + print rows = $rows != 13 + return -1 +endi +print $data[0][0] $data[0][1] +print $data[1][0] $data[1][1] +print $data[2][0] $data[2][1] +print $data[3][0] $data[3][1] +print $data[4][0] $data[4][1] +print $data[5][0] $data[5][1] +print $data[6][0] $data[6][1] +print $data[7][0] $data[7][1] +print $data[8][0] $data[8][1] +print $data[9][0] $data[9][1] +print $data[10][0] $data[10][1] +print $data[11][0] $data[11][1] +print $data[12][0] $data[12][1] + +if $data[0][1] != 44.440000000 then + print $data[0][1] != 44.440000000 + return -1 +endi +if $data[1][1] != 55.550000000 then + print $data[1][1] != 55.550000000 + return -1 +endi +if $data[2][1] != 66.660000000 then + print $data[2][1] != 66.660000000 + return -1 +endi +if $data[3][1] != 77.770000000 then + print $data[3][1] != 77.770000000 + return -1 +endi +if $data[4][1] != 88.880000000 then + print $data[4][1] != 88.880000000 + return -1 +endi +if $data[5][1] != 33.330000000 then + print $data[5][1] != 33.330000000 + return -1 +endi +if $data[6][1] != 22.220000000 then + print $data[6][1] != 22.220000000 + return -1 +endi +if $data[7][1] != 11.110000000 then + print $data[7][1] != 11.110000000 + return -1 +endi +if $data[8][1] != 0.000000000 then + print $data[8][1] != 0.000000000 + return -1 +endi +if $data[9][1] != 0.000000000 then + print $data[9][1] != 0.000000000 + return -1 +endi +if $data[10][1] != -99.990000000 then + print $data[10][1] != -99.990000000 + return -1 +endi +if $data[11][1] != NULL then + print $data[11][1] != NULL + return -1 +endi +if $data[12][1] != -99.990000000 then + print $data[12][1] != -99.990000000 + return -1 +endi + +print =============== select * from ct4 - merge memory and file - loop $reboot_max - $reboot_cnt +sql select * from ct4; +if $rows != 12 then + print rows = $rows != 12 + return -1 +endi + +print $data[0][0] $data[0][1] +print $data[1][0] $data[1][1] +print $data[2][0] $data[2][1] +print $data[3][0] $data[3][1] +print $data[4][0] $data[4][1] +print $data[5][0] $data[5][1] +print $data[6][0] $data[6][1] +print $data[7][0] $data[7][1] +print $data[8][0] $data[8][1] +print $data[9][0] $data[9][1] +print $data[10][0] $data[10][1] +print $data[11][0] $data[11][1] + +if $data[0][1] != NULL then + print $data[0][1] != NULL + return -1 +endi +if $data[1][1] != 88.880000000 then + print $data[1][1] != 88.880000000 + return -1 +endi +if $data[2][1] != 77.770000000 then + print $data[2][1] != 77.770000000 + return -1 +endi +if $data[3][1] != 66.660000000 then + print $data[3][1] != 66.660000000 + return -1 +endi +if $data[4][1] != 55.550000000 then + print $data[4][1] != 55.550000000 + return -1 +endi +if $data[5][1] != NULL then + print $data[5][1] != NULL + return -1 +endi +if $data[6][1] != 44.440000000 then + print $data[6][1] != 44.440000000 + return -1 +endi +if $data[7][1] != 22.220000000 then + print $data[7][1] != 22.220000000 + return -1 +endi +if $data[8][1] != 11.110000000 then + print $data[8][1] != 11.110000000 + return -1 +endi +if $data[9][1] != 0.000000000 then + print $data[9][1] != 0.000000000 + return -1 +endi +if $data[10][1] != 33.330000000 then + print $data[10][1] != 33.330000000 + return -1 +endi +if $data[11][1] != NULL then + print $data[11][1] != NULL + return -1 +endi + + +if $reboot_cnt > $reboot_max then + print reboot_cnt $reboot_cnt > reboot_max $reboot_max + return 0 +else + print reboot_cnt $reboot_cnt <= reboot_max $reboot_max + $reboot_cnt = $reboot_cnt + 1 + goto reboot_and_check +endi diff --git a/tests/script/tsim/insert/update0.sim b/tests/script/tsim/insert/update0.sim index 89eecaf860..3cb5e4008e 100644 --- a/tests/script/tsim/insert/update0.sim +++ b/tests/script/tsim/insert/update0.sim @@ -99,6 +99,23 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + print =============== step3-2 query records of ct1 from file sql select * from ct1; print $data00 $data01 From a146f5249f4a4284c7605c4a632ce0bf7a5d3358 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sun, 22 May 2022 19:41:33 +0800 Subject: [PATCH 65/73] fix double free issue --- source/client/src/clientHb.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b068e13b7d..d01ec501ba 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -310,6 +310,8 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { taosArrayDestroy(desc.subDesc); desc.subDesc = NULL; } + } else { + desc.subDesc = NULL; } releaseRequest(*rid); From 5a0e7dfa74baa6bab50b8a1049926faafe82c240 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 22 May 2022 19:58:42 +0800 Subject: [PATCH 66/73] enh: refator index/transport code --- cmake/cmake.options | 2 +- source/libs/index/inc/indexCache.h | 2 ++ source/libs/index/inc/indexInt.h | 1 + source/libs/index/src/index.c | 38 ++++++++++++++++------------ source/libs/index/src/indexCache.c | 29 ++++++++++++++++----- source/libs/index/test/indexTests.cc | 28 +++++++++++++++++++- 6 files changed, 75 insertions(+), 25 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index d1feb6516a..cb6fd1400d 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -146,6 +146,6 @@ option( option( BUILD_WITH_INVERTEDINDEX "If use invertedIndex" - ON + OFF ) diff --git a/source/libs/index/inc/indexCache.h b/source/libs/index/inc/indexCache.h index 96bfb2cd7f..aff2e0e836 100644 --- a/source/libs/index/inc/indexCache.h +++ b/source/libs/index/inc/indexCache.h @@ -65,6 +65,8 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in void indexCacheForceToMerge(void* cache); void indexCacheDestroy(void* cache); +void indexCacheBroadcast(void* cache); +void indexCacheWait(void* cache); Iterate* indexCacheIteratorCreate(IndexCache* cache); void indexCacheIteratorDestroy(Iterate* iiter); diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 5cb60abe7d..4bce792dd9 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -58,6 +58,7 @@ struct SIndex { SIndexStat stat; TdThreadMutex mtx; + bool quit; }; struct SIndexOpts { diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index d1f50f1979..add59cb0b3 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -124,21 +124,6 @@ END: void indexDestroy(void* handle) { SIndex* sIdx = handle; - // indexAcquireRef(sIdx->refId); - if (sIdx->colObj != NULL) { - void* iter = taosHashIterate(sIdx->colObj, NULL); - while (iter) { - IndexCache** pCache = iter; - indexCacheForceToMerge((void*)(*pCache)); - iter = taosHashIterate(sIdx->colObj, iter); - - indexCacheUnRef(*pCache); - } - taosHashCleanup(sIdx->colObj); - sIdx->colObj = NULL; - return; - } // indexReleaseRef(sIdx->refId); - taosThreadMutexDestroy(&sIdx->mtx); indexTFileDestroy(sIdx->tindex); taosMemoryFree(sIdx->path); @@ -147,6 +132,20 @@ void indexDestroy(void* handle) { } void indexClose(SIndex* sIdx) { indexReleaseRef(sIdx->refId); + bool ref = 0; + if (sIdx->colObj != NULL) { + void* iter = taosHashIterate(sIdx->colObj, NULL); + while (iter) { + IndexCache** pCache = iter; + indexCacheForceToMerge((void*)(*pCache)); + indexCacheWait((void*)(*pCache)); + iter = taosHashIterate(sIdx->colObj, iter); + indexCacheUnRef(*pCache); + } + taosHashCleanup(sIdx->colObj); + sIdx->colObj = NULL; + } + // taosMsleep(1000 * 5); indexRemoveRef(sIdx->refId); } int64_t indexAddRef(void* p) { @@ -183,6 +182,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*)); } } + taosThreadMutexUnlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { SIndexTerm* p = taosArrayGetP(fVals, i); @@ -198,7 +198,6 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { return ret; } } - taosThreadMutexUnlock(&index->mtx); return 0; } int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { @@ -461,6 +460,9 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { indexError("%p immtable is empty, ignore merge opera", pCache); indexCacheDestroyImm(pCache); tfileReaderUnRef(pReader); + if (sIdx->quit) { + indexCacheBroadcast(pCache); + } indexReleaseRef(sIdx->refId); return 0; } @@ -507,6 +509,9 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { indexDestroyFinalResult(result); indexCacheDestroyImm(pCache); + if (sIdx->quit) { + indexCacheBroadcast(pCache); + } indexCacheIteratorDestroy(cacheIter); tfileIteratorDestroy(tfileIter); @@ -521,6 +526,7 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000); } indexReleaseRef(sIdx->refId); + return ret; } void iterateValueDestroy(IterateValue* value, bool destroy) { diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index ef01094b5d..232eca9304 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -23,6 +23,7 @@ #define MEM_TERM_LIMIT 10 * 10000 #define MEM_THRESHOLD 64 * 1024 +#define MEM_SIGNAL_QUIT MEM_THRESHOLD * 20 #define MEM_ESTIMATE_RADIO 1.5 static void indexMemRef(MemTable* tbl); @@ -396,17 +397,24 @@ void indexCacheDestroySkiplist(SSkipList* slt) { tSkipListDestroyIter(iter); tSkipListDestroy(slt); } +void indexCacheBroadcast(void* cache) { + IndexCache* pCache = cache; + taosThreadCondBroadcast(&pCache->finished); +} +void indexCacheWait(void* cache) { + IndexCache* pCache = cache; + taosThreadCondWait(&pCache->finished, &pCache->mtx); +} void indexCacheDestroyImm(IndexCache* cache) { if (cache == NULL) { return; } - MemTable* tbl = NULL; taosThreadMutexLock(&cache->mtx); tbl = cache->imm; cache->imm = NULL; // or throw int bg thread - taosThreadCondBroadcast(&cache->finished); + indexCacheBroadcast(cache); taosThreadMutexUnlock(&cache->mtx); @@ -460,11 +468,13 @@ void indexCacheIteratorDestroy(Iterate* iter) { taosMemoryFree(iter); } -int indexCacheSchedToMerge(IndexCache* pCache) { +int indexCacheSchedToMerge(IndexCache* pCache, bool notify) { SSchedMsg schedMsg = {0}; schedMsg.fp = doMergeWork; schedMsg.ahandle = pCache; - schedMsg.thandle = NULL; + if (notify) { + schedMsg.thandle = taosMemoryMalloc(1); + } schedMsg.msg = NULL; indexAcquireRef(pCache->index->refId); taosScheduleTask(indexQhandle, &schedMsg); @@ -477,8 +487,10 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable - taosThreadCondWait(&cache->finished, &cache->mtx); + indexCacheWait(cache); } else { + bool notifyQuit = cache->occupiedMem >= MEM_SIGNAL_QUIT ? true : false; + indexCacheRef(cache); cache->imm = cache->mem; cache->mem = indexInternalCacheCreate(cache->type); @@ -486,7 +498,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { cache->occupiedMem = 0; // sched to merge // unref cache in bgwork - indexCacheSchedToMerge(cache); + indexCacheSchedToMerge(cache, notifyQuit); } } } @@ -538,7 +550,7 @@ void indexCacheForceToMerge(void* cache) { taosThreadMutexLock(&pCache->mtx); indexInfo("%p is forced to merge into tfile", pCache); - pCache->occupiedMem += MEM_THRESHOLD * 5; + pCache->occupiedMem += MEM_SIGNAL_QUIT; indexCacheMakeRoomForWrite(pCache); taosThreadMutexUnlock(&pCache->mtx); @@ -703,6 +715,9 @@ static MemTable* indexInternalCacheCreate(int8_t type) { static void doMergeWork(SSchedMsg* msg) { IndexCache* pCache = msg->ahandle; SIndex* sidx = (SIndex*)pCache->index; + + sidx->quit = msg->thandle ? true : false; + taosMemoryFree(msg->thandle); indexFlushCacheToTFile(sidx, pCache); } static bool indexCacheIteratorNext(Iterate* itera) { diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 262bbfe55a..222bf51c5f 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -834,7 +834,10 @@ class IndexObj { class IndexEnv2 : public ::testing::Test { protected: - virtual void SetUp() { index = new IndexObj(); } + virtual void SetUp() { + initLog(); + index = new IndexObj(); + } virtual void TearDown() { delete index; } IndexObj* index; }; @@ -906,6 +909,29 @@ TEST_F(IndexEnv2, testIndexOpen) { indexMultiTermQueryDestroy(mq); } } +TEST_F(IndexEnv2, testEmptyIndexOpen) { + std::string path = "/tmp/test"; + if (index->Init(path) != 0) { + std::cout << "failed to init index" << std::endl; + exit(1); + } + + int targetSize = 1; + { + std::string colName("tag1"), colVal("Hello"); + + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < targetSize; i++) { + int tableId = i; + int ret = index->Put(terms, tableId); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + } +} TEST_F(IndexEnv2, testIndex_TrigeFlush) { std::string path = "/tmp/testxxx"; From 30671eaadf73624e3aaa50c8850c8f9aa834771e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sun, 22 May 2022 20:35:57 +0800 Subject: [PATCH 67/73] fix(tmq): skip dropped table for db subscription --- include/util/taoserror.h | 1 + source/dnode/vnode/src/tq/tq.c | 1 + source/dnode/vnode/src/tq/tqRead.c | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 1d7287ed0e..28ed637977 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -419,6 +419,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) #define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) #define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) +#define TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0A0B) // wal #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0671044bad..25fa716d4e 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -600,6 +600,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SSDataBlock block = {0}; if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, &block.info.numOfCols) < 0) { + if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(&block); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 918660a9ec..2559631e05 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -91,7 +91,9 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p if (pHandle->sver != sversion || pHandle->cachedSchemaUid != pHandle->msgIter.suid) { pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); if (pHandle->pSchema == NULL) { - tqError("cannot found schema for table: %ld, version %d", pHandle->msgIter.suid, pHandle->sver); + tqWarn("cannot found schema for table: %ld, version %d, possibly dropped table", pHandle->msgIter.suid, + pHandle->sver); + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; } From 452b154eb931ed1e9ac78984a2f634bd318ae4f8 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sun, 22 May 2022 21:20:36 +0800 Subject: [PATCH 68/73] enh(tmq): add error log --- source/dnode/vnode/src/tq/tqRead.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 2559631e05..be8d786de2 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -91,14 +91,22 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p if (pHandle->sver != sversion || pHandle->cachedSchemaUid != pHandle->msgIter.suid) { pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion); if (pHandle->pSchema == NULL) { - tqWarn("cannot found schema for table: %ld, version %d, possibly dropped table", pHandle->msgIter.suid, - pHandle->sver); + tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table", + pHandle->msgIter.uid, pHandle->msgIter.suid, pHandle->sver); + /*ASSERT(0);*/ terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; } // this interface use suid instead of uid pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.suid, sversion, true); + if (pHandle->pSchemaWrapper == NULL) { + tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table", + pHandle->msgIter.suid, pHandle->sver); + /*ASSERT(0);*/ + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } pHandle->sver = sversion; pHandle->cachedSchemaUid = pHandle->msgIter.suid; } From 3570705ebdb40e3ceff6e651c36d1fa1a5ee0b32 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 22 May 2022 22:12:10 +0800 Subject: [PATCH 69/73] enh: add merge case to ci and code optimization --- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 ++--- tests/script/jenkins/basic.txt | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 8ea4016fe3..47896272b5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1732,6 +1732,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } + *lastRowKey = rowKey; ++nResult; } else if (update) { mergeOption = 2; @@ -1739,8 +1740,6 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa mergeOption = 0; break; } - - *lastRowKey = rowKey; } } else { // TODO: use STSRowIter @@ -1753,6 +1752,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (*lastRowKey != TSKEY_INITIAL_VAL) { ++(*curRow); } + *lastRowKey = rowKey; ++nResult; } else if (update) { mergeOption = 2; @@ -1760,7 +1760,6 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa mergeOption = 0; break; } - *lastRowKey = rowKey; } else { SKvRowIdx* pColIdx = tdKvRowColIdxAt(row, chosen_itr - 1); colId = pColIdx->colId; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index bb96e3642b..12be7c5c46 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -29,6 +29,7 @@ ./test.sh -f tsim/insert/backquote.sim ./test.sh -f tsim/insert/null.sim ./test.sh -f tsim/insert/update0.sim +./test.sh -f tsim/insert/commit-merge0.sim # ---- parser ./test.sh -f tsim/parser/groupby-basic.sim From b181232098f2a553ee011dc0dfdae3365e40a0f4 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 22 May 2022 22:30:52 +0800 Subject: [PATCH 70/73] trigger CI From bad686568d5ed2d9cf8dc6ad22a4f6126113b3dc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 22 May 2022 23:19:35 +0800 Subject: [PATCH 71/73] fix: avoid invalid read/write --- source/libs/index/CMakeLists.txt | 6 +++--- source/libs/index/inc/indexInt.h | 2 ++ source/libs/index/src/index.c | 23 ++++++++++++++++++----- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 7dc66e4789..d5fd574aad 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -31,7 +31,7 @@ if (${BUILD_WITH_INVERTEDINDEX}) endif(${BUILD_WITH_INVERTEDINDEX}) -if (${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) +#if (${BUILD_TEST}) +# add_subdirectory(test) +#endif(${BUILD_TEST}) diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 4bce792dd9..0bdcb131b6 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -58,6 +58,7 @@ struct SIndex { SIndexStat stat; TdThreadMutex mtx; + tsem_t sem; bool quit; }; @@ -70,6 +71,7 @@ struct SIndexOpts { int32_t cacheSize; // MB // add cache module later #endif + int32_t cacheOpt; // MB }; struct SIndexMultiTermQuery { diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index add59cb0b3..2141e90bbd 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -90,6 +90,15 @@ static void indexMergeCacheAndTFile(SArray* result, IterateValue* icache, Iterat // static int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); // int32_t indexSerialKey(ICacheKey* key, char* buf); +static void indexPost(void* idx) { + SIndex* pIdx = idx; + tsem_post(&pIdx->sem); +} +static void indexWait(void* idx) { + SIndex* pIdx = idx; + tsem_wait(&pIdx->sem); +} + int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { taosThreadOnce(&isInit, indexInit); SIndex* sIdx = taosMemoryCalloc(1, sizeof(SIndex)); @@ -107,6 +116,8 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { sIdx->cVersion = 1; sIdx->path = tstrdup(path); taosThreadMutexInit(&sIdx->mtx, NULL); + tsem_init(&sIdx->sem, 0, 0); + // taosThreadCondInit(&sIdx->finished, NULL); sIdx->refId = indexAddRef(sIdx); indexAcquireRef(sIdx->refId); @@ -125,6 +136,7 @@ END: void indexDestroy(void* handle) { SIndex* sIdx = handle; taosThreadMutexDestroy(&sIdx->mtx); + tsem_destroy(&sIdx->sem); indexTFileDestroy(sIdx->tindex); taosMemoryFree(sIdx->path); taosMemoryFree(sIdx); @@ -138,7 +150,7 @@ void indexClose(SIndex* sIdx) { while (iter) { IndexCache** pCache = iter; indexCacheForceToMerge((void*)(*pCache)); - indexCacheWait((void*)(*pCache)); + indexWait((void*)(sIdx)); iter = taosHashIterate(sIdx->colObj, iter); indexCacheUnRef(*pCache); } @@ -461,7 +473,8 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { indexCacheDestroyImm(pCache); tfileReaderUnRef(pReader); if (sIdx->quit) { - indexCacheBroadcast(pCache); + indexPost(sIdx); + // indexCacheBroadcast(pCache); } indexReleaseRef(sIdx->refId); return 0; @@ -509,9 +522,6 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { indexDestroyFinalResult(result); indexCacheDestroyImm(pCache); - if (sIdx->quit) { - indexCacheBroadcast(pCache); - } indexCacheIteratorDestroy(cacheIter); tfileIteratorDestroy(tfileIter); @@ -525,6 +535,9 @@ int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { } else { indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000); } + if (sIdx->quit) { + indexPost(sIdx); + } indexReleaseRef(sIdx->refId); return ret; From 38df8505c385b1f1531c7026dd12749c76684c13 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 23 May 2022 01:57:29 +0800 Subject: [PATCH 72/73] test(stream): change test name --- tests/script/jenkins/basic.txt | 6 +++--- tests/script/tsim/{tstream => stream}/basic0.sim | 0 tests/script/tsim/{tstream => stream}/basic1.sim | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename tests/script/tsim/{tstream => stream}/basic0.sim (100%) rename tests/script/tsim/{tstream => stream}/basic1.sim (100%) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 18fe3b9afe..c669d5a460 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -61,9 +61,9 @@ # ---- table ./test.sh -f tsim/table/basic1.sim -# ---- tstream -./test.sh -f tsim/tstream/basic0.sim -./test.sh -f tsim/tstream/basic1.sim +# ---- stream +./test.sh -f tsim/stream/basic0.sim +./test.sh -f tsim/stream/basic1.sim # ---- transaction ./test.sh -f tsim/trans/create_db.sim diff --git a/tests/script/tsim/tstream/basic0.sim b/tests/script/tsim/stream/basic0.sim similarity index 100% rename from tests/script/tsim/tstream/basic0.sim rename to tests/script/tsim/stream/basic0.sim diff --git a/tests/script/tsim/tstream/basic1.sim b/tests/script/tsim/stream/basic1.sim similarity index 100% rename from tests/script/tsim/tstream/basic1.sim rename to tests/script/tsim/stream/basic1.sim From 4dda27ba83545b62e4603996b41715a4137fa846 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 23 May 2022 08:16:34 +0800 Subject: [PATCH 73/73] test(stream): add case --- tests/script/tsim/stream/basic2.sim | 112 ++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 tests/script/tsim/stream/basic2.sim diff --git a/tests/script/tsim/stream/basic2.sim b/tests/script/tsim/stream/basic2.sim new file mode 100644 index 0000000000..247d8f62ee --- /dev/null +++ b/tests/script/tsim/stream/basic2.sim @@ -0,0 +1,112 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database d0 vgroups 1 +sql show databases +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use d0 + +print =============== create super table, include column type for count/sum/min/max/first +sql create table if not exists stb (ts timestamp, k int) tags (a int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags(1000) +sql create table ct2 using stb tags(2000) +sql create table ct3 using stb tags(3000) + +sql show tables +if $rows != 3 then + return -1 +endi + +sql create stream s1 trigger at_once into outstb as select _wstartts, min(k), max(k), sum(k) as sum_alias from ct1 interval(10m) + +sql show stables +if $rows != 2 then + return -1 +endi + +print =============== insert data + +sql insert into ct1 values('2022-05-08 03:42:00.000', 234) +sleep 100 + +#=================================================================== +print =============== query data from child table + +sql select `_wstartts`,`min(k)`,`max(k)`,sum_alias from outstb +print rows: $rows +print $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi + +if $data01 != 234 then + return -1 +endi + +if $data02 != 234 then + return -1 +endi + +if $data03 != 234 then + return -1 +endi + +#=================================================================== +print =============== insert data + +sql insert into ct1 values('2022-05-08 03:57:00.000', -111) +sleep 100 + + +#=================================================================== +print =============== query data from child table + +sql select `_wstartts`,`min(k)`,`max(k)`,sum_alias from outstb +print rows: $rows +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +if $rows != 2 then + return -1 +endi + +if $data01 != 234 then + return -1 +endi + +if $data02 != 234 then + return -1 +endi + +if $data03 != 234 then + return -1 +endi + +if $data11 != -111 then + return -1 +endi + +if $data12 != -111 then + return -1 +endi + +if $data13 != -111 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT