From c8e10d15a5956dd4883d0296f964478b8df26bff Mon Sep 17 00:00:00 2001 From: zhipz Date: Mon, 8 Jul 2024 09:30:07 +0800 Subject: [PATCH 01/37] Adding test case for TS-5130 --- tests/parallel_test/cases.task | 1 + tests/system-test/99-TDcase/TS-5130.py | 51 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 tests/system-test/99-TDcase/TS-5130.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index aff5bedaf8..a1b90fdd1e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -291,6 +291,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3581.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3311.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3821.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-5130.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py diff --git a/tests/system-test/99-TDcase/TS-5130.py b/tests/system-test/99-TDcase/TS-5130.py new file mode 100644 index 0000000000..a5a9d373a1 --- /dev/null +++ b/tests/system-test/99-TDcase/TS-5130.py @@ -0,0 +1,51 @@ +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +import taos + + + +class TDTestCase: + def init(self, conn, logSQl, replicaVal=1): + self.replicaVar = int(replicaVal) + tdLog.debug(f"start to excute {__file__}") + self.conn = conn + tdSql.init(conn.cursor(), False) + self.passwd = {'root':'taosdata', + 'test':'test'} + def prepare_user(self): + tdSql.execute(f"create user test pass 'test' sysinfo 1") + + def test_connect_user(self, uname): + try: + for db in ['information_schema', 'performance_schema']: + new_conn = taos.connect(host = self.conn._host, config = self.conn._config, + database =db, user=uname, password=self.passwd[uname]) + cursor = new_conn.cursor() + cursor.execute(f'show databases') + result = cursor.fetchall() + dbname = [i for i in result] + assert result == [('information_schema',), ('performance_schema',)] + tdLog.success(f"Test User {uname} for {db} .......[OK]") + new_conn.close() + + except: + tdLog.debug(f"{__file__} Failed!") + exit(-1) + + def run(self): + self.prepare_user() + self.test_connect_user('root') + self.test_connect_user('test') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successful executed") + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + + + + From 466d39fa028c90929eabc785f3dafc3936b2051b Mon Sep 17 00:00:00 2001 From: zhipz Date: Tue, 9 Jul 2024 11:17:31 +0800 Subject: [PATCH 02/37] Adding database parameter in common.py:new_con, common.py:new_cur. Changing some content of TS-5130.py --- tests/pytest/util/common.py | 146 +++---------------------- tests/system-test/99-TDcase/TS-5130.py | 15 +-- 2 files changed, 19 insertions(+), 142 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 7bb2f42495..412ce22545 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -139,7 +139,7 @@ class TDCom: self.stream_suffix = "_stream" self.range_count = 5 self.default_interval = 5 - self.stream_timeout = 60 + self.stream_timeout = 12 self.create_stream_sleep = 0.5 self.record_history_ts = str() self.precision = "ms" @@ -201,9 +201,6 @@ class TDCom: self.cast_tag_stb_filter_des_select_elm = "ts, t1, t2, t3, t4, cast(t1 as TINYINT UNSIGNED), t6, t7, t8, t9, t10, cast(t2 as varchar(256)), t12, cast(t3 as bool)" self.tag_count = len(self.tag_filter_des_select_elm.split(",")) self.state_window_range = list() - - self.custom_col_val = 0 - self.part_val_list = [1, 2] # def init(self, conn, logSql): # # tdSql.init(conn.cursor(), logSql) @@ -509,22 +506,7 @@ class TDCom: if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] break - if platform.system().lower() == 'windows': - win_sep = "\\" - buildPath = buildPath.replace(win_sep,'/') - return buildPath - - def getTaosdPath(self, dnodeID="dnode1"): - buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - taosdPath = buildPath + "/../sim/" + dnodeID - tdLog.info("taosdPath: %s" % taosdPath) - return taosdPath - def getClientCfgPath(self): buildPath = self.getBuildPath() @@ -542,16 +524,16 @@ class TDCom: # print(con) return con - def newcur(self,host='localhost',port=6030,user='root',password='taosdata'): + def newcur(self,host='localhost',port=6030,user='root',password='taosdata', database=None): cfgPath = self.getClientCfgPath() - con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port) + con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port, database=None) cur=con.cursor() # print(cur) return cur - def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata'): + def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata', database=None): newTdSql = TDSql() - cur = self.newcur(host=host,port=port,user=user,password=password) + cur = self.newcur(host=host,port=port,user=user,password=password, database=None) newTdSql.init(cur, False) return newTdSql @@ -1262,7 +1244,7 @@ class TDCom: default_ctbname_index_start_num += 1 tdSql.execute(create_stable_sql) - def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None, additional_ts=None, custom_col_index=None, col_value_type=None, force_pk_val=None): + def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None): """_summary_ Args: @@ -1272,8 +1254,6 @@ class TDCom: """ self.column_value_list = list() self.ts_value = self.genTs()[0] - if additional_ts is not None: - self.additional_ts = self.genTs(additional_ts=additional_ts)[2] if ts_value is not None: self.ts_value = ts_value @@ -1297,22 +1277,7 @@ class TDCom: for i in range(int(len(self.column_value_list)/2)): index_num = random.randint(0, len(self.column_value_list)-1) self.column_value_list[index_num] = None - - if custom_col_index is not None: - if col_value_type == "Random": - pass - elif col_value_type == "Incremental": - self.column_value_list[custom_col_index] = self.custom_col_val - self.custom_col_val += 1 - elif col_value_type == "Part_equal": - self.column_value_list[custom_col_index] = random.choice(self.part_val_list) - - self.column_value_list = [self.ts_value] + [self.additional_ts] + self.column_value_list if additional_ts is not None else [self.ts_value] + self.column_value_list - if col_value_type == "Incremental" and custom_col_index==1: - self.column_value_list[custom_col_index] = self.custom_col_val if force_pk_val is None else force_pk_val - if col_value_type == "Part_equal" and custom_col_index==1: - self.column_value_list[custom_col_index] = random.randint(0, self.custom_col_val) if force_pk_val is None else force_pk_val - + self.column_value_list = [self.ts_value] + self.column_value_list def screate_table(self, dbname=None, tbname="tb", use_name="table", column_elm_list=None, count=1, default_tbname_prefix="tb", default_tbname_index_start_num=1, @@ -1353,7 +1318,7 @@ class TDCom: default_tbname_index_start_num += 1 tdSql.execute(create_table_sql) - def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False, custom_col_index=None, col_value_type="random"): + def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False): """insert rows Args: @@ -1373,7 +1338,7 @@ class TDCom: if tbname is not None: self.tbname = tbname - self.sgen_column_value_list(column_ele_list, need_null, ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) + self.sgen_column_value_list(column_ele_list, need_null, ts_value) # column_value_str = ", ".join(str(v) for v in self.column_value_list) column_value_str = "" for column_value in self.column_value_list: @@ -1390,7 +1355,7 @@ class TDCom: else: for num in range(count): ts_value = self.genTs()[0] - self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s', custom_col_index=custom_col_index, col_value_type=col_value_type) + self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s') column_value_str = "" for column_value in self.column_value_list: if column_value is None: @@ -1708,8 +1673,8 @@ class TDCom: res1 = self.round_handle(res1) res2 = self.round_handle(res2) if latency < self.stream_timeout: - latency += 0.5 - time.sleep(0.5) + latency += 0.2 + time.sleep(0.2) else: if latency == 0: return False @@ -1797,7 +1762,7 @@ class TDCom: self.sdelete_rows(tbname=self.ctb_name, start_ts=self.time_cast(self.record_history_ts, "-")) self.sdelete_rows(tbname=self.tb_name, start_ts=self.time_cast(self.record_history_ts, "-")) - def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None, custom_col_index=None, col_value_type="random"): + def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None): """prepare stream data Args: @@ -1860,89 +1825,11 @@ class TDCom: if fill_history_value == 1: for i in range(self.range_count): ts_value = str(self.date_time)+f'-{self.default_interval*(i+1)}s' - self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) - self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) + self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value) + self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value) if i == 1: self.record_history_ts = ts_value - def get_subtable(self, tbname_pre): - tdSql.query(f'show tables') - tbname_list = list(map(lambda x:x[0], tdSql.queryResult)) - for tbname in tbname_list: - if tbname_pre in tbname: - return tbname - - def get_subtable_wait(self, tbname_pre): - tbname = self.get_subtable(tbname_pre) - latency = 0 - while tbname is None: - tbname = self.get_subtable(tbname_pre) - if latency < self.stream_timeout: - latency += 1 - time.sleep(1) - return tbname - - def get_group_id_from_stb(self, stbname): - tdSql.query(f'select distinct group_id from {stbname}') - cnt = 0 - while len(tdSql.queryResult) == 0: - tdSql.query(f'select distinct group_id from {stbname}') - if cnt < self.default_interval: - cnt += 1 - time.sleep(1) - else: - return False - return tdSql.queryResult[0][0] - - def update_json_file_replica(self, json_file_path, new_replica_value, output_file_path=None): - """ - Read a JSON file, update the 'replica' value, and write the result back to a file. - - Parameters: - json_file_path (str): The path to the original JSON file. - new_replica_value (int): The new 'replica' value to be set. - output_file_path (str, optional): The path to the output file where the updated JSON will be saved. - If not provided, the original file will be overwritten. - - Returns: - None - """ - try: - # Read the JSON file and load its content into a Python dictionary - with open(json_file_path, 'r', encoding='utf-8') as file: - data = json.load(file) - - # Iterate over each item in the 'databases' list to find 'dbinfo' and update 'replica' - for db in data['databases']: - if 'dbinfo' in db: - db['dbinfo']['replica'] = new_replica_value - - # Convert the updated dictionary back into a JSON string with indentation for readability - updated_json_str = json.dumps(data, indent=4, ensure_ascii=False) - - # Write the updated JSON string to a file - if output_file_path: - # If an output file path is provided, write to the new file - with open(output_file_path, 'w', encoding='utf-8') as output_file: - output_file.write(updated_json_str) - else: - # Otherwise, overwrite the original file with the updated content - with open(json_file_path, 'w', encoding='utf-8') as file: - file.write(updated_json_str) - - except json.JSONDecodeError as e: - # Handle JSON decoding error (e.g., if the file is not valid JSON) - print(f"JSON decode error: {e}") - except FileNotFoundError: - # Handle the case where the JSON file is not found at the given path - print(f"File not found: {json_file_path}") - except KeyError as e: - # Handle missing key error (e.g., if 'databases' or 'dbinfo' is not present) - print(f"Key error: {e}") - except Exception as e: - # Handle any other exceptions that may occur - print(f"An error occurred: {e}") - def is_json(msg): if isinstance(msg, str): try: @@ -1977,7 +1864,4 @@ def dict2toml(in_dict: dict, file:str): with open(file, 'w') as f: toml.dump(in_dict, f) - - - tdCom = TDCom() diff --git a/tests/system-test/99-TDcase/TS-5130.py b/tests/system-test/99-TDcase/TS-5130.py index a5a9d373a1..68a9ae9d0e 100644 --- a/tests/system-test/99-TDcase/TS-5130.py +++ b/tests/system-test/99-TDcase/TS-5130.py @@ -20,19 +20,12 @@ class TDTestCase: def test_connect_user(self, uname): try: for db in ['information_schema', 'performance_schema']: - new_conn = taos.connect(host = self.conn._host, config = self.conn._config, - database =db, user=uname, password=self.passwd[uname]) - cursor = new_conn.cursor() - cursor.execute(f'show databases') - result = cursor.fetchall() - dbname = [i for i in result] + new_tdsql = tdCom.newTdSql(user=uname, password=self.passwd[uname], database=db) + result = new_tdsql.getResult(sql=f'show databases') assert result == [('information_schema',), ('performance_schema',)] tdLog.success(f"Test User {uname} for {db} .......[OK]") - new_conn.close() - except: - tdLog.debug(f"{__file__} Failed!") - exit(-1) + tdLog.exit(f'{__file__} failed') def run(self): self.prepare_user() @@ -41,7 +34,7 @@ class TDTestCase: def stop(self): tdSql.close() - tdLog.success(f"{__file__} successful executed") + tdLog.success(f"{__file__} successfully executed") tdCases.addWindows(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase()) From c29d22449c6033d27cf6b20afd3b6377d59d9ac6 Mon Sep 17 00:00:00 2001 From: zhipz Date: Tue, 9 Jul 2024 17:07:47 +0800 Subject: [PATCH 03/37] Using the checkdata. --- tests/system-test/99-TDcase/TS-5130.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/system-test/99-TDcase/TS-5130.py b/tests/system-test/99-TDcase/TS-5130.py index 68a9ae9d0e..504500fa0e 100644 --- a/tests/system-test/99-TDcase/TS-5130.py +++ b/tests/system-test/99-TDcase/TS-5130.py @@ -21,8 +21,9 @@ class TDTestCase: try: for db in ['information_schema', 'performance_schema']: new_tdsql = tdCom.newTdSql(user=uname, password=self.passwd[uname], database=db) - result = new_tdsql.getResult(sql=f'show databases') - assert result == [('information_schema',), ('performance_schema',)] + new_tdsql.query('show databases') + new_tdsql.checkData(0, 0, 'information_schema') + new_tdsql.checkData(1, 0, 'performance_schema') tdLog.success(f"Test User {uname} for {db} .......[OK]") except: tdLog.exit(f'{__file__} failed') From 80611e553c05e40a4dbe9ce66696a5a9a26bf531 Mon Sep 17 00:00:00 2001 From: zhipz Date: Mon, 15 Jul 2024 14:33:51 +0800 Subject: [PATCH 04/37] Fixing the incorrect version of common.py --- tests/pytest/util/common.py | 150 ++++++++++++++++++++++++++++++++---- 1 file changed, 133 insertions(+), 17 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 412ce22545..38d8502572 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -139,7 +139,7 @@ class TDCom: self.stream_suffix = "_stream" self.range_count = 5 self.default_interval = 5 - self.stream_timeout = 12 + self.stream_timeout = 60 self.create_stream_sleep = 0.5 self.record_history_ts = str() self.precision = "ms" @@ -201,6 +201,9 @@ class TDCom: self.cast_tag_stb_filter_des_select_elm = "ts, t1, t2, t3, t4, cast(t1 as TINYINT UNSIGNED), t6, t7, t8, t9, t10, cast(t2 as varchar(256)), t12, cast(t3 as bool)" self.tag_count = len(self.tag_filter_des_select_elm.split(",")) self.state_window_range = list() + + self.custom_col_val = 0 + self.part_val_list = [1, 2] # def init(self, conn, logSql): # # tdSql.init(conn.cursor(), logSql) @@ -506,7 +509,22 @@ class TDCom: if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] break + if platform.system().lower() == 'windows': + win_sep = "\\" + buildPath = buildPath.replace(win_sep,'/') + return buildPath + + def getTaosdPath(self, dnodeID="dnode1"): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + taosdPath = buildPath + "/../sim/" + dnodeID + tdLog.info("taosdPath: %s" % taosdPath) + return taosdPath + def getClientCfgPath(self): buildPath = self.getBuildPath() @@ -519,21 +537,21 @@ class TDCom: tdLog.info("cfgPath: %s" % cfgPath) return cfgPath - def newcon(self,host='localhost',port=6030,user='root',password='taosdata'): - con=taos.connect(host=host, user=user, password=password, port=port) + def newcon(self,host='localhost',port=6030,user='root',password='taosdata', database='None'): + con=taos.connect(host=host, user=user, password=password, port=port, database=database) # print(con) return con - def newcur(self,host='localhost',port=6030,user='root',password='taosdata', database=None): + def newcur(self,host='localhost',port=6030,user='root',password='taosdata',databse='None'): cfgPath = self.getClientCfgPath() - con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port, database=None) + con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port,database='None') cur=con.cursor() # print(cur) return cur - def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata', database=None): + def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata'): newTdSql = TDSql() - cur = self.newcur(host=host,port=port,user=user,password=password, database=None) + cur = self.newcur(host=host,port=port,user=user,password=password) newTdSql.init(cur, False) return newTdSql @@ -1244,7 +1262,7 @@ class TDCom: default_ctbname_index_start_num += 1 tdSql.execute(create_stable_sql) - def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None): + def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None, additional_ts=None, custom_col_index=None, col_value_type=None, force_pk_val=None): """_summary_ Args: @@ -1254,6 +1272,8 @@ class TDCom: """ self.column_value_list = list() self.ts_value = self.genTs()[0] + if additional_ts is not None: + self.additional_ts = self.genTs(additional_ts=additional_ts)[2] if ts_value is not None: self.ts_value = ts_value @@ -1277,7 +1297,22 @@ class TDCom: for i in range(int(len(self.column_value_list)/2)): index_num = random.randint(0, len(self.column_value_list)-1) self.column_value_list[index_num] = None - self.column_value_list = [self.ts_value] + self.column_value_list + + if custom_col_index is not None: + if col_value_type == "Random": + pass + elif col_value_type == "Incremental": + self.column_value_list[custom_col_index] = self.custom_col_val + self.custom_col_val += 1 + elif col_value_type == "Part_equal": + self.column_value_list[custom_col_index] = random.choice(self.part_val_list) + + self.column_value_list = [self.ts_value] + [self.additional_ts] + self.column_value_list if additional_ts is not None else [self.ts_value] + self.column_value_list + if col_value_type == "Incremental" and custom_col_index==1: + self.column_value_list[custom_col_index] = self.custom_col_val if force_pk_val is None else force_pk_val + if col_value_type == "Part_equal" and custom_col_index==1: + self.column_value_list[custom_col_index] = random.randint(0, self.custom_col_val) if force_pk_val is None else force_pk_val + def screate_table(self, dbname=None, tbname="tb", use_name="table", column_elm_list=None, count=1, default_tbname_prefix="tb", default_tbname_index_start_num=1, @@ -1318,7 +1353,7 @@ class TDCom: default_tbname_index_start_num += 1 tdSql.execute(create_table_sql) - def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False): + def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False, custom_col_index=None, col_value_type="random"): """insert rows Args: @@ -1338,7 +1373,7 @@ class TDCom: if tbname is not None: self.tbname = tbname - self.sgen_column_value_list(column_ele_list, need_null, ts_value) + self.sgen_column_value_list(column_ele_list, need_null, ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) # column_value_str = ", ".join(str(v) for v in self.column_value_list) column_value_str = "" for column_value in self.column_value_list: @@ -1355,7 +1390,7 @@ class TDCom: else: for num in range(count): ts_value = self.genTs()[0] - self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s') + self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s', custom_col_index=custom_col_index, col_value_type=col_value_type) column_value_str = "" for column_value in self.column_value_list: if column_value is None: @@ -1673,8 +1708,8 @@ class TDCom: res1 = self.round_handle(res1) res2 = self.round_handle(res2) if latency < self.stream_timeout: - latency += 0.2 - time.sleep(0.2) + latency += 0.5 + time.sleep(0.5) else: if latency == 0: return False @@ -1762,7 +1797,7 @@ class TDCom: self.sdelete_rows(tbname=self.ctb_name, start_ts=self.time_cast(self.record_history_ts, "-")) self.sdelete_rows(tbname=self.tb_name, start_ts=self.time_cast(self.record_history_ts, "-")) - def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None): + def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None, custom_col_index=None, col_value_type="random"): """prepare stream data Args: @@ -1825,11 +1860,89 @@ class TDCom: if fill_history_value == 1: for i in range(self.range_count): ts_value = str(self.date_time)+f'-{self.default_interval*(i+1)}s' - self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value) - self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value) + self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) + self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) if i == 1: self.record_history_ts = ts_value + def get_subtable(self, tbname_pre): + tdSql.query(f'show tables') + tbname_list = list(map(lambda x:x[0], tdSql.queryResult)) + for tbname in tbname_list: + if tbname_pre in tbname: + return tbname + + def get_subtable_wait(self, tbname_pre): + tbname = self.get_subtable(tbname_pre) + latency = 0 + while tbname is None: + tbname = self.get_subtable(tbname_pre) + if latency < self.stream_timeout: + latency += 1 + time.sleep(1) + return tbname + + def get_group_id_from_stb(self, stbname): + tdSql.query(f'select distinct group_id from {stbname}') + cnt = 0 + while len(tdSql.queryResult) == 0: + tdSql.query(f'select distinct group_id from {stbname}') + if cnt < self.default_interval: + cnt += 1 + time.sleep(1) + else: + return False + return tdSql.queryResult[0][0] + + def update_json_file_replica(self, json_file_path, new_replica_value, output_file_path=None): + """ + Read a JSON file, update the 'replica' value, and write the result back to a file. + + Parameters: + json_file_path (str): The path to the original JSON file. + new_replica_value (int): The new 'replica' value to be set. + output_file_path (str, optional): The path to the output file where the updated JSON will be saved. + If not provided, the original file will be overwritten. + + Returns: + None + """ + try: + # Read the JSON file and load its content into a Python dictionary + with open(json_file_path, 'r', encoding='utf-8') as file: + data = json.load(file) + + # Iterate over each item in the 'databases' list to find 'dbinfo' and update 'replica' + for db in data['databases']: + if 'dbinfo' in db: + db['dbinfo']['replica'] = new_replica_value + + # Convert the updated dictionary back into a JSON string with indentation for readability + updated_json_str = json.dumps(data, indent=4, ensure_ascii=False) + + # Write the updated JSON string to a file + if output_file_path: + # If an output file path is provided, write to the new file + with open(output_file_path, 'w', encoding='utf-8') as output_file: + output_file.write(updated_json_str) + else: + # Otherwise, overwrite the original file with the updated content + with open(json_file_path, 'w', encoding='utf-8') as file: + file.write(updated_json_str) + + except json.JSONDecodeError as e: + # Handle JSON decoding error (e.g., if the file is not valid JSON) + print(f"JSON decode error: {e}") + except FileNotFoundError: + # Handle the case where the JSON file is not found at the given path + print(f"File not found: {json_file_path}") + except KeyError as e: + # Handle missing key error (e.g., if 'databases' or 'dbinfo' is not present) + print(f"Key error: {e}") + except Exception as e: + # Handle any other exceptions that may occur + print(f"An error occurred: {e}") + def is_json(msg): if isinstance(msg, str): try: @@ -1864,4 +1977,7 @@ def dict2toml(in_dict: dict, file:str): with open(file, 'w') as f: toml.dump(in_dict, f) + + + tdCom = TDCom() From c0405f76318682a8489d83e28630717510ec8126 Mon Sep 17 00:00:00 2001 From: zhipz Date: Mon, 15 Jul 2024 16:37:21 +0800 Subject: [PATCH 05/37] Adding test case for TS-5142 --- tests/parallel_test/cases.task | 1 + .../system-test/8-stream/state_window_case.py | 60 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 tests/system-test/8-stream/state_window_case.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 69709a201a..6c7a3a4b51 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -55,6 +55,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_session_ext.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/partition_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/state_window_case.py #,,n,system-test,python3 ./test.py -f 8-stream/vnode_restart.py -N 4 #,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4 ,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 diff --git a/tests/system-test/8-stream/state_window_case.py b/tests/system-test/8-stream/state_window_case.py new file mode 100644 index 0000000000..5ecf8d7832 --- /dev/null +++ b/tests/system-test/8-stream/state_window_case.py @@ -0,0 +1,60 @@ +import sys +from util.log import * +from util.sql import * + +from util.cases import * +from util.common import * + + +class TDTestCase: + updatecfgDict = {'debugFlag':135,} + def init(self, conn, logSql, replicaVar = 1): + self.replicaVar = replicaVar + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.tdCom = tdCom + def init_case(self): + tdLog.debug("==========init case==========") + tdSql.execute("create database test") + tdSql.execute("use test") + tdSql.execute("CREATE STABLE `st_variable_data` (`load_time` TIMESTAMP, `collect_time` TIMESTAMP, `var_value` NCHAR(300)) TAGS (`factory_id` NCHAR(30), `device_code` NCHAR(80), `var_name` NCHAR(120), `var_type` NCHAR(30), `var_address` NCHAR(100), `var_attribute` NCHAR(30), `device_name` NCHAR(150), `var_desc` NCHAR(200), `trigger_value` NCHAR(50), `var_category` NCHAR(50), `var_category_desc` NCHAR(200));") + tdSql.execute('CREATE TABLE aaa using `st_variable_data` tags("a1","a2", "a3","a4","a5","a6","a7","a8","a9","a10","a11")') + time.sleep(2) + + def create_stream(self): + tdLog.debug("==========create stream==========") + tdSql.execute("CREATE STREAM stream_device_alarm TRIGGER AT_ONCE DELETE_MARK 30d INTO st_device_alarm tags(factory_id varchar(20), device_code varchar(80), var_name varchar(200))\ + as select _wstart start_time, last(load_time) end_time, first(var_value) var_value, (case when lower(var_value)=lower(trigger_value) then '1' else '0' end) state_flag from st_variable_data\ + PARTITION BY tbname tname, factory_id, device_code, var_name STATE_WINDOW(case when lower(var_value)=lower(trigger_value) then '1' else '0' end)") + time.sleep(2) + tdSql.execute("CREATE STREAM stream_device_alarm2 TRIGGER AT_ONCE DELETE_MARK 30d INTO st_device_alarm2 tags(factory_id varchar(20), device_code varchar(80), var_name varchar(200))\ + as select _wstart start_time, last(load_time) end_time, first(var_value) var_value, 1 state_flag from st_variable_data\ + PARTITION BY tbname tname, factory_id, device_code, var_name STATE_WINDOW(case when lower(var_value)=lower(trigger_value) then '1' else '0' end)") + time.sleep(2) + + def insert_data(self): + try: + tdSql.execute("insert into aaa values('2024-07-15 14:00:00', '2024-07-15 14:00:00', 'a8')", queryTimes=5, show=True) + time.sleep(0.01) + tdSql.execute("insert into aaa values('2024-07-15 14:10:00', '2024-07-15 14:10:00', 'a9')", queryTimes=5, show=True) + time.sleep(1) + except Exception as error: + tdLog.exit(f"insert data failed {error}") + + def run(self): + self.init_case() + self.create_stream() + self.insert_data() + tdSql.query("select state_flag from st_device_alarm") + tdSql.checkData(0, 0, 0, show=True) + tdSql.checkData(1, 0, 1, show=True) + tdSql.query("select state_flag from st_device_alarm2") + tdSql.checkData(0, 0, 1, show=True) + tdSql.checkData(1, 0, 1, show=True) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 0d8caf8dd9281a8237ccd6b71628bfe2e3df9ebf Mon Sep 17 00:00:00 2001 From: zhipz Date: Tue, 16 Jul 2024 09:11:24 +0800 Subject: [PATCH 06/37] Changing the database default values to the python None --- tests/pytest/util/common.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 38d8502572..ef69c065f0 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -537,21 +537,21 @@ class TDCom: tdLog.info("cfgPath: %s" % cfgPath) return cfgPath - def newcon(self,host='localhost',port=6030,user='root',password='taosdata', database='None'): + def newcon(self,host='localhost',port=6030,user='root',password='taosdata', database=None): con=taos.connect(host=host, user=user, password=password, port=port, database=database) # print(con) return con - def newcur(self,host='localhost',port=6030,user='root',password='taosdata',databse='None'): + def newcur(self,host='localhost',port=6030,user='root',password='taosdata',database=None): cfgPath = self.getClientCfgPath() - con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port,database='None') + con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port,database=database) cur=con.cursor() # print(cur) return cur - def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata'): + def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata', database = None): newTdSql = TDSql() - cur = self.newcur(host=host,port=port,user=user,password=password) + cur = self.newcur(host=host,port=port,user=user,password=password, database=database) newTdSql.init(cur, False) return newTdSql From 113cb57326a4c03d459e01f62b0870191a6a90c2 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 17 Jul 2024 19:35:35 +0800 Subject: [PATCH 07/37] adj stream operator result --- include/libs/executor/storageapi.h | 99 +- include/libs/function/function.h | 2 +- include/libs/stream/streamState.h | 38 +- include/libs/stream/tstreamFileState.h | 58 +- source/libs/executor/inc/executorInt.h | 134 +- source/libs/executor/src/groupoperator.c | 166 +- source/libs/executor/src/scanoperator.c | 84 +- .../executor/src/streamcountwindowoperator.c | 259 ++- .../executor/src/streameventwindowoperator.c | 232 ++- source/libs/executor/src/streamfilloperator.c | 421 +++-- .../executor/src/streamtimewindowoperator.c | 1551 ++++++++++++----- source/libs/executor/src/tfill.c | 4 +- source/libs/stream/inc/streamBackendRocksdb.h | 6 +- source/libs/stream/src/streamBackendRocksdb.c | 28 +- source/libs/stream/src/streamSessionState.c | 309 +++- source/libs/stream/src/streamState.c | 887 ++-------- source/libs/stream/src/streamUpdate.c | 24 +- source/libs/stream/src/tstreamFileState.c | 113 +- source/util/src/tscalablebf.c | 6 +- 19 files changed, 2588 insertions(+), 1833 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 0a33798f0c..e91b7c71ea 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -78,7 +78,7 @@ typedef struct SMetaEntry { } smaEntry; }; - uint8_t* pBuf; + uint8_t* pBuf; SColCmprWrapper colCmpr; // col compress alg } SMetaEntry; @@ -105,15 +105,15 @@ typedef struct SMTbCursor { } SMTbCursor; typedef struct SMCtbCursor { - struct SMeta* pMeta; - void* pCur; - tb_uid_t suid; - void* pKey; - void* pVal; - int kLen; - int vLen; - int8_t paused; - int lock; + struct SMeta* pMeta; + void* pCur; + tb_uid_t suid; + void* pKey; + void* pVal; + int kLen; + int vLen; + int8_t paused; + int lock; } SMCtbCursor; typedef struct SRowBuffPos { @@ -135,17 +135,17 @@ typedef struct SMetaTableInfo { typedef struct SSnapContext { struct SMeta* pMeta; - int64_t snapVersion; - void* pCur; - int64_t suid; - int8_t subType; - SHashObj* idVersion; - SHashObj* suidInfo; - SArray* idList; - int32_t index; - int8_t withMeta; - int8_t queryMeta; // true-get meta, false-get data - bool hasPrimaryKey; + int64_t snapVersion; + void* pCur; + int64_t suid; + int8_t subType; + SHashObj* idVersion; + SHashObj* suidInfo; + SArray* idList; + int32_t index; + int8_t withMeta; + int8_t queryMeta; // true-get meta, false-get data + bool hasPrimaryKey; } SSnapContext; typedef struct { @@ -229,15 +229,15 @@ typedef struct SStoreTqReader { bool (*tqReaderCurrentBlockConsumed)(); struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it -// int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it + // int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it int32_t (*tqReaderSetSubmitMsg)(); // todo remove it -// bool (*tqReaderNextBlockFilterOut)(); + // bool (*tqReaderNextBlockFilterOut)(); } SStoreTqReader; typedef struct SStoreSnapshotFn { - bool (*taosXGetTablePrimaryKey)(SSnapContext *ctx); - void (*taosXSetTablePrimaryKey)(SSnapContext *ctx, int64_t uid); + bool (*taosXGetTablePrimaryKey)(SSnapContext* ctx); + void (*taosXSetTablePrimaryKey)(SSnapContext* ctx, int64_t uid); int32_t (*setForSnapShot)(SSnapContext* ctx, int64_t uid); int32_t (*destroySnapshot)(SSnapContext* ctx); SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx); @@ -270,7 +270,7 @@ typedef struct SStoreMeta { int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio); - void* (*storeGetIndexInfo)(void *pVnode); + void* (*storeGetIndexInfo)(void* pVnode); void* (*getInvertIndex)(void* pVnode); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list); @@ -329,29 +329,31 @@ typedef struct { typedef struct SStateStore { int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname); - int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache); + int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, + int32_t* pWinCode); - int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); - int32_t (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used); - int32_t (*streamStateClearBuff)(SStreamState* pState, void* pVal); + int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, + int32_t* pWinCode); + void (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used); + void (*streamStateClearBuff)(SStreamState* pState, void* pVal); void (*streamStateFreeVal)(void* val); int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); - int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode); bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key); int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal); - int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key); - int32_t (*streamStateClear)(SStreamState* pState); + void (*streamStateDel)(SStreamState* pState, const SWinKey* key); + void (*streamStateClear)(SStreamState* pState); void (*streamStateSetNumber)(SStreamState* pState, int32_t number, int32_t tsIdex); - int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); + void (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); - int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key); + void (*streamStateFillDel)(SStreamState* pState, const SWinKey* key); - int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur); - int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur); + void (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur); + void (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur); SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); @@ -363,26 +365,28 @@ typedef struct SStateStore { int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, - int32_t* pVLen); + int32_t* pVLen, int32_t* pWinCode); int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); - int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); - int32_t (*streamStateSessionReset)(SStreamState* pState, void* pVal); - int32_t (*streamStateSessionClear)(SStreamState* pState); + void (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); + void (*streamStateSessionReset)(SStreamState* pState, void* pVal); + void (*streamStateSessionClear)(SStreamState* pState); int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode); int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t (*streamStateCountGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur, const SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, - void** ppVal, int32_t* pVLen); + void** ppVal, int32_t* pVLen, int32_t* pWinCode); int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); - int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); - TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); + int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, + int32_t pkLen, SUpdateInfo** ppInfo); + TSKEY(*updateInfoFillBlockData) + (SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); @@ -391,7 +395,8 @@ typedef struct SStateStore { void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); - int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); + int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); @@ -413,7 +418,7 @@ typedef struct SStateStore { SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId); void (*streamStateClose)(SStreamState* pState, bool remove); int32_t (*streamStateBegin)(SStreamState* pState); - int32_t (*streamStateCommit)(SStreamState* pState); + void (*streamStateCommit)(SStreamState* pState); void (*streamStateDestroy)(SStreamState* pState, bool remove); int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts); diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 924682f223..700cc5ba7f 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -294,7 +294,7 @@ typedef struct SPoint { void *val; } SPoint; -int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2, +void taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2, int32_t inputType); #define LEASTSQUARES_DOUBLE_ITEM_LENGTH 25 diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 5768160fdb..06da578749 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -32,7 +32,7 @@ extern "C" { SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId); void streamStateClose(SStreamState* pState, bool remove); int32_t streamStateBegin(SStreamState* pState); -int32_t streamStateCommit(SStreamState* pState); +void streamStateCommit(SStreamState* pState); void streamStateDestroy(SStreamState* pState, bool remove); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); int32_t streamStateDelTaskDb(SStreamState* pState); @@ -41,22 +41,23 @@ int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); -int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode); bool streamStateCheck(SStreamState* pState, const SWinKey* key); int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal); -int32_t streamStateDel(SStreamState* pState, const SWinKey* key); -int32_t streamStateClear(SStreamState* pState); +void streamStateDel(SStreamState* pState, const SWinKey* key); +void streamStateClear(SStreamState* pState); void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex); -int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); +void streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); // session window -int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen, + int32_t* pWinCode); int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); -int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key); -int32_t streamStateSessionReset(SStreamState* pState, void* pVal); -int32_t streamStateSessionClear(SStreamState* pState); +void streamStateSessionDel(SStreamState* pState, const SSessionKey* key); +void streamStateSessionReset(SStreamState* pState, void* pVal); +void streamStateSessionClear(SStreamState* pState); int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); @@ -70,21 +71,22 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons // state window int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode); // fill int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); -int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key); +void streamStateFillDel(SStreamState* pState, const SWinKey* key); -int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); -int32_t streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used); -int32_t streamStateClearBuff(SStreamState* pState, void* pVal); +int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, + int32_t* pWinCode); +void streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used); +void streamStateClearBuff(SStreamState* pState, void* pVal); void streamStateFreeVal(void* val); // count window int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, - int32_t* pVLen); + int32_t* pVLen, int32_t* pWinCode); int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); @@ -97,11 +99,11 @@ void streamStateResetCur(SStreamStateCur* pCur); int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); -int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); -int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); +void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); +void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname); -int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache); +int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode); void streamStateReloadInfo(SStreamState* pState, TSKEY ts); diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 68b9c4baa2..7d7001fdf9 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -18,10 +18,10 @@ #include "os.h" +#include "storageapi.h" #include "tarray.h" #include "tdef.h" #include "tlist.h" -#include "storageapi.h" #ifdef __cplusplus extern "C" { @@ -30,53 +30,56 @@ extern "C" { typedef struct SStreamFileState SStreamFileState; typedef SList SStreamSnapshot; -typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen); +typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen); typedef int32_t (*_state_buff_remove_fn)(void* pRowBuff, const void* pKey, size_t keyLen); typedef int32_t (*_state_buff_remove_by_pos_fn)(SStreamFileState* pState, SRowBuffPos* pPos); -typedef void (*_state_buff_cleanup_fn)(void* pRowBuff); -typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num); +typedef void (*_state_buff_cleanup_fn)(void* pRowBuff); +typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num); typedef int32_t (*_state_file_remove_fn)(SStreamFileState* pFileState, const void* pKey); typedef int32_t (*_state_file_get_fn)(SStreamFileState* pFileState, void* pKey, void* data, int32_t* pDataLen); typedef int32_t (*_state_file_clear_fn)(SStreamState* pState); -typedef int32_t (*_state_fun_get_fn) (SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); +typedef int32_t (*_state_fun_get_fn)(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, + int32_t* pVLen, int32_t* pWinCode); typedef int32_t (*range_cmpr_fn)(const SSessionKey* pWin1, const SSessionKey* pWin2); SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, - GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, - int64_t checkpointId, int8_t type); + GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, + int8_t type); void streamFileStateDestroy(SStreamFileState* pFileState); void streamFileStateClear(SStreamFileState* pFileState); bool needClearDiskBuff(SStreamFileState* pFileState); void streamFileStateReleaseBuff(SStreamFileState* pFileState, SRowBuffPos* pPos, bool used); -int32_t streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); +void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); -int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); -int32_t deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen); +int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, + int32_t* pWinCode); +void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen); int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal); bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen); void putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); SStreamSnapshot* getSnapshot(SStreamFileState* pFileState); -int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState); -int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); +void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState); +void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list); int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark); int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState); void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts); -void* getRowStateBuff(SStreamFileState* pFileState); -void* getStateFileStore(SStreamFileState* pFileState); -bool isDeteled(SStreamFileState* pFileState, TSKEY ts); -bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap); +void* getRowStateBuff(SStreamFileState* pFileState); +void* getStateFileStore(SStreamFileState* pFileState); +bool isDeteled(SStreamFileState* pFileState, TSKEY ts); +bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap); SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState); -int32_t getRowStateRowSize(SStreamFileState* pFileState); +int32_t getRowStateRowSize(SStreamFileState* pFileState); // session window -int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen); +int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen, + int32_t* pWinCode); int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen); @@ -85,7 +88,7 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen); SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen); -int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId); +void recoverSesssion(SStreamFileState* pFileState, int64_t ckId); void sessionWinStateClear(SStreamFileState* pFileState); void sessionWinStateCleanup(void* pBuff); @@ -94,20 +97,23 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState, SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey); SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey); SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count); -int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); -int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur); -int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, range_cmpr_fn cmpFn); +int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); +void sessionWinStateMoveToNext(SStreamStateCur* pCur); +int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, + range_cmpr_fn cmpFn); // state window int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode); // count window -int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen); +int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, + int32_t* pVLen, int32_t* pWinCode); int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); -//function -int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); +// function +int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, + int32_t* pWinCode); int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); #ifdef __cplusplus diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 6db3c780e2..c9acead357 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -221,10 +221,10 @@ enum { }; typedef enum ETableCountState { - TABLE_COUNT_STATE_NONE = 0, // before start scan - TABLE_COUNT_STATE_SCAN = 1, // cur group scanning - TABLE_COUNT_STATE_PROCESSED = 2, // cur group processed - TABLE_COUNT_STATE_END = 3, // finish or noneed to process + TABLE_COUNT_STATE_NONE = 0, // before start scan + TABLE_COUNT_STATE_SCAN = 1, // cur group scanning + TABLE_COUNT_STATE_PROCESSED = 2, // cur group processed + TABLE_COUNT_STATE_END = 3, // finish or noneed to process } ETableCountState; typedef struct SAggSupporter { @@ -272,14 +272,14 @@ typedef struct STableScanInfo { int32_t scanTimes; SSDataBlock* pResBlock; SHashObj* pIgnoreTables; - SSampleExecInfo sample; // sample execution info - int32_t tableStartIndex; // current group scan start - int32_t tableEndIndex; // current group scan end + SSampleExecInfo sample; // sample execution info + int32_t tableStartIndex; // current group scan start + int32_t tableEndIndex; // current group scan end int32_t currentGroupId; - int32_t currentTable; + int32_t currentTable; int8_t scanMode; int8_t assignBlockUid; - uint8_t countState; // empty table count state + uint8_t countState; // empty table count state bool hasGroupByTag; bool filesetDelimited; bool needCountEmptyTable; @@ -291,19 +291,19 @@ typedef enum ESubTableInputType { } ESubTableInputType; typedef struct STmsSubTableInput { - STsdbReader* pReader; + STsdbReader* pReader; SQueryTableDataCond tblCond; - STableKeyInfo* pKeyInfo; - bool bInMemReader; - ESubTableInputType type; - SSDataBlock* pReaderBlock; + STableKeyInfo* pKeyInfo; + bool bInMemReader; + ESubTableInputType type; + SSDataBlock* pReaderBlock; - SArray* aBlockPages; + SArray* aBlockPages; SSDataBlock* pPageBlock; - int32_t pageIdx; + int32_t pageIdx; - int32_t rowIdx; - int64_t* aTs; + int32_t rowIdx; + int64_t* aTs; SSDataBlock* pInputBlock; } STmsSubTableInput; @@ -312,10 +312,10 @@ typedef struct STmsSubTablesMergeInfo { SBlockOrderInfo* pTsOrderInfo; SBlockOrderInfo* pPkOrderInfo; - int32_t numSubTables; - STmsSubTableInput* aInputs; + int32_t numSubTables; + STmsSubTableInput* aInputs; SMultiwayMergeTreeInfo* pTree; - int32_t numSubTablesCompleted; + int32_t numSubTablesCompleted; int32_t numTableBlocksInMem; SDiskbasedBuf* pBlocksBuf; @@ -382,7 +382,7 @@ typedef struct STagScanInfo { SArray* aUidTags; // SArray SArray* aFilterIdxs; // SArray SStorageAPI* pStorageAPI; - SLimitInfo limitInfo; + SLimitInfo limitInfo; } STagScanInfo; typedef enum EStreamScanMode { @@ -400,18 +400,18 @@ enum { }; typedef struct SStreamAggSupporter { - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - SSDataBlock* pScanBlock; - SStreamState* pState; - int64_t gap; // stream session window gap - SqlFunctionCtx* pDummyCtx; // for combine - SSHashObj* pResultRows; - int32_t stateKeySize; - int16_t stateKeyType; - SDiskbasedBuf* pResultBuf; - SStateStore stateStore; - STimeWindow winRange; - SStorageAPI* pSessionAPI; + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + SSDataBlock* pScanBlock; + SStreamState* pState; + int64_t gap; // stream session window gap + SqlFunctionCtx* pDummyCtx; // for combine + SSHashObj* pResultRows; + int32_t stateKeySize; + int16_t stateKeyType; + SDiskbasedBuf* pResultBuf; + SStateStore stateStore; + STimeWindow winRange; + SStorageAPI* pSessionAPI; struct SUpdateInfo* pUpdateInfo; int32_t windowCount; int32_t windowSliding; @@ -577,7 +577,7 @@ typedef struct SOpCheckPointInfo { } SOpCheckPointInfo; typedef struct SStreamIntervalOperatorInfo { - SOptrBasicInfo binfo; // basic info + SOptrBasicInfo binfo; // basic info SSteamOpBasicInfo basic; SAggSupporter aggSup; // aggregate supporter SExprSupp scalarSupp; // supporter for perform scalar function @@ -835,7 +835,8 @@ void doBuildResultDatablock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbI * @brief copydata from hash table, instead of copying from SGroupResInfo's pRow */ int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, - SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, bool ignoreGroup); + SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, + bool ignoreGroup); bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo); @@ -855,7 +856,8 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* de extern void doDestroyExchangeOperatorInfo(void* param); -int32_t doFilterImpl(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo, SColumnInfoData** pResCol); +int32_t doFilterImpl(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo, + SColumnInfoData** pResCol); int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache); @@ -865,8 +867,9 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, void setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId); void setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer); -int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); -void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); +int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, + int32_t* rowEntryInfoOffset); +void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, int32_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, @@ -894,8 +897,8 @@ bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, SStateStore* pStore); -void appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, - uint64_t* pGp, void* pTbName); +void appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, + void* pTbName); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); @@ -906,7 +909,7 @@ bool groupbyTbname(SNodeList* pGroupList); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int64_t* pData); -void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, +int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI); SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); @@ -929,36 +932,39 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, i SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr, SStorageAPI* pApi, int32_t tsIndex); -int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, - STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic); -void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); +int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, + int32_t tsColIndex, STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic); +int32_t getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey); -void deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd); -int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated); +int32_t deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, + SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd); +void getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated); int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed); -int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar); +int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar); int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2); void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins); int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, - int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, - struct SOperatorInfo* pOperator, int64_t winDelta); -int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo); + int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, + struct SOperatorInfo* pOperator, int64_t winDelta); +void setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo); int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo); int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated); -void saveDeleteRes(SSHashObj* pStDelete, SSessionKey key); +int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key); void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey); void doBuildDeleteDataBlock(struct SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite); -void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock); +void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, + SSDataBlock* pBlock); void getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo); void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin); -void compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, +int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, SExecTaskInfo* pTaskInfo, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap); -int32_t releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI); +void releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI); void resetWinRange(STimeWindow* winRange); -bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); +bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, + TSKEY ts, void* pPkVal, int32_t len); int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval); void resetUnCloseSessionWinInfo(SSHashObj* winMap); void setStreamOperatorCompleted(struct SOperatorInfo* pOperator); @@ -979,26 +985,24 @@ void freeExchangeGetBasicOperatorParam(void* pParam); void freeOperatorParam(SOperatorParam* pParam, SOperatorParamType type); void freeResetOperatorParams(struct SOperatorInfo* pOperator, SOperatorParamType type, bool allFree); SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int32_t idx, bool clearParam); -void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange); -bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey); +void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange); +void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey); -void saveDeleteInfo(SArray* pWins, SSessionKey key); -void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins); -void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted); -void copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest); +int32_t saveDeleteInfo(SArray* pWins, SSessionKey key); +void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins); +int32_t copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted); +void copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest); bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo); bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType); bool compareVal(const char* v, const SStateKeys* pKey); bool inWinRange(STimeWindow* range, STimeWindow* cur); -void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result); +int32_t doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result); int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, TSKEY* primaryKeys, int32_t prevPosition, int32_t order); void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, int32_t status); - - #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index c7236a88e8..2e65ca2a7a 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -179,7 +179,7 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData size_t numOfGroupCols = taosArrayGetSize(pGroupCols); for (int32_t i = 0; i < numOfGroupCols; ++i) { - SColumn* pCol = (SColumn*) taosArrayGet(pGroupCols, i); + SColumn* pCol = (SColumn*)taosArrayGet(pGroupCols, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); // valid range check. todo: return error code. @@ -372,7 +372,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { bool hasRemainResultByHash(SOperatorInfo* pOperator) { SGroupbyOperatorInfo* pInfo = pOperator->info; - SSHashObj* pHashmap = pInfo->aggSup.pResultRowHashTable; + SSHashObj* pHashmap = pInfo->aggSup.pResultRowHashTable; return pInfo->groupResInfo.index < tSimpleHashGetSize(pHashmap); } @@ -413,7 +413,7 @@ void doBuildResultDatablockByHash(SOperatorInfo* pOperator, SOptrBasicInfo* pbIn static SSDataBlock* buildGroupResultDataBlockByHash(SOperatorInfo* pOperator) { SGroupbyOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pRes = pInfo->binfo.pRes; + SSDataBlock* pRes = pInfo->binfo.pRes; // after filter, if result block turn to null, get next from whole set while (1) { @@ -448,8 +448,8 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { return buildGroupResultDataBlockByHash(pOperator); } SGroupResInfo* pGroupResInfo = &pInfo->groupResInfo; - - int32_t order = pInfo->binfo.inputTsOrder; + + int32_t order = pInfo->binfo.inputTsOrder; int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; @@ -585,7 +585,7 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc blockDataDestroy(pDstBlock); return NULL; } - for(int i = 0; i < numOfCols; ++i) { + for (int i = 0; i < numOfCols; ++i) { pDstBlock->pBlockAgg[i].colId = -1; } } @@ -601,12 +601,12 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc if (pDataBlock->pBlockAgg && pDataBlock->pBlockAgg[slotId].colId != -1) { pDstBlock->pBlockAgg[i] = pDataBlock->pBlockAgg[slotId]; } else { - int32_t code = doEnsureCapacity(pDst, &pDstBlock->info, pDataBlock->info.rows, false); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - blockDataDestroy(pDstBlock); - return NULL; - } + int32_t code = doEnsureCapacity(pDst, &pDstBlock->info, pDataBlock->info.rows, false); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + blockDataDestroy(pDstBlock); + return NULL; + } colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); } @@ -840,9 +840,9 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SDataGroupInfo* pGroupInfo = (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { - if(pGroupInfo != NULL) { + if (pGroupInfo != NULL) { SSDataBlock* ret = buildPartitionResultForNotLoadBlock(pGroupInfo); - if(ret != NULL) return ret; + if (ret != NULL) return ret; } // try next group data if (pInfo->groupIndex + 1 >= taosArrayGetSize(pInfo->sortedGroupArray)) { @@ -885,7 +885,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { pInfo->binfo.pRes->info.id.groupId = pGroupInfo->groupId; pInfo->binfo.pRes->info.dataLoad = 1; pInfo->orderedRows = 0; - } else if (pInfo->pOrderInfoArr == NULL) { + } else if (pInfo->pOrderInfoArr == NULL) { qError("Exception, remainRows not zero, but pOrderInfoArr is NULL"); } @@ -1069,8 +1069,9 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - pInfo->rowCapacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, getBufPageSize(pInfo->pBuf), - blockDataGetSerialMetaSize(taosArrayGetSize(pInfo->binfo.pRes->pDataBlock))); + pInfo->rowCapacity = + blockDataGetCapacityInRow(pInfo->binfo.pRes, getBufPageSize(pInfo->pBuf), + blockDataGetSerialMetaSize(taosArrayGetSize(pInfo->binfo.pRes->pDataBlock))); pInfo->columnOffset = setupColumnOffset(pInfo->binfo.pRes, pInfo->rowCapacity); code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols); if (code != TSDB_CODE_SUCCESS) { @@ -1084,8 +1085,8 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -1110,8 +1111,8 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - SResultRow* pResultRow = - doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup, false); + SResultRow* pResultRow = doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, + false, pAggSup, false); return setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset); } @@ -1134,6 +1135,8 @@ static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; } static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -1159,8 +1162,13 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { } pDest->info.parTbName[0] = 0; if (pInfo->tbnameCalSup.numOfExprs > 0) { - void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname, false) == 0) { + void* tbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->stateStore.streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname, + false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode == TSDB_CODE_SUCCESS) { memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); pAPI->stateStore.streamStateFreeVal(tbname); } @@ -1174,20 +1182,35 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pDest->info.rows; pInfo->parIte = taosHashIterate(pInfo->pPartitions, pInfo->parIte); ASSERT(pDest->info.rows > 0); + +_end: + if (code != TSDB_CODE_SUCCESS) { + blockDataCleanup(pDest); + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } printDataBlock(pDest, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pDest; } -void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, - SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) { - void* pValue = NULL; - if (pAPI->streamStateGetParName(pState, groupId, &pValue, true) != 0) { +int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock, SStateStore* pAPI) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + void* pValue = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->streamStateGetParName(pState, groupId, &pValue, true, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId); memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); pTmpBlock->info.id.groupId = groupId; char* tbName = pSrcBlock->info.parTbName; if (pTableSup->numOfExprs > 0) { - projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs, NULL); + code = projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs, + NULL); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); memset(tbName, 0, TSDB_TABLE_NAME_LEN); int32_t len = 0; @@ -1198,7 +1221,8 @@ void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); memcpy(tbName, varDataVal(pData), len); - pAPI->streamStatePutParName(pState, groupId, tbName); + code = pAPI->streamStatePutParName(pState, groupId, tbName); + TSDB_CHECK_CODE(code, lino, _end); } memcpy(pTmpBlock->info.parTbName, tbName, len); pDestBlock->info.rows--; @@ -1209,42 +1233,58 @@ void appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTagSup } if (pTagSup->numOfExprs > 0) { - projectApplyFunctions(pTagSup->pExprInfo, pDestBlock, pTmpBlock, pTagSup->pCtx, pTagSup->numOfExprs, NULL); + code = projectApplyFunctions(pTagSup->pExprInfo, pDestBlock, pTmpBlock, pTagSup->pCtx, pTagSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); pDestBlock->info.rows--; } else { memcpy(pDestBlock->info.parTbName, pTmpBlock->info.parTbName, TSDB_TABLE_NAME_LEN); } void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); - colDataSetVal(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); + code = colDataSetVal(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); + TSDB_CHECK_CODE(code, lino, _end); pDestBlock->info.rows++; blockDataDestroy(pTmpBlock); } else { memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN); } pAPI->streamStateFreeVal(pValue); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { - SExecTaskInfo* pTask = pOperator->pTaskInfo; - +static int32_t buildStreamCreateTableResult(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SExecTaskInfo* pTask = pOperator->pTaskInfo; SStreamPartitionOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pSrc = pInfo->pInputDataBlock; if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) || taosHashGetSize(pInfo->pPartitions) == 0) { - return NULL; + goto _end; } blockDataCleanup(pInfo->pCreateTbRes); - blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions)); - SSDataBlock* pSrc = pInfo->pInputDataBlock; + code = blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions)); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->pTbNameIte != NULL) { SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte; int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0); - appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, - pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore); + code = appendCreateTableRow(pTask->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, pParInfo->groupId, + pSrc, rowId, pInfo->pCreateTbRes, &pTask->storageAPI.stateStore); + TSDB_CHECK_CODE(code, lino, _end); pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte); } - return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDataBlock* pBlock) { @@ -1267,18 +1307,20 @@ static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDat } static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamPartitionOperatorInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { return NULL; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStreamPartitionOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pCtRes = NULL; - if (hasRemainTbName(pInfo)) { - pCtRes = buildStreamCreateTableResult(pOperator); - if (pCtRes != NULL) { - return pCtRes; + code = buildStreamCreateTableResult(pOperator); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->pCreateTbRes->info.rows > 0) { + return pInfo->pCreateTbRes; } } @@ -1310,7 +1352,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { } break; case STREAM_CREATE_CHILD_TABLE: case STREAM_RETRIEVE: - case STREAM_CHECKPOINT: + case STREAM_CHECKPOINT: case STREAM_GET_ALL: { return pBlock; } @@ -1330,11 +1372,18 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL); pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, NULL); - pCtRes = buildStreamCreateTableResult(pOperator); - if (pCtRes != NULL) { - return pCtRes; + code = buildStreamCreateTableResult(pOperator); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->pCreateTbRes->info.rows > 0) { + return pInfo->pCreateTbRes; } return buildStreamPartitionResult(pOperator); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return NULL; } static void destroyStreamPartitionOperatorInfo(void* param) { @@ -1358,9 +1407,10 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosMemoryFreeClear(param); } -int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; +int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, + SExprSupp* pTbnameExpr) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -1372,7 +1422,8 @@ int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pPar pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartTbnameSup = pTbnameExpr; if (!pScanInfo->pUpdateInfo) { - code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); + code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, + pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); } return code; } @@ -1512,8 +1563,9 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, - destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 678d8d569e..cf464ad918 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1866,17 +1866,29 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS return TSDB_CODE_SUCCESS; } -static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) { +static int32_t calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; blockDataCleanup(pInfo->pCreateTbRes); if (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) { pBlock->info.parTbName[0] = 0; } else { - appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, - pBlock->info.id.groupId, pBlock, rowId, pInfo->pCreateTbRes, &pInfo->stateStore); + code = appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, + &pInfo->tagCalSup, pBlock->info.id.groupId, pBlock, rowId, pInfo->pCreateTbRes, + &pInfo->stateStore); + TSDB_CHECK_CODE(code, lino, _end); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); @@ -1897,9 +1909,13 @@ static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock* for (int32_t preJ = 0; preJ < pPreRes->info.rows; preJ++) { groupId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, preJ); if (pInfo->pPartTbnameSup) { - void* parTbname = NULL; - int32_t code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname, false); - if (code != TSDB_CODE_SUCCESS) { + void* parTbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, + &parTbname, false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { calBlockTbName(pInfo, pPreRes, preJ); memcpy(varDataVal(tbname), pPreRes->info.parTbName, strlen(pPreRes->info.parTbName)); } else { @@ -1912,10 +1928,17 @@ static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock* tbname[0] == 0 ? NULL : tbname); } } - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { + int32_t code = TSDB_CODE_SUCCESS; +int32_t lino = 0; SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); @@ -1944,8 +1967,11 @@ static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock } if (pInfo->tbnameCalSup.pExprInfo) { void* parTbname = NULL; - int32_t code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname, false); - if (code != TSDB_CODE_SUCCESS) { + int32_t winCode = TSDB_CODE_SUCCESS; + code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId, &parTbname, false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, srcStartTsCol[i], srcStartTsCol[i], ver); printDataBlock(pPreRes, "pre res", GET_TASKID(pInfo->pStreamScanOp->pTaskInfo)); calBlockTbName(pInfo, pPreRes, 0); @@ -1959,7 +1985,12 @@ static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock appendDataToSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId, tbname[0] == 0 ? NULL : tbname); } - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { @@ -2565,6 +2596,8 @@ static bool isStreamWindow(SStreamScanInfo* pInfo) { static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; const char* id = GET_TASKID(pTaskInfo); @@ -2667,7 +2700,11 @@ FETCH_NEXT_BLOCK: SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.parTbName[0]) { - pAPI->stateStore.streamStatePutParName(pStreamInfo->pState, pBlock->info.id.groupId, pBlock->info.parTbName); + code = + pAPI->stateStore.streamStatePutParName(pStreamInfo->pState, pBlock->info.id.groupId, pBlock->info.parTbName); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } // TODO move into scan @@ -3161,17 +3198,27 @@ _end: } void streamScanReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamScanInfo* pInfo = pOperator->info; if (!pInfo->pState) { return; } void* pBuff = NULL; int32_t len = 0; - pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), &pBuff, &len); + code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, + strlen(STREAM_SCAN_OP_STATE_NAME), &pBuff, &len); + TSDB_CHECK_CODE(code, lino, _end); + SUpdateInfo* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); - int32_t code = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo); + if (!pUpInfo) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + int32_t winCode = pInfo->stateStore.updateInfoDeserialize(pBuff, len, pUpInfo); taosMemoryFree(pBuff); - if (code == TSDB_CODE_SUCCESS && pInfo->pUpdateInfo) { + if (winCode == TSDB_CODE_SUCCESS && pInfo->pUpdateInfo) { if (pInfo->pUpdateInfo->minTS < 0) { pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo); pInfo->pUpdateInfo = pUpInfo; @@ -3181,9 +3228,9 @@ void streamScanReloadState(SOperatorInfo* pOperator) { ASSERT(pInfo->pUpdateInfo->minTS > pUpInfo->minTS); pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pUpInfo->maxDataVersion); SHashObj* curMap = pInfo->pUpdateInfo->pMap; - void *pIte = taosHashIterate(curMap, NULL); + void* pIte = taosHashIterate(curMap, NULL); while (pIte != NULL) { - size_t keySize = 0; + size_t keySize = 0; int64_t* pUid = taosHashGetKey(pIte, &keySize); taosHashPut(pUpInfo->pMap, pUid, sizeof(int64_t), pIte, sizeof(TSKEY)); pIte = taosHashIterate(curMap, pIte); @@ -3196,6 +3243,11 @@ void streamScanReloadState(SOperatorInfo* pOperator) { } else { pInfo->stateStore.updateInfoDestroy(pUpInfo); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void addPrimaryKeyCol(SSDataBlock* pBlock, uint8_t type, int32_t bytes) { diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 14f2dfbada..a1702885c8 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -55,13 +55,13 @@ void destroyStreamCountAggOperatorInfo(void* param) { pInfo->pUpdated = NULL; colDataDestroy(&pInfo->twAggSup.timeWindowData); - blockDataDestroy(pInfo->pDelRes); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); tSimpleHashCleanup(pInfo->pStUpdated); tSimpleHashCleanup(pInfo->pStDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - blockDataDestroy(pInfo->pCheckpointRes); + pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); @@ -70,9 +70,11 @@ void destroyStreamCountAggOperatorInfo(void* param) { bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; } -void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin, - SBuffInfo* pBuffInfo) { +int32_t setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin, + SBuffInfo* pBuffInfo) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t winCode = TSDB_CODE_SUCCESS; int32_t size = pAggSup->resultRowSize; pCurWin->winInfo.sessionWin.groupId = groupId; pCurWin->winInfo.sessionWin.win.skey = ts; @@ -80,26 +82,30 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, if (isSlidingCountWindow(pAggSup)) { if (pBuffInfo->winBuffOp == CREATE_NEW_WINDOW) { - pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); - code = TSDB_CODE_FAILED; + code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, + (void**)&pCurWin->winInfo.pStatePos, &size); + TSDB_CHECK_CODE(code, lino, _end); + + winCode = TSDB_CODE_FAILED; } else if (pBuffInfo->winBuffOp == MOVE_NEXT_WINDOW) { ASSERT(pBuffInfo->pCur); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur); - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); - if (code == TSDB_CODE_FAILED) { - pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, + (void**)&pCurWin->winInfo.pStatePos, &size); + if (winCode == TSDB_CODE_FAILED) { + code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, + (void**)&pCurWin->winInfo.pStatePos, &size); + TSDB_CHECK_CODE(code, lino, _end); } } else { pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount); - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); - if (code == TSDB_CODE_FAILED) { - pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, - (void**)&pCurWin->winInfo.pStatePos, &size); + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, + (void**)&pCurWin->winInfo.pStatePos, &size); + if (winCode == TSDB_CODE_FAILED) { + code = pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, + (void**)&pCurWin->winInfo.pStatePos, &size); + TSDB_CHECK_CODE(code, lino, _end); } } if (ts < pCurWin->winInfo.sessionWin.win.ekey) { @@ -108,10 +114,11 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, } else { code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, - (void**)&pCurWin->winInfo.pStatePos, &size); + (void**)&pCurWin->winInfo.pStatePos, &size, &winCode); + TSDB_CHECK_CODE(code, lino, _end); } - if (code == TSDB_CODE_SUCCESS) { + if (winCode == TSDB_CODE_SUCCESS) { pCurWin->winInfo.isOutput = true; } pCurWin->pWindowCount = @@ -120,19 +127,34 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, if (*pCurWin->pWindowCount == pAggSup->windowCount) { pBuffInfo->rebuildWindow = true; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void removeCountResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) { SSessionKey key = {0}; getSessionHashKey(pKey, &key); - tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); - tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); + int32_t code = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + if (code != TSDB_CODE_SUCCESS) { + qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } + + code = tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); + if (code != TSDB_CODE_SUCCESS) { + qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs, int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t num = 0; for (int32_t i = start; i < rows; i++) { if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) { @@ -150,7 +172,10 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI if (pWinInfo->winInfo.sessionWin.win.skey > pTs[start]) { needDelState = true; if (pStDeleted && pWinInfo->winInfo.isOutput) { - saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); + code = saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } pWinInfo->winInfo.sessionWin.win.skey = pTs[start]; @@ -218,11 +243,12 @@ bool inCountSlidingWindow(SStreamAggSupporter* pAggSup, STimeWindow* pWin, SData static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated, SSHashObj* pStDeleted) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamCountAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.id.groupId; - int64_t code = TSDB_CODE_SUCCESS; SResultRow* pResult = NULL; int32_t rows = pSDataBlock->info.rows; int32_t winRows = 0; @@ -236,8 +262,14 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; - blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); + if (!pStartTsCol) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; + code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); + TSDB_CHECK_CODE(code, lino, _end); + SStreamStateCur* pCur = NULL; COUNT_TYPE slidingRows = 0; @@ -250,7 +282,9 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } SCountWindowInfo curWin = {0}; buffInfo.rebuildWindow = false; - setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo); + code = setCountOutputBuf(pAggSup, startTsCols[i], groupId, &curWin, &buffInfo); + TSDB_CHECK_CODE(code, lino, _end); + if (!inCountSlidingWindow(pAggSup, &curWin.winInfo.sessionWin.win, &pSDataBlock->info)) { buffInfo.winBuffOp = MOVE_NEXT_WINDOW; continue; @@ -276,28 +310,27 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator, 0); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - qError("%s do stream count aggregate impl error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } - saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); + + code = saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_COUNT_OP(pOperator)) { - saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(curWin.winInfo, pStUpdated); - if (code != TSDB_CODE_SUCCESS) { - qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } if (isSlidingCountWindow(pAggSup)) { @@ -314,6 +347,11 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } i += winRows; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } destroySBuffInfo(pAggSup, &buffInfo); } @@ -376,10 +414,13 @@ int32_t doStreamCountEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper return tlen; } -void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { +int32_t doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamCountAggOperatorInfo* pInfo = pOperator->info; if (!pInfo) { - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } // 4.checksum @@ -387,8 +428,8 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato int32_t dataLen = len - sizeof(uint32_t); void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { - qError("stream count state is invalid"); - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } } @@ -400,10 +441,13 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SCountWindowInfo curWin = {0}; buf = decodeSSessionKey(buf, &key); SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; - setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); + code = setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); + TSDB_CHECK_CODE(code, lino, _end); + buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, - sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, + sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } // 2.twAggSup @@ -411,21 +455,37 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato // 3.dataVersion buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); - return buf; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + void* pBuf = NULL; SStreamCountAggOperatorInfo* pInfo = pOperator->info; if (needSaveStreamOperatorInfo(&pInfo->basic)) { int32_t len = doStreamCountEncodeOpState(NULL, 0, pOperator, true); - void* buf = taosMemoryCalloc(1, len); - void* pBuf = buf; + pBuf = taosMemoryCalloc(1, len); + if (!pBuf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME, - strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), buf, len); - taosMemoryFree(buf); + strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), pBuf, len); saveStreamOperatorStateComplete(&pInfo->basic); } + +_end: + taosMemoryFreeClear(pBuf); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { @@ -467,7 +527,9 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) { } } -void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { +int32_t doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -482,35 +544,61 @@ void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SAr SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]}; while (1) { SSessionKey curWin = {0}; - int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); - if (code == TSDB_CODE_FAILED) { + int32_t winCode = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); + if (winCode != TSDB_CODE_SUCCESS) { break; } doDeleteSessionWindow(pAggSup, &curWin); if (result) { - saveDeleteInfo(result, curWin); + code = saveDeleteInfo(result, curWin); + TSDB_CHECK_CODE(code, lino, _end); } } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, - SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd) { +int32_t deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, + SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); + if (!pWins) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + if (isSlidingCountWindow(pAggSup)) { - doDeleteCountWindows(pAggSup, pBlock, pWins); + code = doDeleteCountWindows(pAggSup, pBlock, pWins); + TSDB_CHECK_CODE(code, lino, _end); } else { - doDeleteTimeWindows(pAggSup, pBlock, pWins); + code = doDeleteTimeWindows(pAggSup, pBlock, pWins); + TSDB_CHECK_CODE(code, lino, _end); } removeSessionResults(pAggSup, pMapUpdate, pWins); - copyDeleteWindowInfo(pWins, pMapDelete); + code = copyDeleteWindowInfo(pWins, pMapDelete); + TSDB_CHECK_CODE(code, lino, _end); if (needAdd) { - copyDeleteWindowInfo(pWins, pPkDelete); + code = copyDeleteWindowInfo(pWins, pPkDelete); + TSDB_CHECK_CODE(code, lino, _end); } taosArrayDestroy(pWins); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SStreamCountAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -558,7 +646,9 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { bool add = pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator); - deleteCountWinState(&pInfo->streamAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, pInfo->pPkDeleted, add); + code = deleteCountWinState(&pInfo->streamAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, pInfo->pPkDeleted, + add); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_CLEAR) { doResetCountWindows(&pInfo->streamAggSup, pBlock); @@ -572,7 +662,8 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_CHECKPOINT) { pAggSup->stateStore.streamStateCommit(pAggSup->pState); doStreamCountSaveCheckpoint(pOperator); - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); continue; } else { ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); @@ -580,7 +671,8 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -591,12 +683,17 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); - copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + code = closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); + + code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated); initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator)) { copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted); @@ -607,14 +704,24 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { return opRes; } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } setStreamOperatorCompleted(pOperator); return NULL; } void streamCountReleaseState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamEventAggOperatorInfo* pInfo = pOperator->info; int32_t resSize = sizeof(TSKEY); char* pBuff = taosMemoryCalloc(1, resSize); + if (pBuff) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _end); + } memcpy(pBuff, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); qDebug("===stream=== count window operator relase state. "); pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_STATE_NAME, @@ -625,17 +732,25 @@ void streamCountReleaseState(SOperatorInfo* pOperator) { if (downstream->fpSet.releaseStreamStateFn) { downstream->fpSet.releaseStreamStateFn(downstream); } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void streamCountReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamCountAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; int32_t size = 0; void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME, - strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size); - TSKEY ts = *(TSKEY*)pBuf; + code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME, + strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size); + TSDB_CHECK_CODE(code, lino, _end); + + TSKEY ts = *(TSKEY*)pBuf; pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); taosMemoryFree(pBuf); @@ -644,6 +759,11 @@ void streamCountReloadState(SOperatorInfo* pOperator) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -725,7 +845,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME, strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), &buff, &len); if (res == TSDB_CODE_SUCCESS) { - doStreamCountDecodeOpState(buff, len, pOperator, true); + code = doStreamCountDecodeOpState(buff, len, pOperator, true); + TSDB_CHECK_CODE(code, lino, _error); taosMemoryFree(buff); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAgg, NULL, destroyStreamCountAggOperatorInfo, diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index e6ff4fe242..f79f29dc55 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -60,14 +60,14 @@ void destroyStreamEventOperatorInfo(void* param) { taosArrayDestroy(pInfo->pChildren); } colDataDestroy(&pInfo->twAggSup.timeWindowData); - blockDataDestroy(pInfo->pDelRes); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); tSimpleHashCleanup(pInfo->pSeUpdated); tSimpleHashCleanup(pInfo->pAllUpdated); tSimpleHashCleanup(pInfo->pSeDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - blockDataDestroy(pInfo->pCheckpointRes); + pInfo->pDelRes = blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); @@ -108,14 +108,15 @@ int32_t getEndCondIndex(bool* pEnd, int32_t start, int32_t rows) { static bool isWindowIncomplete(SEventWindowInfo* pWinInfo) { return !(pWinInfo->pWinFlag->startFlag && pWinInfo->pWinFlag->endFlag); } -int32_t reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { +void reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateReleaseBuf(pState, pPos, true); - return TSDB_CODE_SUCCESS; } -void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd, - int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey) { +int32_t setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd, + int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t winCode = TSDB_CODE_SUCCESS; int32_t size = pAggSup->resultRowSize; TSKEY ts = pTs[index]; bool start = pStart[index]; @@ -128,8 +129,8 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI SSessionKey leftWinKey = {.groupId = groupId}; void* pVal = NULL; int32_t len = 0; - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); - if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) { + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); + if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) { bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0); setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin); if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) { @@ -140,9 +141,9 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI pAggSup->stateStore.streamStateFreeCur(pCur); pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); SSessionKey rightWinKey = {.groupId = groupId}; - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0); - if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { + if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { int32_t endi = getEndCondIndex(pEnd, index, rows); if (endi < 0 || pTs[endi] >= rightWinKey.win.skey) { setEventWindowInfo(pAggSup, &rightWinKey, pVal, pCurWin); @@ -152,7 +153,9 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI } SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; - pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); + code = pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); + TSDB_CHECK_CODE(code, lino, _error); + setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin); pCurWin->pWinFlag->startFlag = start; pCurWin->pWinFlag->endFlag = end; @@ -162,8 +165,8 @@ _end: reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur); pNextWinKey->groupId = groupId; - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pNextWinKey, NULL, 0); - if (code != TSDB_CODE_SUCCESS) { + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pNextWinKey, NULL, 0); + if (winCode != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pNextWinKey); } if (pCurWin->winInfo.pStatePos->needFree) { @@ -172,11 +175,19 @@ _end: pAggSup->stateStore.streamStateFreeCur(pCur); qDebug("===stream===set event next win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey, pCurWin->winInfo.sessionWin.win.ekey); + +_error: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey, TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; *pRebuild = false; if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) { return 1; @@ -198,7 +209,10 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW if (pWin->skey > pTsData[i]) { if (pStDeleted && pWinInfo->winInfo.isOutput) { - saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); + code = saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->winInfo.sessionWin); pWin->skey = pTsData[i]; @@ -232,10 +246,11 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - int32_t winNum = 0; SStreamEventAggOperatorInfo* pInfo = pOperator->info; SResultRow* pCurResult = NULL; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; @@ -252,18 +267,23 @@ static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pC break; } setEventWindowFlag(pAggSup, &nextWinInfo); - compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, - pStDeleted, false); + code = compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, + pStUpdated, pStDeleted, false); + TSDB_CHECK_CODE(code, lino, _end); + pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag; - winNum++; } - return winNum; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -bool doDeleteEventWindow(SStreamAggSupporter* pAggSup, SSHashObj* pSeUpdated, SSessionKey* pKey) { +void doDeleteEventWindow(SStreamAggSupporter* pAggSup, SSHashObj* pSeUpdated, SSessionKey* pKey) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey); removeSessionResult(pAggSup, pSeUpdated, pAggSup->pResultRows, pKey); - return true; } static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, @@ -273,7 +293,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl SStreamEventAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.id.groupId; - int64_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t winRows = 0; @@ -289,6 +310,10 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + if (!pColDataInfo) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } tsCols = (int64_t*)pColDataInfo->pData; } else { return; @@ -302,7 +327,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl goto _end; } int32_t statusStart = 0; - filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); + code = filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); + TSDB_CHECK_CODE(code, lino, _end); SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; @@ -313,10 +339,13 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } int32_t statusEnd = 0; - filterExecute(pInfo->pEndCondInfo, pSDataBlock, &pColEnd, NULL, paramEnd.numOfCols, &statusEnd); + code = filterExecute(pInfo->pEndCondInfo, pSDataBlock, &pColEnd, NULL, paramEnd.numOfCols, &statusEnd); + TSDB_CHECK_CODE(code, lino, _end); int32_t rows = pSDataBlock->info.rows; - blockDataEnsureCapacity(pAggSup->pScanBlock, rows); + code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows); + TSDB_CHECK_CODE(code, lino, _end); + for (int32_t i = 0; i < rows; i += winRows) { if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i], NULL, 0)) { @@ -327,8 +356,10 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl bool allEqual = true; SEventWindowInfo curWin = {0}; SSessionKey nextWinKey = {0}; - setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, - &nextWinKey); + code = setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, + &nextWinKey); + TSDB_CHECK_CODE(code, lino, _end); + setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); bool rebuild = false; winRows = @@ -339,28 +370,35 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); - tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); + code = tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); + TSDB_CHECK_CODE(code, lino, _end); + doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && !isWindowIncomplete(&curWin)) { - saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); SSessionKey tmpSeInfo = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &tmpSeInfo); - tSimpleHashPut(pStDeleted, &tmpSeInfo, sizeof(SSessionKey), NULL, 0); + code = tSimpleHashPut(pStDeleted, &tmpSeInfo, sizeof(SSessionKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); continue; } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator, 0); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - break; - } - compactEventWindow(pOperator, &curWin, pInfo->pSeUpdated, pInfo->pSeDeleted, false); - saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); + + code = compactEventWindow(pOperator, &curWin, pInfo->pSeUpdated, pInfo->pSeDeleted, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->isHistoryOp) { - saveResult(curWin.winInfo, pInfo->pAllUpdated); + code = saveResult(curWin.winInfo, pInfo->pAllUpdated); + TSDB_CHECK_CODE(code, lino, _end); } if (isWindowIncomplete(&curWin)) { @@ -369,18 +407,22 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator)) { - saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResult(curWin.winInfo, pSeUpdated); + code = saveResult(curWin.winInfo, pSeUpdated); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } } @@ -389,6 +431,9 @@ _end: taosMemoryFree(pColStart); colDataDestroy(pColEnd); taosMemoryFree(pColEnd); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator) { @@ -429,10 +474,13 @@ int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper return tlen; } -void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) { +int32_t doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamEventAggOperatorInfo* pInfo = pOperator->info; if (!pInfo) { - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; @@ -440,9 +488,9 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato int32_t dataLen = len - sizeof(uint32_t); void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { - ASSERT(0); // debug qError("stream event state is invalid"); - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } // 1.streamAggSup.pResultRows @@ -452,10 +500,16 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); - pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap, - (void**)&winfo.pStatePos, &pAggSup->resultRowSize); + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateSessionAddIfNotExist( + pAggSup->pState, &winfo.sessionWin, pAggSup->gap, (void**)&winfo.pStatePos, &pAggSup->resultRowSize, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + ASSERT(winCode == TSDB_CODE_SUCCESS); + buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } // 2.twAggSup @@ -463,7 +517,12 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato // 3.dataVersion buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); - return buf; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) { @@ -504,6 +563,8 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { return NULL; } + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SStreamEventAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -549,7 +610,9 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { bool add = pInfo->destHasPrimaryKey && IS_NORMAL_EVENT_OP(pOperator); - deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted, pInfo->pPkDeleted, add); + code = deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted, + pInfo->pPkDeleted, add); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { pInfo->recvGetAll = true; @@ -561,7 +624,8 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); doStreamEventSaveCheckpoint(pOperator); pInfo->reCkBlock = true; - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); continue; } else { ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); @@ -569,7 +633,8 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -579,14 +644,27 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); - copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + code = closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); + TSDB_CHECK_CODE(code, lino, _end); + + code = copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated); if (pInfo->isHistoryOp) { SArray* pHisWins = taosArrayInit(16, sizeof(SEventWindowInfo)); - copyUpdateResult(&pInfo->pAllUpdated, pHisWins, sessionKeyCompareAsc); - getMaxTsWins(pHisWins, pInfo->historyWins); + if (!pHisWins) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = copyUpdateResult(&pInfo->pAllUpdated, pHisWins, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + + code = getMaxTsWins(pHisWins, pInfo->historyWins); + TSDB_CHECK_CODE(code, lino, _end); + taosArrayDestroy(pHisWins); } if (pInfo->destHasPrimaryKey && IS_NORMAL_EVENT_OP(pOperator)) { @@ -595,12 +673,18 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* resBlock = buildEventResult(pOperator); if (resBlock != NULL) { return resBlock; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } setStreamOperatorCompleted(pOperator); return NULL; } @@ -626,6 +710,8 @@ void streamEventReleaseState(SOperatorInfo* pOperator) { } void streamEventReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamEventAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; resetWinRange(&pAggSup->winRange); @@ -633,9 +719,11 @@ void streamEventReloadState(SOperatorInfo* pOperator) { SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0}; int32_t size = 0; void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_EVENT_OP_STATE_NAME, - strlen(STREAM_EVENT_OP_STATE_NAME), &pBuf, &size); - int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); + code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_EVENT_OP_STATE_NAME, + strlen(STREAM_EVENT_OP_STATE_NAME), &pBuf, &size); + TSDB_CHECK_CODE(code, lino, _end); + + int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); qDebug("===stream=== event window operator reload state. get result count:%d", num); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -663,15 +751,19 @@ void streamEventReloadState(SOperatorInfo* pOperator) { } setEventWindowFlag(pAggSup, &curInfo); if (!curInfo.pWinFlag->startFlag || curInfo.pWinFlag->endFlag) { - saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + TSDB_CHECK_CODE(code, lino, _end); continue; } - compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); + code = compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, curInfo.winInfo.sessionWin.groupId); + TSDB_CHECK_CODE(code, lino, _end); + if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { - saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + TSDB_CHECK_CODE(code, lino, _end); } if (!curInfo.pWinFlag->endFlag) { @@ -679,14 +771,18 @@ void streamEventReloadState(SOperatorInfo* pOperator) { } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResult(curInfo.winInfo, pInfo->pSeUpdated); + code = saveResult(curInfo.winInfo, pInfo->pSeUpdated); + TSDB_CHECK_CODE(code, lino, _end); } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) { - saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } SSessionKey key = {0}; getSessionHashKey(&curInfo.winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } } taosMemoryFree(pBuf); @@ -696,6 +792,11 @@ void streamEventReloadState(SOperatorInfo* pOperator) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -787,8 +888,9 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME, strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), &buff, &len); if (res == TSDB_CODE_SUCCESS) { - doStreamEventDecodeOpState(buff, len, pOperator); + code = doStreamEventDecodeOpState(buff, len, pOperator); taosMemoryFree(buff); + TSDB_CHECK_CODE(code, lino, _error); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 5f188c2c8a..518f7a33fb 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -26,12 +26,11 @@ #include "ttime.h" #include "function.h" +#include "operator.h" #include "querynodes.h" +#include "querytask.h" #include "tdatablock.h" #include "tfill.h" -#include "operator.h" -#include "querytask.h" - #define FILL_POS_INVALID 0 #define FILL_POS_START 1 @@ -56,8 +55,8 @@ TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { return win.skey; } -void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { - colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull); +int32_t setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) { + return colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull); } SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { @@ -83,7 +82,7 @@ void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) return NULL; } -void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { +void destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); tSimpleHashCleanup(pFillSup->pResMap); pFillSup->pResMap = NULL; @@ -96,36 +95,36 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { taosMemoryFree(pFillSup->nextNext.pRowVal); taosMemoryFree(pFillSup); - return NULL; } void destroySPoint(void* ptr) { - SPoint* point = (SPoint*) ptr; + SPoint* point = (SPoint*)ptr; taosMemoryFreeClear(point->val); } -void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { +void destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint); taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint); taosMemoryFree(pFillLinear); - return NULL; } -void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { + +void destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F || pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) { taosMemoryFreeClear(pFillInfo->pResRow->pRowVal); taosMemoryFreeClear(pFillInfo->pResRow); } - pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); + destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); + pFillInfo->pLinearInfo = NULL; + taosArrayDestroy(pFillInfo->delRanges); taosMemoryFree(pFillInfo); - return NULL; } static void destroyStreamFillOperatorInfo(void* param) { SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; - pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); - pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); + destroyStreamFillInfo(pInfo->pFillInfo); + destroyStreamFillSupporter(pInfo->pFillSup); pInfo->pRes = blockDataDestroy(pInfo->pRes); pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); @@ -161,19 +160,21 @@ void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupI int32_t curVLen = 0; int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); - ASSERT(code == TSDB_CODE_SUCCESS); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } pFillSup->cur.key = key.ts; } void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - void* pState = pOperator->pTaskInfo->streamInfo.pState; + void* pState = pOperator->pTaskInfo->streamInfo.pState; resetPrevAndNextWindow(pFillSup, pState, pAPI); SWinKey key = {.ts = ts, .groupId = groupId}; void* curVal = NULL; int32_t curVLen = 0; - bool hasCurKey = true; + bool hasCurKey = true; int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); if (code == TSDB_CODE_SUCCESS) { pFillSup->cur.key = key.ts; @@ -196,15 +197,10 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, pFillSup->prev.pRowVal = preVal; if (hasCurKey) { - code = pAPI->stateStore.streamStateCurNext(pState, pCur); - ASSERT(code == TSDB_CODE_SUCCESS); + pAPI->stateStore.streamStateCurNext(pState, pCur); } - code = pAPI->stateStore.streamStateCurNext(pState, pCur); - if (code != TSDB_CODE_SUCCESS) { - pAPI->stateStore.streamStateFreeCur(pCur); - pCur = NULL; - } + pAPI->stateStore.streamStateCurNext(pState, pCur); } else { pAPI->stateStore.streamStateFreeCur(pCur); pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key); @@ -218,16 +214,14 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, pFillSup->next.key = nextKey.ts; pFillSup->next.pRowVal = nextVal; if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { - code = pAPI->stateStore.streamStateCurNext(pState, pCur); + pAPI->stateStore.streamStateCurNext(pState, pCur); + SWinKey nextNextKey = {.groupId = groupId}; + void* nextNextVal = NULL; + int32_t nextNextVLen = 0; + code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); if (code == TSDB_CODE_SUCCESS) { - SWinKey nextNextKey = {.groupId = groupId}; - void* nextNextVal = NULL; - int32_t nextNextVLen = 0; - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); - if (code == TSDB_CODE_SUCCESS) { - pFillSup->nextNext.key = nextNextKey.ts; - pFillSup->nextNext.pRowVal = nextNextVal; - } + pFillSup->nextNext.key = nextNextKey.ts; + pFillSup->nextNext.pRowVal = nextNextVal; } } } @@ -286,13 +280,12 @@ static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pR } } -static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, - int32_t numOfCol) { +static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, int32_t numOfCol) { for (int32_t i = 0; i < numOfCol; i++) { if (!pFillCol[i].notFillCol) { int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); SResultCellData* pECell = getResultCell(pEndRow, slotId); - SPoint* pPoint = taosArrayGet(pEndPoins, slotId); + SPoint* pPoint = taosArrayGet(pEndPoins, slotId); pPoint->key = pEndRow->key; memcpy(pPoint->val, pECell->pData, pECell->bytes); } @@ -489,7 +482,10 @@ static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t group if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { return false; } - tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + int32_t code = tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } return true; } @@ -513,7 +509,10 @@ static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFill bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows); if (!filled) { SResultCellData* pCell = getResultCell(pResRow, slotId); - setRowCell(pColData, pBlock->info.rows, pCell); + int32_t code = setRowCell(pColData, pBlock->info.rows, pCell); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } } pBlock->info.rows++; @@ -531,7 +530,11 @@ static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; if (inWinRange(&pFillSup->winRange, &st)) { - buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + bool res = buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); + if (!res) { + int32_t code = TSDB_CODE_FAILED; + qError("%s failed at line %d since block is full", __func__, __LINE__); + } } pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillSup->interval.precision); @@ -540,10 +543,11 @@ static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { - uint64_t groupId = pBlock->info.id.groupId; - SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; + uint64_t groupId = pBlock->info.id.groupId; + SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; - if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) { + if ((pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId)) || + !inWinRange(&pFillSup->winRange, &st)) { pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillSup->interval.precision); pFillInfo->pLinearInfo->winIndex++; @@ -566,7 +570,10 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* if (pFillCol->notFillCol) { bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index); if (!filled) { - setRowCell(pColData, index, pCell); + int32_t code = setRowCell(pColData, index, pCell); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } } else { if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { @@ -575,7 +582,7 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* } SPoint* pEnd = taosArrayGet(pFillInfo->pLinearInfo->pEndPoints, slotId); double vCell = 0; - SPoint start = {0}; + SPoint start = {0}; start.key = pFillInfo->pResRow->key; start.val = pCell->pData; @@ -583,7 +590,10 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* cur.key = pFillInfo->current; cur.val = taosMemoryCalloc(1, pCell->bytes); taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type); - colDataSetVal(pColData, index, (const char*)cur.val, false); + int32_t code = colDataSetVal(pColData, index, (const char*)cur.val, false); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } destroySPoint(&cur); } } @@ -604,7 +614,11 @@ static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SRes static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { if (pFillInfo->needFill == false) { - buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + bool res = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); + if (!res) { + int32_t code = TSDB_CODE_FAILED; + qError("%s failed at line %d since block is full", __func__, __LINE__); + } return; } @@ -663,6 +677,8 @@ static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillS } static void doStreamFillImpl(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamFillSupporter* pFillSup = pInfo->pFillSup; @@ -685,20 +701,28 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) { keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) { - blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + code = blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + TSDB_CHECK_CODE(code, lino, _end); return; } pInfo->srcRowIndex++; } doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes); - blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + code = blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol); + TSDB_CHECK_CODE(code, lino, _end); blockDataCleanup(pInfo->pSrcBlock); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } -static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { - SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; - void* pState = pOp->pTaskInfo->streamInfo.pState; - +static int32_t buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; + void* pState = pOp->pTaskInfo->streamInfo.pState; SSDataBlock* pBlock = delRes; SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -707,61 +731,101 @@ static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_ SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false); - colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false); + code = colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false); + TSDB_CHECK_CODE(code, lino, _end); + colDataSetNULL(pUidCol, pBlock->info.rows); - colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); + code = colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false); + TSDB_CHECK_CODE(code, lino, _end); + colDataSetNULL(pCalStartCol, pBlock->info.rows); colDataSetNULL(pCalEndCol, pBlock->info.rows); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); - void* tbname = NULL; - pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname, false); - if (tbname == NULL) { + void* tbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname, false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + if (winCode != TSDB_CODE_SUCCESS) { colDataSetNULL(pTableCol, pBlock->info.rows); } else { + ASSERT(tbname); char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); - colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + TSDB_CHECK_CODE(code, lino, _end); pAPI->stateStore.streamStateFreeVal(tbname); } pBlock->info.rows++; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, - SSDataBlock* delRes) { +static int32_t buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SSDataBlock* delRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillSupporter* pFillSup = pInfo->pFillSup; if (hasPrevWindow(pFillSup)) { TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval); - buildDeleteRange(pOperator, start, endTs, groupId, delRes); + code = buildDeleteRange(pOperator, start, endTs, groupId, delRes); + TSDB_CHECK_CODE(code, lino, _end); } else if (hasNextWindow(pFillSup)) { TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval); - buildDeleteRange(pOperator, startTs, end, groupId, delRes); + code = buildDeleteRange(pOperator, startTs, end, groupId, delRes); + TSDB_CHECK_CODE(code, lino, _end); } else { - buildDeleteRange(pOperator, startTs, endTs, groupId, delRes); + code = buildDeleteRange(pOperator, startTs, endTs, groupId, delRes); + TSDB_CHECK_CODE(code, lino, _end); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; +static int32_t doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStreamFillOperatorInfo* pInfo = pOperator->info; getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup); setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo); SWinKey key = {.ts = startTs, .groupId = groupId}; pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key); if (!pInfo->pFillInfo->needFill) { - buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); + code = buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes); + TSDB_CHECK_CODE(code, lino, _end); } else { STimeRange tw = { .skey = startTs, .ekey = endTs, .groupId = groupId, }; - taosArrayPush(pInfo->pFillInfo->delRanges, &tw); + void* tmp = taosArrayPush(pInfo->pFillInfo->delRanges, &tw); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void getWindowInfoByKey(SStorageAPI* pAPI, void* pState, TSKEY ts, int64_t groupId, SResultRowData* pWinData) { @@ -799,7 +863,8 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) { getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup); TSKEY realEnd = range->ekey + 1; if (pInfo->pFillInfo->type == TSDB_FILL_NEXT && pInfo->pFillSup->next.key != realEnd) { - getWindowInfoByKey(pAPI, pOperator->pTaskInfo->streamInfo.pState, realEnd, range->groupId, &pInfo->pFillSup->next); + getWindowInfoByKey(pAPI, pOperator->pTaskInfo->streamInfo.pState, realEnd, range->groupId, + &pInfo->pFillSup->next); } setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo); pFillInfo->delIndex++; @@ -810,9 +875,10 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) { } } -static void doDeleteFillResult(SOperatorInfo* pOperator) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - +static int32_t doDeleteFillResult(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStreamFillOperatorInfo* pInfo = pOperator->info; SStreamFillInfo* pFillInfo = pInfo->pFillInfo; SSDataBlock* pBlock = pInfo->pSrcDelBlock; @@ -837,7 +903,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { while (pInfo->srcDelRowIndex < pBlock->info.rows) { TSKEY delTs = tsStarts[pInfo->srcDelRowIndex]; uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex]; - int32_t code = TSDB_CODE_SUCCESS; + int32_t winCode = TSDB_CODE_SUCCESS; if (groupId != delGroupId) { break; } @@ -847,10 +913,8 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { SWinKey delKey = {.groupId = delGroupId, .ts = delTs}; if (delTs == nextKey.ts) { - code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); - if (code == TSDB_CODE_SUCCESS) { - code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL); - } + pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur); + winCode = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL); // ts will be deleted later if (delTs != ts) { pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey); @@ -858,7 +922,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey); } endTs = TMAX(delTs, nextKey.ts - 1); - if (code != TSDB_CODE_SUCCESS) { + if (winCode != TSDB_CODE_SUCCESS) { break; } } @@ -866,10 +930,17 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { } pAPI->stateStore.streamStateFreeCur(pCur); - doDeleteFillResultImpl(pOperator, ts, endTs, groupId); + code = doDeleteFillResultImpl(pOperator, ts, endTs, groupId); + TSDB_CHECK_CODE(code, lino, _end); } pFillInfo->current = pFillInfo->end + 1; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { @@ -879,25 +950,41 @@ static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) { pInfo->pFillInfo->delIndex = 0; } -static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) { +static int32_t doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, + SSDataBlock* pDstBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillOperatorInfo* pInfo = pOperator->info; SExprSupp* pSup = &pOperator->exprSupp; blockDataCleanup(pDstBlock); - blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); + code = blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows); + TSDB_CHECK_CODE(code, lino, _end); + setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); - projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + code = projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); pDstBlock->info.rows = 0; pSup = &pInfo->pFillSup->notFillExprSup; setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false); - projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + code = projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); + pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId; - blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); + code = blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -959,7 +1046,9 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { pInfo->srcDelRowIndex = 0; blockDataCleanup(pInfo->pDelRes); pInfo->pFillSup->hasDelete = true; - doDeleteFillResult(pOperator); + code = doDeleteFillResult(pOperator); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pDelRes; @@ -967,9 +1056,11 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { continue; } break; case STREAM_NORMAL: - case STREAM_INVALID: + case STREAM_INVALID: case STREAM_PULL_DATA: { - doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); + code = doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); + TSDB_CHECK_CODE(code, lino, _end); + memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); pInfo->srcRowIndex = -1; } break; @@ -983,7 +1074,9 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { } doStreamFillImpl(pOperator); - doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); + code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo); + TSDB_CHECK_CODE(code, lino, _end); + memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN); pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; if (pInfo->pRes->info.rows > 0) { @@ -1003,6 +1096,14 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pInfo->pRes->info.rows; printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pRes; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + setOperatorCompleted(pOperator); + resetStreamFillInfo(pInfo); + return NULL; } static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { @@ -1026,9 +1127,12 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval, SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter)); if (!pFillSup) { - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } pFillSup->numOfFillCols = numOfFillCols; int32_t numOfNotFillCols = 0; @@ -1040,48 +1144,89 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod pFillSup->interval = *pInterval; pFillSup->pAPI = pAPI; - int32_t code = initResultBuf(pFillSup); - if (code != TSDB_CODE_SUCCESS) { - destroyStreamFillSupporter(pFillSup); - return NULL; - } + code = initResultBuf(pFillSup); + TSDB_CHECK_CODE(code, lino, _end); SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore); - if (code != TSDB_CODE_SUCCESS) { - destroyStreamFillSupporter(pFillSup); - return NULL; - } + TSDB_CHECK_CODE(code, lino, _end); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pFillSup->pResMap = tSimpleHashInit(16, hashFn); pFillSup->hasDelete = false; + +_end: + if (code != TSDB_CODE_SUCCESS) { + destroyStreamFillSupporter(pFillSup); + pFillSup = NULL; + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return pFillSup; } SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo)); + if (!pFillInfo) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + pFillInfo->start = INT64_MIN; pFillInfo->current = INT64_MIN; pFillInfo->end = INT64_MIN; pFillInfo->preRowKey = INT64_MIN; pFillInfo->needFill = false; pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo)); + if (!pFillInfo) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + pFillInfo->pLinearInfo->hasNext = false; pFillInfo->pLinearInfo->nextEnd = INT64_MIN; pFillInfo->pLinearInfo->pEndPoints = NULL; pFillInfo->pLinearInfo->pNextEndPoints = NULL; if (pFillSup->type == TSDB_FILL_LINEAR) { pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); + if (!pFillInfo->pLinearInfo->pEndPoints) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); + if (!pFillInfo->pLinearInfo->pNextEndPoints) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) { SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); - SPoint value = {0}; + SPoint value = {0}; value.val = taosMemoryCalloc(1, pColData->info.bytes); - taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value); + if (!value.val) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + void* tmpRes = taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value); + if (!tmpRes) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } value.val = taosMemoryCalloc(1, pColData->info.bytes); - taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value); + if (!value.val) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + tmpRes = taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value); + if (!tmpRes) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } } pFillInfo->pLinearInfo->winIndex = 0; @@ -1090,8 +1235,18 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_SET_VALUE_F || pFillSup->type == TSDB_FILL_NULL || pFillSup->type == TSDB_FILL_NULL_F) { pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData)); + if (!pFillInfo->pResRow) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + pFillInfo->pResRow->key = INT64_MIN; pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize); + if (!pFillInfo->pResRow->pRowVal) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i); @@ -1102,17 +1257,32 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pFillInfo->type = pFillSup->type; pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange)); + if (!pFillInfo->delRanges) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + pFillInfo->delIndex = 0; pFillInfo->curGroupId = 0; return pFillInfo; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + destroyStreamFillInfo(pFillInfo); + return NULL; } SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; @@ -1120,14 +1290,18 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); if (!pInfo->pFillSup) { - goto _error; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _error); } initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _error); + + code = blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _error); pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes); if (!pInfo->pFillInfo) { @@ -1166,42 +1340,43 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi } pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); - blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); + if (!pInfo->pDelRes) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } + + code = blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _error); pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; int32_t numOfOutputCols = 0; - int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, - &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols, + COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); + TSDB_CHECK_CODE(code, lino, _error); code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); pInfo->srcRowIndex = -1; setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); return pOperator; _error: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } destroyStreamFillOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 96c6712866..1ee6270677 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -171,8 +171,7 @@ static int32_t saveWinResult(SWinKey* pKey, SRowBuffPos* pPos, SSHashObj* pUpdat static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) { SWinKey key = {.ts = ts, .groupId = groupId}; - saveWinResult(&key, pPos, pUpdatedMap); - return TSDB_CODE_SUCCESS; + return saveWinResult(&key, pPos, pUpdatedMap); } static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) { @@ -183,7 +182,8 @@ static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) { if (tmp) { void* value = *(void**)tmp; taosMemoryFree(value); - tSimpleHashRemove(pUpdatedMap, pW, sizeof(SWinKey)); + int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, pW, sizeof(SWinKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } } } @@ -219,20 +219,22 @@ bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) { bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); } -static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) { +static void doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) { SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SWinKey key = {.ts = ts, .groupId = groupId}; - tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); + int32_t tmpRes = tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); pAPI->stateStore.streamStateDel(pInfo->pState, &key); - return true; } static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } -static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins, - SSHashObj* pUpdatedMap, SHashObj* pInvalidWins) { +static int32_t doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins, + SSHashObj* pUpdatedMap, SHashObj* pInvalidWins) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startTsCols = (TSKEY*)pStartTsCol->pData; @@ -269,7 +271,8 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa if (pInvalidWins) { qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, winRes.groupId, childId); - taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0); + code = taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } SArray* chArray = *(void**)chIds; @@ -281,16 +284,26 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa continue; } } - bool res = doDeleteWindow(pOperator, win.skey, winGpId); - if (pUpWins && res) { - taosArrayPush(pUpWins, &winRes); + doDeleteWindow(pOperator, win.skey, winGpId); + if (pUpWins) { + void* tmp = taosArrayPush(pUpWins, &winRes); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } if (pUpdatedMap) { - tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); + int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); } while (win.ekey <= endTsCols[i]); } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SSHashObj* resWins) { @@ -317,6 +330,8 @@ static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp SHashObj* pPullDataMap, SSHashObj* closeWins, SArray* pDelWins, SOperatorInfo* pOperator) { qDebug("===stream===close interval window"); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; void* pIte = NULL; int32_t iter = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; @@ -351,15 +366,19 @@ static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp } if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - int32_t code = saveWinResult(pWinKey, *(SRowBuffPos**)pIte, closeWins); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = saveWinResult(pWinKey, *(SRowBuffPos**)pIte, closeWins); + TSDB_CHECK_CODE(code, lino, _end); } - tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter); + int32_t tmpRes = tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } } - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { @@ -370,22 +389,30 @@ STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWins, int32_t* index, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; blockDataCleanup(pBlock); int32_t size = taosArrayGetSize(pWins); if (*index == size) { *index = 0; taosArrayClear(pWins); - return; + goto _end; } - blockDataEnsureCapacity(pBlock, size - *index); + code = blockDataEnsureCapacity(pBlock, size - *index); + TSDB_CHECK_CODE(code, lino, _end); + uint64_t uid = 0; for (int32_t i = *index; i < size; i++) { SWinKey* pWin = taosArrayGet(pWins, i); void* tbname = NULL; - pInfo->stateStore.streamStateGetParName(pInfo->pState, pWin->groupId, &tbname, false); - if (tbname == NULL) { + int32_t winCode = TSDB_CODE_SUCCESS; + code = pInfo->stateStore.streamStateGetParName(pInfo->pState, pWin->groupId, &tbname, false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL); } else { + ASSERT(tbname); char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName); @@ -393,6 +420,11 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin pInfo->stateStore.streamStateFreeVal(tbname); (*index)++; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void destroyFlusedPos(void* pRes) { @@ -433,11 +465,11 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { taosHashCleanup(pInfo->pPullDataMap); taosHashCleanup(pInfo->pFinalPullDataMap); taosArrayDestroy(pInfo->pPullWins); - blockDataDestroy(pInfo->pPullDataRes); + pInfo->pPullDataRes = blockDataDestroy(pInfo->pPullDataRes); taosArrayDestroy(pInfo->pDelWins); - blockDataDestroy(pInfo->pDelRes); - blockDataDestroy(pInfo->pMidRetriveRes); - blockDataDestroy(pInfo->pMidPulloverRes); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + pInfo->pMidRetriveRes = blockDataDestroy(pInfo->pMidRetriveRes); + pInfo->pMidPulloverRes = blockDataDestroy(pInfo->pMidPulloverRes); pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState); taosArrayDestroy(pInfo->pMidPullDatas); @@ -454,7 +486,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { pInfo->pUpdatedMap = NULL; tSimpleHashCleanup(pInfo->pDeletedMap); - blockDataDestroy(pInfo->pCheckpointRes); + pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); taosMemoryFreeClear(param); } @@ -508,8 +540,10 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt pInfo->basic.primaryPkIndex = pScanInfo->primaryKeyIndex; } -void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput, - SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) { +int32_t compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput, + SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; for (int32_t k = 0; k < numOfOutput; ++k) { if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) { if (!pTimeWindowData) { @@ -525,19 +559,27 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 SScalarParam out = {.columnData = &idata}; SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; - pDestCtx[k].sfp.process(&tw, 1, &out); + code = pDestCtx[k].sfp.process(&tw, 1, &out); + TSDB_CHECK_CODE(code, lino, _end); + pEntryInfo->numOfRes = 1; } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) { - int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]); - if (code != TSDB_CODE_SUCCESS) { - qError("%s apply combine functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code)); - } + code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]); + TSDB_CHECK_CODE(code, lino, _end); } else if (pDestCtx[k].fpSet.combine == NULL) { char* funName = fmGetFuncName(pDestCtx[k].functionId); qError("%s error, combine funcion for %s is not implemented", GET_TASKID(pTaskInfo), funName); taosMemoryFreeClear(funName); + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) { @@ -546,19 +588,28 @@ bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) { int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, - SAggSupporter* pAggSup, SStateStore* pStore) { + SAggSupporter* pAggSup, SStateStore* pStore, int32_t* pWinCode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SWinKey key = {.ts = win->skey, .groupId = groupId}; char* value = NULL; int32_t size = pAggSup->resultRowSize; - int32_t code = pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size); + code = pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size, pWinCode); + TSDB_CHECK_CODE(code, lino, _end); *pResult = (SRowBuffPos*)value; SResultRow* res = (SResultRow*)((*pResult)->pRowBuff); // set time window for current result res->win = (*win); - setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); + code = setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -582,12 +633,24 @@ int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC); } -void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) { +int32_t addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SArray* childIds = taosArrayInit(8, sizeof(int32_t)); for (int32_t i = 0; i < size; i++) { - taosArrayPush(childIds, &i); + void* tmp = taosArrayPush(childIds, &i); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } - taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*)); + code = taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*)); + TSDB_CHECK_CODE(code, lino, _end); +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) { @@ -606,12 +669,16 @@ static void clearSpecialDataBlock(SSDataBlock* pBlock) { } static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; clearSpecialDataBlock(pBlock); int32_t size = taosArrayGetSize(array); if (size - (*pIndex) == 0) { - return; + goto _end; } - blockDataEnsureCapacity(pBlock, size - (*pIndex)); + code = blockDataEnsureCapacity(pBlock, size - (*pIndex)); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); @@ -619,22 +686,40 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); for (; (*pIndex) < size; (*pIndex)++) { SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex)); - colDataSetVal(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); - colDataSetVal(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); - colDataSetVal(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); - colDataSetVal(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false); - colDataSetVal(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false); + code = colDataSetVal(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = colDataSetVal(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = colDataSetVal(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = colDataSetVal(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false); + TSDB_CHECK_CODE(code, lino, _end); + + code = colDataSetVal(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false); + TSDB_CHECK_CODE(code, lino, _end); + pBlock->info.rows++; } if ((*pIndex) == size) { *pIndex = 0; taosArrayClear(array); } - blockDataUpdateTsWindow(pBlock, 0); + code = blockDataUpdateTsWindow(pBlock, 0); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } -static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, - SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) { +static int32_t processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, + SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator, bool* pBeOver) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); TSKEY* tsData = (TSKEY*)pStartCol->pData; @@ -658,13 +743,15 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina if (taosArrayGetSize(chArray) == 0) { // pull data is over taosArrayDestroy(chArray); - taosHashRemove(pMap, &winRes, sizeof(SWinKey)); + int32_t tmpRes = taosHashRemove(pMap, &winRes, sizeof(SWinKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); res = true; qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts); void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey)); if (pFinalCh) { - taosHashRemove(pFinalMap, &winRes, sizeof(SWinKey)); + int32_t tmpRes = taosHashRemove(pFinalMap, &winRes, sizeof(SWinKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); doDeleteWindow(pOperator, winRes.ts, winRes.groupId); STimeWindow nextWin = getFinalTimeWindow(winRes.ts, pInterval); SPullWindowInfo pull = {.window = nextWin, @@ -676,12 +763,25 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina winRes.ts, winRes.groupId, numOfCh); if (IS_MID_INTERVAL_OP(pOperator)) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; - taosArrayPush(pInfo->pMidPullDatas, &winRes); + + void* tmp = taosArrayPush(pInfo->pMidPullDatas, &winRes); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } else if (savePullWindow(&pull, pPullWins) == TSDB_CODE_SUCCESS) { - taosArrayPush(pInfo->pDelWins, &winRes); - addPullWindow(pMap, &winRes, numOfCh); + void* tmp = taosArrayPush(pInfo->pDelWins, &winRes); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = addPullWindow(pMap, &winRes, numOfCh); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); + code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh); } @@ -692,10 +792,20 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina winTs = taosTimeAdd(winTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); } } - return res; + if (pBeOver) { + *pBeOver = res; + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, int32_t childId) { +static int32_t addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, int32_t childId) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); @@ -706,9 +816,12 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { - addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); + code = addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0); + code = tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild); } @@ -718,10 +831,17 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i qDebug("===stream===check final retrive %" PRId64 ",chid:%d", winKey->ts, index); if (index == -1) { qDebug("===stream===add final retrive %" PRId64, winKey->ts); - taosHashPut(pInfo->pFinalPullDataMap, winKey, sizeof(SWinKey), NULL, 0); + code = taosHashPut(pInfo->pFinalPullDataMap, winKey, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void clearFunctionContext(SExprSupp* pSup) { @@ -734,11 +854,12 @@ int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SSta return pStore->streamStateGetByPos(pState, pPos, (void**)pResult); } -int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, - SGroupResInfo* pGroupResInfo) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - +void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, + SGroupResInfo* pGroupResInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; @@ -749,9 +870,9 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i); SResultRow* pRow = NULL; - int32_t code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore); - uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId; - ASSERT(code == 0); + code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore); + TSDB_CHECK_CODE(code, lino, _end); + uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId; doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); // no results, continue to check the next one if (pRow->numOfRows == 0) { @@ -760,9 +881,12 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat } if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = groupId; - void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, - false) < 0) { + void* tbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, + false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + if (winCode != TSDB_CODE_SUCCESS) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -800,7 +924,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + TSDB_CHECK_CODE(code, lino, _end); } } } @@ -809,8 +934,13 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat } pBlock->info.dataLoad = 1; - blockDataUpdateTsWindow(pBlock, 0); - return TSDB_CODE_SUCCESS; + code = blockDataUpdateTsWindow(pBlock, 0); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, @@ -844,6 +974,8 @@ bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkInde static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId, SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); @@ -871,7 +1003,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) { qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64, pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); - blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); + code = blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); + TSDB_CHECK_CODE(code, lino, _end); // timestamp of the data is incorrect if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { @@ -922,9 +1055,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { - addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild); + code = addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey) { - tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); + code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } } } else { @@ -950,13 +1086,13 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat } } - int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, - pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore); + int32_t winCode = TSDB_CODE_SUCCESS; + code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, + pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + pResult = (SResultRow*)pResPos->pRowBuff; - if (pResult == NULL) { - qError("%s set interval output buff error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } + if (IS_FINAL_INTERVAL_OP(pOperator)) { forwardRows = 1; } else { @@ -969,17 +1105,20 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat .groupId = groupId, }; - if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) { - tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0); + if (pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) { + code = tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { - saveWinResult(&key, pResPos, pUpdatedMap); + code = saveWinResult(&key, pResPos, pUpdatedMap); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { pResPos->beUpdated = true; - tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES); + code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1); @@ -1001,6 +1140,10 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat break; } } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } static inline int winPosCmprImpl(const void* pKey1, const void* pKey2) { @@ -1098,15 +1241,27 @@ int32_t encodeSPullWindowInfoArray(void** buf, SArray* pPullInfos) { return tlen; } -void* decodeSPullWindowInfoArray(void* buf, SArray* pPullInfos) { +int32_t decodeSPullWindowInfoArray(void* buf, SArray* pPullInfos, void** ppBuf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t size = 0; buf = taosDecodeFixedI32(buf, &size); for (int32_t i = 0; i < size; i++) { SPullWindowInfo item = {0}; buf = decodeSPullWindowInfo(buf, &item); - taosArrayPush(pPullInfos, &item); + void* tmp = taosArrayPush(pPullInfos, &item); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } - return buf; + (*ppBuf) = buf; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator) { @@ -1167,17 +1322,20 @@ int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pO } void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; if (!pInfo) { - return; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } // 6.checksum int32_t dataLen = len - sizeof(uint32_t); void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { - qError("stream interval state is invalid"); - return; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } // 1.pResultRowHashTable @@ -1188,8 +1346,13 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera buf = decodeSWinKey(buf, &key); SRowBuffPos* pPos = NULL; int32_t resSize = pInfo->aggSup.resultRowSize; - pInfo->stateStore.streamStateAddIfNotExist(pInfo->pState, &key, (void**)&pPos, &resSize); - tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pPos, POINTER_BYTES); + int32_t winCode = TSDB_CODE_SUCCESS; + code = pInfo->stateStore.streamStateAddIfNotExist(pInfo->pState, &key, (void**)&pPos, &resSize, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + ASSERT(winCode == TSDB_CODE_SUCCESS); + + code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pPos, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } // 2.twAggSup @@ -1201,22 +1364,38 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera for (int32_t i = 0; i < size; i++) { SWinKey key = {0}; SArray* pArray = taosArrayInit(0, sizeof(int32_t)); + if (!pArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + buf = decodeSWinKey(buf, &key); int32_t chSize = 0; buf = taosDecodeFixedI32(buf, &chSize); for (int32_t i = 0; i < chSize; i++) { int32_t chId = 0; buf = taosDecodeFixedI32(buf, &chId); - taosArrayPush(pArray, &chId); + void* tmp = taosArrayPush(pArray, &chId); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } - taosHashPut(pInfo->pPullDataMap, &key, sizeof(SWinKey), &pArray, POINTER_BYTES); + code = taosHashPut(pInfo->pPullDataMap, &key, sizeof(SWinKey), &pArray, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } // 4.pPullWins - buf = decodeSPullWindowInfoArray(buf, pInfo->pPullWins); + code = decodeSPullWindowInfoArray(buf, pInfo->pPullWins, &buf); + TSDB_CHECK_CODE(code, lino, _end); // 5.dataVersion buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) { @@ -1233,14 +1412,26 @@ void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) { } } -static void copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) { +static int32_t copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pMap, pIte, &iter)) != NULL) { void* pKey = tSimpleHashGetKey(pIte, NULL); - taosArrayPush(pWins, pKey); + void* tmp = taosArrayPush(pWins, pKey); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } tSimpleHashClear(pMap); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* buildIntervalResult(SOperatorInfo* pOperator) { @@ -1280,18 +1471,31 @@ static SSDataBlock* buildIntervalResult(SOperatorInfo* pOperator) { } int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(*ppWinUpdated, pIte, &iter)) != NULL) { - taosArrayPush(pUpdated, pIte); + void* tmp = taosArrayPush(pUpdated, pIte); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } taosArraySort(pUpdated, compar); tSimpleHashCleanup(*ppWinUpdated); *ppWinUpdated = NULL; - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; @@ -1375,18 +1579,30 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pBlock->info.type == STREAM_CLEAR) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL; - doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); + code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); + TSDB_CHECK_CODE(code, lino, _end); + if (IS_FINAL_INTERVAL_OP(pOperator)) { int32_t chId = getChildIndex(pBlock); - addRetriveWindow(delWins, pInfo, chId); + code = addRetriveWindow(delWins, pInfo, chId); + TSDB_CHECK_CODE(code, lino, _end); + if (pBlock->info.type != STREAM_CLEAR) { - taosArrayAddAll(pInfo->pDelWins, delWins); + void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } taosArrayDestroy(delWins); continue; } removeResults(delWins, pInfo->pUpdatedMap); - taosArrayAddAll(pInfo->pDelWins, delWins); + void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } taosArrayDestroy(delWins); doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); @@ -1404,27 +1620,34 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_INTERVAL_OP(pOperator)) { pInfo->recvGetAll = true; - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); + code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_RETRIEVE) { if (!IS_FINAL_INTERVAL_OP(pOperator)) { pInfo->recvRetrive = true; - copyDataBlock(pInfo->pMidRetriveRes, pBlock); + code = copyDataBlock(pInfo->pMidRetriveRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE; - doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap, NULL); + code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap, NULL); + TSDB_CHECK_CODE(code, lino, _end); break; } continue; } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_INTERVAL_OP(pOperator)) { - processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins, - pInfo->numOfChild, pOperator); + code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins, + pInfo->numOfChild, pOperator, NULL); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; } else if (pBlock->info.type == STREAM_CHECKPOINT) { pAPI->stateStore.streamStateCommit(pInfo->pState); doStreamIntervalSaveCheckpoint(pOperator); - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + continue; } else if (IS_FINAL_INTERVAL_OP(pOperator) && pBlock->info.type == STREAM_MID_RETRIEVE) { continue; @@ -1434,7 +1657,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap, pInfo->pDeletedMap); @@ -1447,19 +1671,24 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); } if (IS_FINAL_INTERVAL_OP(pOperator)) { - closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, - pInfo->pPullDataMap, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); + code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, + pInfo->pPullDataMap, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey) { - copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins); + code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins); + TSDB_CHECK_CODE(code, lino, _end); } } pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; - copyUpdateResult(&pInfo->pUpdatedMap, pInfo->pUpdated, winPosCmprImpl); + code = copyUpdateResult(&pInfo->pUpdatedMap, pInfo->pUpdated, winPosCmprImpl); + TSDB_CHECK_CODE(code, lino, _end); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* resBlock = buildIntervalResult(pOperator); if (resBlock != NULL) { @@ -1472,6 +1701,11 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pMidRetriveRes; } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + setStreamOperatorCompleted(pOperator); return NULL; } @@ -1530,25 +1764,32 @@ static void streamIntervalReleaseState(SOperatorInfo* pOperator) { } void streamIntervalReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL && pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { int32_t size = 0; void* pBuf = NULL; - int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, - strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); - if (code == 0) { - TSKEY ts = *(TSKEY*)pBuf; - taosMemoryFree(pBuf); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); - pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts); - } + code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, + strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); + TSDB_CHECK_CODE(code, lino, _end); + + TSKEY ts = *(TSKEY*)pBuf; + taosMemoryFree(pBuf); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); + pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts); } SOperatorInfo* downstream = pOperator->pDownstream[0]; if (downstream->fpSet.reloadStreamStateFn) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadFromDownStream(downstream, pInfo); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -1622,8 +1863,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo)); pInfo->pullIndex = 0; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); - pInfo->pFinalPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); + pInfo->pPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + pInfo->pFinalPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE); pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->ignoreExpiredDataSaved = false; @@ -1700,7 +1941,7 @@ _error: void destroyStreamAggSupporter(SStreamAggSupporter* pSup) { tSimpleHashCleanup(pSup->pResultRows); destroyDiskbasedBuf(pSup->pResultBuf); - blockDataDestroy(pSup->pScanBlock); + pSup->pScanBlock = blockDataDestroy(pSup->pScanBlock); pSup->stateStore.streamFileStateDestroy(pSup->pState->pFileState); taosMemoryFreeClear(pSup->pState); taosMemoryFreeClear(pSup->pDummyCtx); @@ -1725,14 +1966,14 @@ void destroyStreamSessionAggOperatorInfo(void* param) { } colDataDestroy(&pInfo->twAggSup.timeWindowData); - blockDataDestroy(pInfo->pDelRes); - blockDataDestroy(pInfo->pWinBlock); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + pInfo->pWinBlock = blockDataDestroy(pInfo->pWinBlock); tSimpleHashCleanup(pInfo->pStUpdated); tSimpleHashCleanup(pInfo->pStDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - blockDataDestroy(pInfo->pCheckpointRes); + pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); taosMemoryFreeClear(param); @@ -1869,24 +2110,27 @@ bool inWinRange(STimeWindow* range, STimeWindow* cur) { return false; } -int32_t clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { - return pAPI->streamStateClearBuff(pState, pPos); -} +void clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateClearBuff(pState, pPos); } -void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - SResultWindowInfo* pCurWin) { +int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SResultWindowInfo* pCurWin) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; pCurWin->sessionWin.groupId = groupId; pCurWin->sessionWin.win.skey = startTs; pCurWin->sessionWin.win.ekey = endTs; int32_t size = pAggSup->resultRowSize; - int32_t code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, - pAggSup->gap, (void**)&pCurWin->pStatePos, &size); - if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) { - code = TSDB_CODE_FAILED; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap, + (void**)&pCurWin->pStatePos, &size, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) { + winCode = TSDB_CODE_FAILED; clearOutputBuf(pAggSup->pState, pCurWin->pStatePos, &pAggSup->pSessionAPI->stateStore); } - if (code == TSDB_CODE_SUCCESS) { + if (winCode == TSDB_CODE_SUCCESS) { pCurWin->isOutput = true; if (pCurWin->pStatePos->needFree) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin); @@ -1897,43 +2141,62 @@ void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endT } qDebug("===stream===set session window buff .start:%" PRId64 ",end:%" PRId64 ",groupid:%" PRIu64, pCurWin->sessionWin.win.skey, pCurWin->sessionWin.win.ekey, pCurWin->sessionWin.groupId); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) { +void getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo, + int32_t* pWinCode) { int32_t size = 0; - int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, - (void**)&pWinInfo->pStatePos, &size); - if (code != TSDB_CODE_SUCCESS) { - return code; + (*pWinCode) = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, + (void**)&pWinInfo->pStatePos, &size); + if ((*pWinCode) != TSDB_CODE_SUCCESS) { + return; } pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur); - return TSDB_CODE_SUCCESS; } -void saveDeleteInfo(SArray* pWins, SSessionKey key) { + +int32_t saveDeleteInfo(SArray* pWins, SSessionKey key) { // key.win.ekey = key.win.skey; - taosArrayPush(pWins, &key); -} - -void saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) { - key.win.ekey = key.win.skey; - tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0); -} - -int32_t releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { - pAPI->streamStateReleaseBuf(pState, pPos, false); + void* res = taosArrayPush(pWins, &key); + if (!res) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return TSDB_CODE_OUT_OF_MEMORY; + } return TSDB_CODE_SUCCESS; } +int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) { + key.win.ekey = key.win.skey; + return tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0); +} + +void releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { + pAPI->streamStateReleaseBuf(pState, pPos, false); +} + void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSessionKey key = {0}; getSessionHashKey(pKey, &key); void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey)); if (pVal) { releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore); - tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + code = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + if (code != TSDB_CODE_SUCCESS) { + qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } + } + code = tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); + if (code != TSDB_CODE_SUCCESS) { + qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); } - tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); } void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey) { @@ -1951,7 +2214,8 @@ void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins) { if (!pWin) continue; SSessionKey key = {0}; getSessionHashKey(&pWin->sessionWin, &key); - tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } } @@ -1968,7 +2232,8 @@ void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SAr void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey)); if (pVal) { releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore); - tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } } } @@ -1982,7 +2247,10 @@ int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* } if (pWinInfo->sessionWin.win.skey > pStartTs[i]) { if (pStDeleted && pWinInfo->isOutput) { - saveDeleteRes(pStDeleted, pWinInfo->sessionWin); + int32_t code = saveDeleteRes(pStDeleted, pWinInfo->sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->sessionWin); pWinInfo->sessionWin.win.skey = pStartTs[i]; @@ -2008,32 +2276,39 @@ static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pR int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, SOperatorInfo* pOperator, int64_t winDelta) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - int32_t code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset); - if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset); + TSDB_CHECK_CODE(code, lino, _end); + updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, winDelta); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput); - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { +void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey); SSessionKey hashKey = {0}; getSessionHashKey(pKey, &hashKey); - tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey)); - return true; + int32_t code = tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey)); + if (code != TSDB_CODE_SUCCESS) { + qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } -int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) { +void setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) { void* pVal = tSimpleHashGet(pStUpdated, &pWinInfo->sessionWin, sizeof(SSessionKey)); if (pVal) { SResultWindowInfo* pWin = pVal; pWinInfo->isOutput = pWin->isOutput; } - return TSDB_CODE_SUCCESS; } void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, @@ -2051,14 +2326,20 @@ void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, pAggSup->stateStore.streamStateFreeCur(pCur); } -void compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, - SExecTaskInfo* pTaskInfo, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, - SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { +int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, + SExecTaskInfo* pTaskInfo, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, + SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SResultRow* pCurResult = NULL; int32_t numOfOutput = pSup->numOfExprs; - initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + code = initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + TSDB_CHECK_CODE(code, lino, _end); + SResultRow* pWinResult = NULL; - initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); + code = initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); + TSDB_CHECK_CODE(code, lino, _end); + pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, pNextWin->sessionWin.win.ekey); memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey)); @@ -2067,30 +2348,44 @@ void compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindo winDelta = pAggSup->gap; } updateTimeWindowInfo(&pTwAggSup->timeWindowData, &pCurWin->sessionWin.win, winDelta); - compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pTwAggSup->timeWindowData); - tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey)); + code = compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pTwAggSup->timeWindowData); + TSDB_CHECK_CODE(code, lino, _end); + + int32_t tmpRes = tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey)); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + if (pNextWin->isOutput && pStDeleted) { qDebug("===stream=== save delete window info %" PRId64 ", %" PRIu64, pNextWin->sessionWin.win.skey, pNextWin->sessionWin.groupId); - saveDeleteRes(pStDeleted, pNextWin->sessionWin); + int32_t code = saveDeleteRes(pStDeleted, pNextWin->sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } removeSessionResult(pAggSup, pStUpdated, pAggSup->pResultRows, &pNextWin->sessionWin); doDeleteSessionWindow(pAggSup, &pNextWin->sessionWin); releaseOutputBuf(pAggSup->pState, pNextWin->pStatePos, &pAggSup->pSessionAPI->stateStore); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated, - SSHashObj* pStDeleted, bool addGap) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - int32_t winNum = 0; - + SSHashObj* pStDeleted, bool addGap, int32_t* pWinNum) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t winNum = 0; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SResultRow* pCurResult = NULL; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - // initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + // Just look for the window behind StartIndex while (1) { SResultWindowInfo winInfo = {0}; @@ -2100,10 +2395,20 @@ static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); break; } - compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, &winInfo, pStUpdated, pStDeleted, true); + code = + compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, &winInfo, pStUpdated, pStDeleted, true); + TSDB_CHECK_CODE(code, lino, _end); winNum++; } - return winNum; + if (pWinNum) { + (*pWinNum) = winNum; + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void compactSessionSemiWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin) { @@ -2143,7 +2448,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData SStreamSessionAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.id.groupId; - int64_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SResultRow* pResult = NULL; int32_t rows = pSDataBlock->info.rows; int32_t winRows = 0; @@ -2185,7 +2491,9 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData continue; } SResultWindowInfo winInfo = {0}; - setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo); + code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); + // coverity scan error if (!winInfo.pStatePos) { continue; @@ -2200,36 +2508,42 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator, winDelta); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - qError("%s do stream session aggregate impl error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } - compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap); - saveSessionOutputBuf(pAggSup, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); + + code = compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap, NULL); + TSDB_CHECK_CODE(code, lino, _end); + + code = saveSessionOutputBuf(pAggSup, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->destHasPrimaryKey && winInfo.isOutput && IS_NORMAL_SESSION_OP(pOperator)) { - saveDeleteRes(pInfo->pPkDeleted, winInfo.sessionWin); + code = saveDeleteRes(pInfo->pPkDeleted, winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(winInfo, pStUpdated); - if (code != TSDB_CODE_SUCCESS) { - qError("%s do stream session aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), - tstrerror(code)); - break; - } + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } i += winRows; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } -void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { +int32_t doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -2245,10 +2559,17 @@ void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArr } doDeleteSessionWindow(pAggSup, &curWin); if (result) { - saveDeleteInfo(result, curWin); + code = saveDeleteInfo(result, curWin); + TSDB_CHECK_CODE(code, lino, _end); } } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) { @@ -2273,6 +2594,8 @@ inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) { } void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI; blockDataCleanup(pBlock); @@ -2280,7 +2603,9 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo if (size == 0) { return; } - blockDataEnsureCapacity(pBlock, size); + code = blockDataEnsureCapacity(pBlock, size); + TSDB_CHECK_CODE(code, lino, _end); + int32_t iter = 0; while (((*Ite) = tSimpleHashIterate(pStDeleted, *Ite, &iter)) != NULL) { if (pBlock->info.rows + 1 > pBlock->info.capacity) { @@ -2288,38 +2613,63 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo } SSessionKey* res = tSimpleHashGetKey(*Ite, NULL); SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); + code = colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); + code = colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); colDataSetNULL(pUidCol, pBlock->info.rows); + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - colDataSetVal(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false); + code = colDataSetVal(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); colDataSetNULL(pCalStCol, pBlock->info.rows); + SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); colDataSetNULL(pCalEdCol, pBlock->info.rows); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); + if (!pTableCol) { + TSDB_CHECK_CODE(code, lino, _end); + } - void* tbname = NULL; - pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname, false); - if (tbname == NULL) { + void* tbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, res->groupId, &tbname, false, + &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { colDataSetNULL(pTableCol, pBlock->info.rows); } else { char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); - colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + TSDB_CHECK_CODE(code, lino, _end); pAPI->stateStore.streamStateFreeVal(tbname); } pBlock->info.rows += 1; } + +_end: if ((*Ite) == NULL) { tSimpleHashClear(pStDeleted); } + + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } -static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) { +static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) { + int32_t winCode = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; @@ -2346,23 +2696,22 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS while (1) { SResultWindowInfo childWin = {0}; childWin.sessionWin = *pWinKey; - int32_t code = getSessionWinBuf(pChAggSup, pCur, &childWin); + getSessionWinBuf(pChAggSup, pCur, &childWin, &winCode); - if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &childWin.sessionWin.win)) { + if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &childWin.sessionWin.win)) { releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore); continue; } - if (code == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) { + if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) { if (num == 0) { - setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin); + code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin); + TSDB_CHECK_CODE(code, lino, _end); + parentWin.sessionWin = childWin.sessionWin; memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey)); code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore); - break; - } + TSDB_CHECK_CODE(code, lino, _end); } num++; parentWin.sessionWin.win.skey = TMIN(parentWin.sessionWin.win.skey, childWin.sessionWin.win.skey); @@ -2370,10 +2719,17 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey)); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, pAggSup->gap); - initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput, - pChild->exprSupp.rowEntryInfoOffset); - compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); - compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true); + code = initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput, + pChild->exprSupp.rowEntryInfoOffset); + TSDB_CHECK_CODE(code, lino, _end); + + code = compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, + &pInfo->twAggSup.timeWindowData); + TSDB_CHECK_CODE(code, lino, _end); + + code = compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true, NULL); + TSDB_CHECK_CODE(code, lino, _end); + releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore); } else { releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore); @@ -2383,42 +2739,65 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS pAPI->stateStore.streamStateFreeCur(pCur); } if (num > 0) { - saveResult(parentWin, pStUpdated); - saveSessionOutputBuf(pAggSup, &parentWin); + code = saveResult(parentWin, pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); + + code = saveSessionOutputBuf(pAggSup, &parentWin); + TSDB_CHECK_CODE(code, lino, _end); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } + return code; } int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { SResultWindowInfo* pWinInfo = pIte; if (isCloseWindow(&pWinInfo->sessionWin.win, pTwSup)) { if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) { - int32_t code = saveResult(*pWinInfo, pClosed); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = saveResult(*pWinInfo, pClosed); + TSDB_CHECK_CODE(code, lino, _end); } SSessionKey* pKey = tSimpleHashGetKey(pIte, NULL); - tSimpleHashIterateRemove(pHashMap, pKey, sizeof(SSessionKey), &pIte, &iter); + code = tSimpleHashIterateRemove(pHashMap, pKey, sizeof(SSessionKey), &pIte, &iter); + TSDB_CHECK_CODE(code, lino, _end); } } - return TSDB_CODE_SUCCESS; +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) { +static int32_t closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t size = taosArrayGetSize(pChildren); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); - closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL); + code = closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL); + TSDB_CHECK_CODE(code, lino, _end); } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { +void getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { @@ -2427,20 +2806,32 @@ int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { continue; } pWinInfo->pStatePos->beUpdated = false; - saveResult(*pWinInfo, pStUpdated); + int32_t code = saveResult(*pWinInfo, pStUpdated); + if (code != TSDB_CODE_SUCCESS) { + pWinInfo->pStatePos->beUpdated = true; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } - return TSDB_CODE_SUCCESS; } -void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) { +int32_t copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t size = taosArrayGetSize(pResWins); for (int32_t i = 0; i < size; i++) { SSessionKey* pWinKey = taosArrayGet(pResWins, i); if (!pWinKey) continue; SSessionKey winInfo = {0}; getSessionHashKey(pWinKey, &winInfo); - tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0); + code = tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // the allocated memory comes from outer function. @@ -2453,6 +2844,8 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayL int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; SExprInfo* pExprInfo = pSup->pExprInfo; @@ -2471,9 +2864,13 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa if (pBlock->info.id.groupId == 0) { pBlock->info.id.groupId = pKey->groupId; - void* tbname = NULL; - if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, - false) < 0) { + void* tbname = NULL; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, + &tbname, false, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode != TSDB_CODE_SUCCESS) { pBlock->info.parTbName[0] = 0; } else { memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); @@ -2487,8 +2884,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa } int32_t code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow); - - if (code == -1) { + if (code == TSDB_CODE_FAILED) { // for history qWarn("===stream===not found session result key:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64 "", pKey->win.skey, pKey->win.ekey, pKey->groupId); @@ -2515,11 +2911,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (pCtx[j].fpSet.finalize) { - int32_t code1 = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); - if (TAOS_FAILED(code1)) { - qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code1)); - T_LONG_JMP(pTaskInfo->env, code1); - } + code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + TSDB_CHECK_CODE(code, lino, _end); } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { // do nothing, todo refactor } else { @@ -2528,7 +2921,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + TSDB_CHECK_CODE(code, lino, _end); } } } @@ -2536,11 +2930,19 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa pBlock->info.dataLoad = 1; pBlock->info.rows += pRow->numOfRows; } - blockDataUpdateTsWindow(pBlock, 0); - return TSDB_CODE_SUCCESS; + code = blockDataUpdateTsWindow(pBlock, 0); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // set output datablock version pBlock->info.version = pTaskInfo->version; @@ -2548,15 +2950,22 @@ void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* blockDataCleanup(pBlock); if (!hasRemainResults(pGroupResInfo)) { cleanupGroupResInfo(pGroupResInfo); - return; + goto _end; } // clear the existed group id pBlock->info.id.groupId = 0; - buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); + code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); + TSDB_CHECK_CODE(code, lino, _end); + if (pBlock->info.rows == 0) { cleanupGroupResInfo(pGroupResInfo); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } static SSDataBlock* buildSessionResult(SOperatorInfo* pOperator) { @@ -2578,26 +2987,43 @@ static SSDataBlock* buildSessionResult(SOperatorInfo* pOperator) { return NULL; } -void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) { +int32_t getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t size = taosArrayGetSize(pAllWins); if (size == 0) { - return; + goto _end; } SResultWindowInfo* pWinInfo = taosArrayGet(pAllWins, size - 1); SSessionKey* pSeKey = &pWinInfo->sessionWin; - taosArrayPush(pMaxWins, pSeKey); + void* tmp = taosArrayPush(pMaxWins, pSeKey); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + if (pSeKey->groupId == 0) { - return; + goto _end; } uint64_t preGpId = pSeKey->groupId; for (int32_t i = size - 2; i >= 0; i--) { pWinInfo = taosArrayGet(pAllWins, i); pSeKey = &pWinInfo->sessionWin; if (preGpId != pSeKey->groupId) { - taosArrayPush(pMaxWins, pSeKey); + void* tmp = taosArrayPush(pMaxWins, pSeKey); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } preGpId = pSeKey->groupId; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t encodeSSessionKey(void** buf, SSessionKey* key) { @@ -2674,10 +3100,13 @@ int32_t doStreamSessionEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOp return tlen; } -void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { +int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; if (!pInfo) { - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; @@ -2687,7 +3116,8 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { qError("stream session state is invalid"); - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } } @@ -2698,10 +3128,16 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); - pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap, - (void**)&winfo.pStatePos, &pAggSup->resultRowSize); + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateSessionAddIfNotExist( + pAggSup->pState, &winfo.sessionWin, pAggSup->gap, (void**)&winfo.pStatePos, &pAggSup->resultRowSize, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + ASSERT(winCode == TSDB_CODE_SUCCESS); + buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } // 2.twAggSup @@ -2713,12 +3149,21 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera ASSERT(size <= taosArrayGetSize(pInfo->pChildren)); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i); - buf = doStreamSessionDecodeOpState(buf, 0, pChOp, false); + code = doStreamSessionDecodeOpState(buf, 0, pChOp, false, &buf); + TSDB_CHECK_CODE(code, lino, _end); } // 4.dataVersion buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); - return buf; + if (ppBuf) { + (*ppBuf) = buf; + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) { @@ -2753,12 +3198,17 @@ void copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest) { size_t keyLen = 0; while ((pIte = tSimpleHashIterate(source, pIte, &iter)) != NULL) { SSessionKey* pKey = tSimpleHashGetKey(pIte, &keyLen); - saveDeleteRes(dest, *pKey); + int32_t code = saveDeleteRes(dest, *pKey); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } tSimpleHashClear(source); } static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -2808,19 +3258,27 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); // gap must be 0 - doDeleteTimeWindows(pAggSup, pBlock, pWins); + code = doDeleteTimeWindows(pAggSup, pBlock, pWins); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionResults(pAggSup, pInfo->pStUpdated, pWins); if (IS_FINAL_SESSION_OP(pOperator)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 - doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL); - rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated); + code = doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL); + TSDB_CHECK_CODE(code, lino, _end); + + code = rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); } - copyDeleteWindowInfo(pWins, pInfo->pStDeleted); + code = copyDeleteWindowInfo(pWins, pInfo->pStDeleted); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) { - copyDeleteWindowInfo(pWins, pInfo->pPkDeleted); + code = copyDeleteWindowInfo(pWins, pInfo->pPkDeleted); + TSDB_CHECK_CODE(code, lino, _end); } taosArrayDestroy(pWins); continue; @@ -2833,7 +3291,9 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_CHECKPOINT) { pAggSup->stateStore.streamStateCommit(pAggSup->pState); doStreamSessionSaveCheckpoint(pOperator); - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + continue; } else { ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); @@ -2841,7 +3301,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -2856,9 +3317,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0, NULL); if (!pChildOp) { qError("%s create stream child of final session error", GET_TASKID(pTaskInfo)); - T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } + void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } - taosArrayPush(pInfo->pChildren, &pChildOp); } SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -2870,27 +3336,39 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); - closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); - copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + code = closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); + + code = closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); + TSDB_CHECK_CODE(code, lino, _end); + + code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + if (!pInfo->destHasPrimaryKey) { removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated); } if (pInfo->isHistoryOp) { - getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) { copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted); } initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* opRes = buildSessionResult(pOperator); if (opRes) { return opRes; } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } setStreamOperatorCompleted(pOperator); return NULL; } @@ -2945,6 +3423,8 @@ void reloadAggSupFromDownStream(SOperatorInfo* downstream, SStreamAggSupporter* } void streamSessionSemiReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; resetWinRange(&pAggSup->winRange); @@ -2952,10 +3432,12 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { SResultWindowInfo winInfo = {0}; int32_t size = 0; void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); - int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); - SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; + code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + TSDB_CHECK_CODE(code, lino, _end); + + int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); + SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); for (int32_t i = 0; i < num; i++) { SResultWindowInfo winInfo = {0}; @@ -2964,7 +3446,8 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { continue; } compactSessionSemiWindow(pOperator, &winInfo); - saveSessionOutputBuf(pAggSup, &winInfo); + code = saveSessionOutputBuf(pAggSup, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); } TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY)); taosMemoryFree(pBuf); @@ -2976,17 +3459,27 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void streamSessionReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; resetWinRange(&pAggSup->winRange); - int32_t size = 0; - void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, - strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + int32_t size = 0; + void* pBuf = NULL; + code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, + strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); + + TSDB_CHECK_CODE(code, lino, _end); + int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -3005,22 +3498,36 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { if (!IS_VALID_SESSION_WIN(winInfo)) { continue; } - int32_t winNum = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true); + + int32_t winNum = 0; + code = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true, &winNum); + TSDB_CHECK_CODE(code, lino, _end); + if (winNum > 0) { qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, winInfo.sessionWin.win.skey, winInfo.sessionWin.groupId); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResult(winInfo, pInfo->pStUpdated); + code = saveResult(winInfo, pInfo->pStUpdated); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { if (!isCloseWindow(&winInfo.sessionWin.win, &pInfo->twAggSup)) { - saveDeleteRes(pInfo->pStDeleted, winInfo.sessionWin); + code = saveDeleteRes(pInfo->pStDeleted, winInfo.sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } SSessionKey key = {0}; getSessionHashKey(&winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } } - saveSessionOutputBuf(pAggSup, &winInfo); + code = saveSessionOutputBuf(pAggSup, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); } taosMemoryFree(pBuf); @@ -3029,6 +3536,11 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -3122,8 +3634,9 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME, strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), &buff, &len); if (res == TSDB_CODE_SUCCESS) { - doStreamSessionDecodeOpState(buff, len, pOperator, true); + code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL); taosMemoryFree(buff); + TSDB_CHECK_CODE(code, lino, _error); } } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, @@ -3157,19 +3670,38 @@ static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { pInfo->clearState = false; } -void deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, - SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd) { +int32_t deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, + SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); - doDeleteTimeWindows(pAggSup, pBlock, pWins); + if (!pWins) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + code = doDeleteTimeWindows(pAggSup, pBlock, pWins); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionResults(pAggSup, pMapUpdate, pWins); - copyDeleteWindowInfo(pWins, pMapDelete); + code = copyDeleteWindowInfo(pWins, pMapDelete); + TSDB_CHECK_CODE(code, lino, _end); + if (needAdd) { - copyDeleteWindowInfo(pWins, pPkDelete); + code = copyDeleteWindowInfo(pWins, pPkDelete); + TSDB_CHECK_CODE(code, lino, _end); } taosArrayDestroy(pWins); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; TSKEY maxTs = INT64_MIN; @@ -3228,7 +3760,8 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { // gap must be 0 - deleteSessionWinState(pAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, NULL, false); + code = deleteSessionWinState(pAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, NULL, false); + TSDB_CHECK_CODE(code, lino, _end); pInfo->clearState = true; break; } else if (pBlock->info.type == STREAM_GET_ALL) { @@ -3246,7 +3779,8 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -3257,22 +3791,31 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs; - copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated); if (pInfo->isHistoryOp) { - getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + TSDB_CHECK_CODE(code, lino, _end); } initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* opRes = buildSessionResult(pOperator); if (opRes) { return opRes; } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + clearFunctionContext(&pOperator->exprSupp); // semi session operator clear disk buffer clearStreamSessionOperator(pInfo); @@ -3283,10 +3826,12 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle); if (pOperator == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } SStorageAPI* pAPI = &pTaskInfo->storageAPI; @@ -3307,12 +3852,17 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream for (int32_t i = 0; i < numOfChild; i++) { SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0, pHandle); if (pChildOp == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i, pInfo->primaryTsIndex); - taosArrayPush(pInfo->pChildren, &pChildOp); + void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } void* buff = NULL; @@ -3321,8 +3871,9 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME, strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), &buff, &len); if (res == TSDB_CODE_SUCCESS) { - doStreamSessionDecodeOpState(buff, len, pOperator, true); + code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL); taosMemoryFree(buff); + TSDB_CHECK_CODE(code, lino, _error); } } @@ -3338,6 +3889,9 @@ _error: } taosMemoryFreeClear(pOperator); pTaskInfo->code = code; + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return NULL; } @@ -3359,13 +3913,13 @@ void destroyStreamStateOperatorInfo(void* param) { taosArrayDestroy(pInfo->pChildren); } colDataDestroy(&pInfo->twAggSup.timeWindowData); - blockDataDestroy(pInfo->pDelRes); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); tSimpleHashCleanup(pInfo->pSeUpdated); tSimpleHashCleanup(pInfo->pSeDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - blockDataDestroy(pInfo->pCheckpointRes); + pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); taosMemoryFreeClear(param); @@ -3442,15 +3996,21 @@ void getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SS pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey); } -void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, - SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) { - int32_t size = pAggSup->resultRowSize; +int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, + SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) { + int32_t size = pAggSup->resultRowSize; + SStreamStateCur* pCur = NULL; pCurWin->winInfo.sessionWin.groupId = groupId; pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts; - int32_t code = pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, - pKeyData, pAggSup->stateKeySize, compareStateKey, - (void**)&pCurWin->winInfo.pStatePos, &size); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, + pAggSup->stateKeySize, compareStateKey, + (void**)&pCurWin->winInfo.pStatePos, &size, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + pCurWin->pStateKey = (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize)); pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys); @@ -3458,8 +4018,8 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys); pCurWin->pStateKey->isNull = false; - if (code == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->winInfo.sessionWin.win)) { - code = TSDB_CODE_FAILED; + if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->winInfo.sessionWin.win)) { + winCode = TSDB_CODE_FAILED; clearOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); pCurWin->pStateKey = (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize)); @@ -3474,7 +4034,7 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pCurWin->winInfo.sessionWin.win.ekey); } - if (code == TSDB_CODE_SUCCESS) { + if (winCode == TSDB_CODE_SUCCESS) { pCurWin->winInfo.isOutput = true; if (pCurWin->winInfo.pStatePos->needFree) { pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); @@ -3491,12 +4051,11 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pCurWin->winInfo.sessionWin.win.ekey); pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin; - SStreamStateCur* pCur = - pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin); + pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin); int32_t nextSize = pAggSup->resultRowSize; - code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, - (void**)&pNextWin->winInfo.pStatePos, &nextSize); - if (code != TSDB_CODE_SUCCESS) { + winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, + (void**)&pNextWin->winInfo.pStatePos, &nextSize); + if (winCode != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_INVALID(pNextWin->winInfo); } else { pNextWin->pStateKey = @@ -3507,9 +4066,14 @@ void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, pNextWin->pStateKey->isNull = false; pNextWin->winInfo.isOutput = true; } - pAggSup->stateStore.streamStateFreeCur(pCur); qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64, pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey); +_end: + pAggSup->stateStore.streamStateFreeCur(pCur); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, @@ -3533,7 +4097,10 @@ int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pW if (pWinInfo->winInfo.sessionWin.win.skey > pTs[i]) { if (pSeDeleted && pWinInfo->winInfo.isOutput) { - saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin); + int32_t code = saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } removeSessionResult(pAggSup, pSeUpdated, pResultRows, &pWinInfo->winInfo.sessionWin); pWinInfo->winInfo.sessionWin.win.skey = pTs[i]; @@ -3555,7 +4122,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl SStreamStateAggOperatorInfo* pInfo = pOperator->info; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.id.groupId; - int64_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t winRows = 0; @@ -3569,13 +4137,19 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + if (!pColDataInfo) { + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); + } tsCols = (int64_t*)pColDataInfo->pData; } else { return; } int32_t rows = pSDataBlock->info.rows; - blockDataEnsureCapacity(pAggSup->pScanBlock, rows); + code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows); + TSDB_CHECK_CODE(code, lino, _end); + SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); for (int32_t i = 0; i < rows; i += winRows) { if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, @@ -3589,7 +4163,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl bool allEqual = true; SStateWindowInfo curWin = {0}; SStateWindowInfo nextWin = {0}; - setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); + code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); + TSDB_CHECK_CODE(code, lino, _end); + releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore); setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); @@ -3599,38 +4175,45 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); - tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); + code = tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); + TSDB_CHECK_CODE(code, lino, _end); + doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); continue; } + code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator, 0); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - qError("%s do one window aggregate impl error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } - saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); + + code = saveSessionOutputBuf(pAggSup, &curWin.winInfo); + TSDB_CHECK_CODE(code, lino, _end); if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_STATE_OP(pOperator)) { - saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { code = saveResult(curWin.winInfo, pSeUpdated); - if (code != TSDB_CODE_SUCCESS) { - qError("%s do stream state aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { curWin.winInfo.pStatePos->beUpdated = true; SSessionKey key = {0}; getSessionHashKey(&curWin.winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { @@ -3681,12 +4264,15 @@ int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOper return tlen; } -void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent) { +int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; if (!pInfo) { - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; // 5.checksum if (isParent) { @@ -3694,7 +4280,8 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { qError("stream state_window state is invalid"); - return buf; + code = TSDB_CODE_FAILED; + TSDB_CHECK_CODE(code, lino, _end); } } @@ -3705,11 +4292,16 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); - pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize, - compareStateKey, (void**)&winfo.pStatePos, - &pAggSup->resultRowSize); + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateStateAddIfNotExist( + pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize, compareStateKey, (void**)&winfo.pStatePos, + &pAggSup->resultRowSize, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + code = + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + TSDB_CHECK_CODE(code, lino, _end); } // 2.twAggSup @@ -3721,12 +4313,22 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato ASSERT(size <= taosArrayGetSize(pInfo->pChildren)); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i); - buf = doStreamStateDecodeOpState(buf, 0, pChOp, false); + code = doStreamStateDecodeOpState(buf, 0, pChOp, false, &buf); + TSDB_CHECK_CODE(code, lino, _end); } // 4.dataVersion buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); - return buf; + + if (ppBuf) { + (*ppBuf) = buf; + } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) { @@ -3767,6 +4369,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return NULL; } + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SExprSupp* pSup = &pOperator->exprSupp; SStreamStateAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -3812,7 +4416,9 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { bool add = pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator); - deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted, pInfo->pPkDeleted, add); + code = deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted, + pInfo->pPkDeleted, add); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { pInfo->recvGetAll = true; @@ -3823,7 +4429,9 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_CHECKPOINT) { pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); doStreamStateSaveCheckpoint(pOperator); - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + continue; } else { ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); @@ -3831,7 +4439,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); @@ -3841,12 +4450,17 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); - copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + code = closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); + TSDB_CHECK_CODE(code, lino, _end); + + code = copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + TSDB_CHECK_CODE(code, lino, _end); + removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated); if (pInfo->isHistoryOp) { - getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator)) { copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted); @@ -3854,12 +4468,18 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* resBlock = buildStateResult(pOperator); if (resBlock != NULL) { return resBlock; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } setStreamOperatorCompleted(pOperator); return NULL; } @@ -3883,16 +4503,18 @@ void streamStateReleaseState(SOperatorInfo* pOperator) { } } -static void compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, - SSHashObj* pStUpdated, SSHashObj* pStDeleted) { +static int32_t compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, + SSHashObj* pStUpdated, SSHashObj* pStDeleted) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamStateAggOperatorInfo* pInfo = pOperator->info; - compactTimeWindow(pSup, &pInfo->streamAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, pNextWin, pStUpdated, pStDeleted, - false); + return compactTimeWindow(pSup, &pInfo->streamAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, pNextWin, pStUpdated, + pStDeleted, false); } void streamStateReloadState(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamStateAggOperatorInfo* pInfo = pOperator->info; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; resetWinRange(&pAggSup->winRange); @@ -3900,9 +4522,11 @@ void streamStateReloadState(SOperatorInfo* pOperator) { SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0}; int32_t size = 0; void* pBuf = NULL; - int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, - strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); - int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); + code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, + strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); + TSDB_CHECK_CODE(code, lino, _end); + + int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); qDebug("===stream=== reload state. get result count:%d", num); SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); @@ -3929,25 +4553,38 @@ void streamStateReloadState(SOperatorInfo* pOperator) { qDebug("===stream=== reload state. next window info %" PRId64 ", %" PRIu64 ", compare:%d", nextInfo.winInfo.sessionWin.win.skey, nextInfo.winInfo.sessionWin.groupId, cpRes); if (cpRes) { - compactStateWindow(pOperator, &curInfo.winInfo, &nextInfo.winInfo, pInfo->pSeUpdated, pInfo->pSeDeleted); + code = compactStateWindow(pOperator, &curInfo.winInfo, &nextInfo.winInfo, pInfo->pSeUpdated, pInfo->pSeDeleted); qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, curInfo.winInfo.sessionWin.groupId); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResult(curInfo.winInfo, pInfo->pSeUpdated); + code = saveResult(curInfo.winInfo, pInfo->pSeUpdated); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) { - saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); + code = saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } SSessionKey key = {0}; getSessionHashKey(&curInfo.winInfo.sessionWin, &key); - tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); + code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, + sizeof(SResultWindowInfo)); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } } else if (IS_VALID_SESSION_WIN(nextInfo.winInfo)) { releaseOutputBuf(pAggSup->pState, nextInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); } if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { - saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + TSDB_CHECK_CODE(code, lino, _end); } } taosMemoryFree(pBuf); @@ -3957,6 +4594,11 @@ void streamStateReloadState(SOperatorInfo* pOperator) { downstream->fpSet.reloadStreamStateFn(downstream); } reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -4042,8 +4684,9 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME, strlen(STREAM_STATE_OP_CHECKPOINT_NAME), &buff, &len); if (res == TSDB_CODE_SUCCESS) { - doStreamStateDecodeOpState(buff, len, pOperator, true); + code = doStreamStateDecodeOpState(buff, len, pOperator, true, NULL); taosMemoryFree(buff); + TSDB_CHECK_CODE(code, lino, _error); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, @@ -4079,6 +4722,8 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type #endif static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; @@ -4137,11 +4782,13 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap, NULL); + code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap, NULL); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { pInfo->recvGetAll = true; - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); + code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); @@ -4150,7 +4797,9 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { pAPI->stateStore.streamStateCommit(pInfo->pState); doStreamIntervalSaveCheckpoint(pOperator); pInfo->reCkBlock = true; - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + continue; } else { ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); @@ -4163,7 +4812,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the @@ -4184,26 +4834,42 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (!pInfo->destHasPrimaryKey) { removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); } - closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, - pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); + code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, + pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->destHasPrimaryKey && IS_NORMAL_INTERVAL_OP(pOperator)) { - copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins); + code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins); + TSDB_CHECK_CODE(code, lino, _end); } void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) { - taosArrayPush(pInfo->pUpdated, pIte); + void* tmp = taosArrayPush(pInfo->pUpdated, pIte); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } taosArraySort(pInfo->pUpdated, winPosCmprImpl); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); + tSimpleHashCleanup(pInfo->pUpdatedMap); pInfo->pUpdatedMap = NULL; return buildIntervalResult(pOperator); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + setStreamOperatorCompleted(pOperator); + return NULL; } SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -4338,6 +5004,8 @@ _error: } static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pUpdatedMap) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); @@ -4387,20 +5055,21 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS continue; } - int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, - pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore); + int32_t winCode = TSDB_CODE_SUCCESS; + code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput, + pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore, &winCode); + TSDB_CHECK_CODE(code, lino, _end); + pResult = (SResultRow*)pResPos->pRowBuff; - if (pResult == NULL) { - qError("%s set interval output buff error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - break; - } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveWinResult(&key, pResPos, pUpdatedMap); + code = saveWinResult(&key, pResPos, pUpdatedMap); + TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES); + code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1); @@ -4416,7 +5085,8 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64, pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); - blockDataUpdateTsWindow(pSDataBlock, 0); + code = blockDataUpdateTsWindow(pSDataBlock, 0); + TSDB_CHECK_CODE(code, lino, _end); // timestamp of the data is incorrect if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { @@ -4429,18 +5099,31 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS break; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } -static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) { +static int32_t addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); if (!chIds) { - addPullWindow(pMidPullMap, winKey, numOfChild); + code = addPullWindow(pMidPullMap, winKey, numOfChild); qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild); + TSDB_CHECK_CODE(code, lino, _end); } } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* buildMidIntervalResult(SOperatorInfo* pOperator) { @@ -4472,6 +5155,8 @@ static SSDataBlock* buildMidIntervalResult(SOperatorInfo* pOperator) { } static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; @@ -4542,9 +5227,20 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); - doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, pInfo->pFinalPullDataMap); + if (!delWins) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + code = + doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, pInfo->pFinalPullDataMap); + TSDB_CHECK_CODE(code, lino, _end); + removeResults(delWins, pInfo->pUpdatedMap); - taosArrayAddAll(pInfo->pDelWins, delWins); + void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } taosArrayDestroy(delWins); doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); @@ -4563,10 +5259,14 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; } else if (pBlock->info.type == STREAM_PULL_OVER) { - pInfo->recvPullover = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, - pInfo->pPullWins, pInfo->numOfChild, pOperator); + code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins, + pInfo->numOfChild, pOperator, &pInfo->recvPullover); + TSDB_CHECK_CODE(code, lino, _end); + if (pInfo->recvPullover) { - copyDataBlock(pInfo->pMidPulloverRes, pBlock); + code = copyDataBlock(pInfo->pMidPulloverRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + pInfo->clearState = true; break; } @@ -4574,15 +5274,27 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_CHECKPOINT) { pAPI->stateStore.streamStateCommit(pInfo->pState); doStreamIntervalSaveCheckpoint(pOperator); - copyDataBlock(pInfo->pCheckpointRes, pBlock); + code = copyDataBlock(pInfo->pCheckpointRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + continue; } else if (pBlock->info.type == STREAM_MID_RETRIEVE) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); - doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, NULL); - addMidRetriveWindow(delWins, pInfo->pPullDataMap, pInfo->numOfChild); + if (!delWins) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, NULL); + TSDB_CHECK_CODE(code, lino, _end); + + code = addMidRetriveWindow(delWins, pInfo->pPullDataMap, pInfo->numOfChild); + TSDB_CHECK_CODE(code, lino, _end); + taosArrayDestroy(delWins); pInfo->recvRetrive = true; - copyDataBlock(pInfo->pMidRetriveRes, pBlock); + code = copyDataBlock(pInfo->pMidRetriveRes, pBlock); + TSDB_CHECK_CODE(code, lino, _end); + pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE; pInfo->clearState = true; break; @@ -4592,7 +5304,8 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->scalarSupp.pExprInfo != NULL) { SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + TSDB_CHECK_CODE(code, lino, _end); } setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamMidIntervalAggImpl(pOperator, pBlock, pInfo->pUpdatedMap); @@ -4607,7 +5320,11 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) { - taosArrayPush(pInfo->pUpdated, pIte); + void* tmp = taosArrayPush(pInfo->pUpdated, pIte); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } tSimpleHashCleanup(pInfo->pUpdatedMap); @@ -4616,7 +5333,8 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + TSDB_CHECK_CODE(code, lino, _end); SSDataBlock* resBlock = buildIntervalResult(pOperator); if (resBlock != NULL) { @@ -4633,6 +5351,11 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); clearStreamIntervalOperator(pInfo); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return NULL; } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index f16e187789..2be8024aa8 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -602,7 +602,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; } -int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, +void taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType) { double v1 = -1, v2 = -1; GET_TYPED_DATA(v1, double, inputType, point1->val); @@ -615,8 +615,6 @@ int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* r = (v1 < 1 || v2 < 1) ? 0 : 1; } SET_TYPED_DATA(point->val, outputType, r); - - return TSDB_CODE_SUCCESS; } int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity) { diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index f0647f44a3..c3eeb12209 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -161,11 +161,11 @@ int32_t streamStatePut_rocksdb(SStreamState* pState, const SWinKey* key, const v int32_t streamStateGet_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key); int32_t streamStateClear_rocksdb(SStreamState* pState); -int32_t streamStateCurNext_rocksdb(SStreamState* pState, SStreamStateCur* pCur); +void streamStateCurNext_rocksdb(SStreamStateCur* pCur); int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key); int32_t streamStateGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); -int32_t streamStateCurPrev_rocksdb(SStreamStateCur* pCur); +void streamStateCurPrev_rocksdb(SStreamStateCur* pCur); int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); SStreamStateCur* streamStateGetAndCheckCur_rocksdb(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateSeekKeyNext_rocksdb(SStreamState* pState, const SWinKey* key); @@ -193,7 +193,7 @@ int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSes int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); -int32_t streamStateSessionClear_rocksdb(SStreamState* pState); +void streamStateSessionClear_rocksdb(SStreamState* pState); int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 8b87019ee0..abb5200009 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -3289,12 +3289,10 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { return 0; } -int32_t streamStateCurNext_rocksdb(SStreamState* pState, SStreamStateCur* pCur) { - if (!pCur) { - return -1; +void streamStateCurNext_rocksdb(SStreamStateCur* pCur) { + if (pCur) { + rocksdb_iter_next(pCur->iter); } - rocksdb_iter_next(pCur->iter); - return 0; } int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key) { stDebug("streamStateGetFirst_rocksdb"); @@ -3337,11 +3335,10 @@ int32_t streamStateAddIfNotExist_rocksdb(SStreamState* pState, const SWinKey* ke memset(*pVal, 0, size); return 0; } -int32_t streamStateCurPrev_rocksdb(SStreamStateCur* pCur) { - if (!pCur) return -1; - - rocksdb_iter_prev(pCur->iter); - return 0; +void streamStateCurPrev_rocksdb(SStreamStateCur* pCur) { + if (pCur) { + rocksdb_iter_prev(pCur->iter); + } } int32_t streamStateGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { if (!pCur) return -1; @@ -3990,7 +3987,7 @@ int32_t streamStateSessionGetKeyByRange_rocksdb(SStreamState* pState, const SSes } if (c > 0) { - streamStateCurNext_rocksdb(pState, pCur); + streamStateCurNext_rocksdb(pCur); code = streamStateSessionGetKVByCur_rocksdb(pCur, &resKey, NULL, 0); if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { *curKey = resKey; @@ -4037,7 +4034,7 @@ int32_t streamStateSessionAddIfNotExist_rocksdb(SStreamState* pState, SSessionKe goto _end; } taosMemoryFreeClear(*pVal); - streamStateCurNext_rocksdb(pState, pCur); + streamStateCurNext_rocksdb(pCur); } else { *key = originKey; streamStateFreeCur(pCur); @@ -4063,7 +4060,7 @@ _end: streamStateFreeCur(pCur); return res; } -int32_t streamStateSessionClear_rocksdb(SStreamState* pState) { +void streamStateSessionClear_rocksdb(SStreamState* pState) { stDebug("streamStateSessionClear_rocksdb"); SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext_rocksdb(pState, &key); @@ -4083,10 +4080,9 @@ int32_t streamStateSessionClear_rocksdb(SStreamState* pState) { } taosMemoryFreeClear(buf); - streamStateCurNext_rocksdb(pState, pCur); + streamStateCurNext_rocksdb(pCur); } streamStateFreeCur(pCur); - return -1; } int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { @@ -4114,7 +4110,7 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey* goto _end; } - streamStateCurNext_rocksdb(pState, pCur); + streamStateCurNext_rocksdb(pCur); } else { *key = tmpKey; streamStateFreeCur(pCur); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 0887f9e965..bca9f5cdfb 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -81,21 +81,47 @@ SStreamStateCur* createSessionStateCursor(SStreamFileState* pFileState) { return pCur; } -static SRowBuffPos* addNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey) { +static int32_t addNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, + SRowBuffPos** ppPos) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); ASSERT(pNewPos->pRowBuff); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); - taosArrayPush(pWinInfos, &pNewPos); - return pNewPos; + void* tmp = taosArrayPush(pWinInfos, &pNewPos); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + (*ppPos) = pNewPos; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -static SRowBuffPos* insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, - int32_t index) { +static int32_t insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, + int32_t index, SRowBuffPos** ppPos) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); ASSERT(pNewPos->pRowBuff); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); - taosArrayInsert(pWinInfos, index, &pNewPos); - return pNewPos; + void* tmp = taosArrayInsert(pWinInfos, index, &pNewPos); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + *ppPos = pNewPos; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen) { @@ -103,7 +129,7 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); pNewPos->needFree = true; pNewPos->beFlushed = true; - if(p) { + if (p) { memcpy(pNewPos->pRowBuff, p, *pVLen); } else { int32_t len = getRowStateRowSize(pFileState); @@ -113,9 +139,11 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe return pNewPos; } -int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, - int32_t* pVLen) { - int32_t code = TSDB_CODE_SUCCESS; +int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen, + int32_t* pWinCode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + (*pWinCode) = TSDB_CODE_SUCCESS; SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SArray* pWinStates = NULL; void** ppBuff = tSimpleHashGet(pSessionBuff, &pKey->groupId, sizeof(uint64_t)); @@ -123,7 +151,12 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, pWinStates = (SArray*)(*ppBuff); } else { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (!pWinStates) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + code = tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } TSKEY startTs = pKey->win.skey; @@ -136,12 +169,13 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); - code = code_file; + (*pWinCode) = code_file; qDebug("===stream===0 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file); } else { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pKey); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, pKey, (SRowBuffPos**)pVal); + (*pWinCode) = TSDB_CODE_FAILED; taosMemoryFree(p); + TSDB_CHECK_CODE(code, lino, _end); } goto _end; } @@ -181,7 +215,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); - code = code_file; + (*pWinCode) = code_file; qDebug("===stream===1 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file); goto _end; } else { @@ -191,25 +225,36 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, } if (index == size - 1) { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pKey); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, pKey, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCode) = TSDB_CODE_FAILED; goto _end; } - (*pVal) = insertNewSessionWindow(pFileState, pWinStates, pKey, index + 1); - code = TSDB_CODE_FAILED; + + code = insertNewSessionWindow(pFileState, pWinStates, pKey, index + 1, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCode) = TSDB_CODE_FAILED; _end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } -int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen) { +int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, + int32_t* pWinCode) { SWinKey* pTmpkey = pKey; ASSERT(keyLen == sizeof(SWinKey)); SSessionKey pWinKey = {.groupId = pTmpkey->groupId, .win.skey = pTmpkey->ts, .win.ekey = pTmpkey->ts}; - return getSessionWinResultBuff(pFileState, &pWinKey, 0, pVal, pVLen); + return getSessionWinResultBuff(pFileState, &pWinKey, 0, pVal, pVLen, pWinCode); } int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSessionKey* pKey = pPos->pKey; SArray* pWinStates = NULL; @@ -218,26 +263,47 @@ int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) pWinStates = (SArray*)(*ppBuff); } else { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (!pWinStates) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = tSimpleHashPut(pSessionBuff, &pKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } int32_t size = taosArrayGetSize(pWinStates); if (size == 0) { - taosArrayPush(pWinStates, &pPos); + void* tmp = taosArrayPush(pWinStates, &pPos); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } goto _end; } // find the first position which is smaller than the pKey int32_t index = binarySearch(pWinStates, size, pKey, sessionStateKeyCompare); if (index >= 0) { - taosArrayInsert(pWinStates, index, &pPos); + void* tmp = taosArrayInsert(pWinStates, index, &pPos); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } else { - taosArrayInsert(pWinStates, 0, &pPos); + void* tmp = taosArrayInsert(pWinStates, 0, &pPos); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } _end: pPos->needFree = false; - return TSDB_CODE_SUCCESS; + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { @@ -299,28 +365,42 @@ int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffP int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRowBuffPos* pNewPos = NULL; SSHashObj* pSessionBuff = getRowStateBuff(pFileState); void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); SArray* pWinStates = NULL; if (!ppBuff) { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (!pWinStates) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } else { pWinStates = (SArray*)(*ppBuff); } if (!pCur) { - pNewPos = addNewSessionWindow(pFileState, pWinStates, pWinKey); + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, &pNewPos); + TSDB_CHECK_CODE(code, lino, _end); + goto _end; } int32_t size = taosArrayGetSize(pWinStates); if (pCur->buffIndex >= 0) { if (pCur->buffIndex >= size) { - pNewPos = addNewSessionWindow(pFileState, pWinStates, pWinKey); + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, &pNewPos); + TSDB_CHECK_CODE(code, lino, _end); + goto _end; } - pNewPos = insertNewSessionWindow(pFileState, pWinStates, pWinKey, pCur->buffIndex); + code = insertNewSessionWindow(pFileState, pWinStates, pWinKey, pCur->buffIndex, &pNewPos); + TSDB_CHECK_CODE(code, lino, _end); + goto _end; } else { if (size > 0) { @@ -328,8 +408,10 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream if (sessionWinKeyCmpr(pWinKey, pPos->pKey) >= 0) { // pCur is invalid SSessionKey pTmpKey = *pWinKey; - int32_t code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen); - ASSERT(code == TSDB_CODE_FAILED); + int32_t winCode = TSDB_CODE_SUCCESS; + code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen, &winCode); + ASSERT(winCode == TSDB_CODE_FAILED); + TSDB_CHECK_CODE(code, lino, _end); goto _end; } } @@ -341,7 +423,10 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream _end: (*ppVal) = pNewPos; - return TSDB_CODE_SUCCESS; + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void sessionWinStateClear(SStreamFileState* pFileState) { @@ -484,12 +569,12 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS SArray* pWinStates = NULL; int32_t index = -1; SStreamStateCur* pBuffCur = seekKeyCurrentPrev_buff(pFileState, pWinKey, &pWinStates, &index); - int32_t resSize = getRowStateRowSize(pFileState); - COUNT_TYPE winCount = 0; + int32_t resSize = getRowStateRowSize(pFileState); + COUNT_TYPE winCount = 0; if (pBuffCur) { while (index >= 0) { SRowBuffPos* pPos = taosArrayGetP(pWinStates, index); - winCount = *((COUNT_TYPE*) ((char*)pPos->pRowBuff + (resSize - sizeof(COUNT_TYPE)))); + winCount = *((COUNT_TYPE*)((char*)pPos->pRowBuff + (resSize - sizeof(COUNT_TYPE)))); if (sessionStateRangeKeyCompare(pWinKey, pWinStates, index) == 0 || winCount < count) { index--; } else if (index >= 0) { @@ -503,19 +588,19 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS pBuffCur->buffIndex = 0; } - void* pFileStore = getStateFileStore(pFileState); + void* pFileStore = getStateFileStore(pFileState); SStreamStateCur* pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey); if (pCur) { pCur->pStreamFileState = pFileState; SSessionKey key = {0}; - void* pVal = NULL; - int len = 0; - int32_t code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &len); + void* pVal = NULL; + int len = 0; + int32_t code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &len); if (code == TSDB_CODE_FAILED) { streamStateFreeCur(pCur); return pBuffCur; } - winCount = *((COUNT_TYPE*) ((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); + winCount = *((COUNT_TYPE*)((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); taosMemoryFreeClear(pVal); streamStateFreeCur(pBuffCur); if (sessionRangeKeyCmpr(pWinKey, &key) != 0 && winCount == count) { @@ -529,7 +614,7 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS streamStateCurNext(pFileStore, pCur); return pCur; } - winCount = *((COUNT_TYPE*) ((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); + winCount = *((COUNT_TYPE*)((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); taosMemoryFreeClear(pVal); if (sessionRangeKeyCmpr(pWinKey, &key) == 0 || winCount < count) { streamStateCurPrev(pFileStore, pCur); @@ -590,17 +675,17 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void return code; } -int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur) { +void sessionWinStateMoveToNext(SStreamStateCur* pCur) { qTrace("move cursor to next"); if (pCur && pCur->buffIndex >= 0) { pCur->buffIndex++; } else { - streamStateCurNext_rocksdb(NULL, pCur); + streamStateCurNext_rocksdb(pCur); } - return TSDB_CODE_SUCCESS; } -int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, range_cmpr_fn cmpFn) { +int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, + range_cmpr_fn cmpFn) { SStreamStateCur* pCur = sessionWinStateSeekKeyCurrentPrev(pFileState, key); SSessionKey tmpKey = *key; int32_t code = sessionWinStateGetKVByCur(pCur, &tmpKey, NULL, NULL); @@ -639,18 +724,28 @@ _end: } int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + (*pWinCode) = TSDB_CODE_SUCCESS; + SSessionKey* pWinKey = key; TSKEY gap = 0; - int32_t code = TSDB_CODE_SUCCESS; SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (!pWinStates) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } TSKEY startTs = pWinKey->win.skey; @@ -663,13 +758,14 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch int32_t code_file = streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); - code = code_file; + (*pWinCode) = code_file; qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); } else { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, key); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, key, (SRowBuffPos**)pVal); + (*pWinCode) = TSDB_CODE_FAILED; taosMemoryFree(p); + TSDB_CHECK_CODE(code, lino, _end); } goto _end; } @@ -714,7 +810,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); - code = code_file; + (*pWinCode) = code_file; qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); goto _end; @@ -725,14 +821,21 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch } if (index == size - 1) { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, key); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, key, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCode) = TSDB_CODE_FAILED; goto _end; } - (*pVal) = insertNewSessionWindow(pFileState, pWinStates, key, index + 1); - code = TSDB_CODE_FAILED; + code = insertNewSessionWindow(pFileState, pWinStates, key, index + 1, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCode) = TSDB_CODE_FAILED; _end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -751,18 +854,30 @@ int32_t getCountWinStateFromDisc(SStreamState* pState, SSessionKey* pKey, void** return code; } -int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen) { - SSessionKey* pWinKey = pKey; - const TSKEY gap = 0; +int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, + int32_t* pVLen, int32_t* pWinCount) { int32_t code = TSDB_CODE_SUCCESS; - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + int32_t lino = 0; + (*pWinCount) = TSDB_CODE_SUCCESS; + + SSessionKey* pWinKey = pKey; + const TSKEY gap = 0; + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (!pWinStates) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + + code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + } } TSKEY startTs = pWinKey->win.skey; @@ -772,11 +887,11 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C if (size == 0) { void* pFileStore = getStateFileStore(pFileState); void* pRockVal = NULL; - code = getCountWinStateFromDisc(pFileStore, pWinKey, &pRockVal, pVLen); - if (code == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { + (*pWinCount) = getCountWinStateFromDisc(pFileStore, pWinKey, &pRockVal, pVLen); + if ((*pWinCount) == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, - pWinKey->win.ekey, code); - if (code == TSDB_CODE_SUCCESS) { + pWinKey->win.ekey, (*pWinCount)); + if ((*pWinCount) == TSDB_CODE_SUCCESS) { int32_t valSize = *pVLen; COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)((char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE))); if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) { @@ -789,8 +904,10 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C (*pVal) = createSessionWinBuff(pFileState, pWinKey, NULL, NULL); taosMemoryFree(pRockVal); } else { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCount) = TSDB_CODE_FAILED; } goto _end; } @@ -802,8 +919,8 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C if (index >= 0) { pPos = taosArrayGetP(pWinStates, index); - COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)( (char*)(pPos->pRowBuff) + (valSize - sizeof(COUNT_TYPE)) ); - if (inSessionWindow(pPos->pKey, startTs, gap) || (index == size - 1 && (*pWinStateCout) < winCount) ) { + COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)((char*)(pPos->pRowBuff) + (valSize - sizeof(COUNT_TYPE))); + if (inSessionWindow(pPos->pKey, startTs, gap) || (index == size - 1 && (*pWinStateCout) < winCount)) { (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; @@ -825,7 +942,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C if (tmpKey.win.ekey < pFirstWinKey->win.skey) { *pWinKey = tmpKey; (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); - code = code_file; + (*pWinCount) = code_file; qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); goto _end; @@ -845,25 +962,32 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C goto _end; } - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); + + (*pWinCount) = TSDB_CODE_FAILED; _end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { SSessionKey* pWinKey = pKey; - const TSKEY gap = 0; - int32_t code = TSDB_CODE_SUCCESS; - SSHashObj* pSessionBuff = getRowStateBuff(pFileState); - SArray* pWinStates = NULL; - void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + const TSKEY gap = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + SArray* pWinStates = NULL; + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (ppBuff) { pWinStates = (SArray*)(*ppBuff); } else { pWinStates = taosArrayInit(16, POINTER_BYTES); - tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); } TSKEY startTs = pWinKey->win.skey; @@ -871,24 +995,27 @@ int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey int32_t size = taosArrayGetSize(pWinStates); if (size == 0) { - void* pFileStore = getStateFileStore(pFileState); - void* p = NULL; + void* pFileStore = getStateFileStore(pFileState); + void* p = NULL; int32_t code_file = getCountWinStateFromDisc(pFileStore, pWinKey, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS && isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); - code = code_file; - qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); + qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, + pWinKey->win.ekey, code_file); } else { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); - code = TSDB_CODE_FAILED; + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal); taosMemoryFree(p); + TSDB_CHECK_CODE(code, lino, _end); } - goto _end; } else { - (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey); + code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal); + TSDB_CHECK_CODE(code, lino, _end); } _end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index a817070d5b..7c1d3a8baa 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -99,18 +99,20 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { } SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); stDebug("open stream state %p, %s", pState, path); if (pState == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } pState->pTdbState = taosMemoryCalloc(1, sizeof(STdbState)); if (pState->pTdbState == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; streamStateDestroy(pState, true); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } SStreamTask* pStreamTask = pTask; @@ -118,101 +120,27 @@ SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, i pState->taskId = taskId; sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-0x%x", pState->streamId, pState->taskId); - streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr); + code = streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr); + TSDB_CHECK_CODE(code, lino, _end); -#ifdef USE_ROCKSDB SStreamMeta* pMeta = pStreamTask->pMeta; pState->pTdbState->pOwner = pTask; pState->pFileState = NULL; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT); - pState->parNameMap = tSimpleHashInit(1024, hashFn); + if (!pState->parNameMap) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } stInfo("open state %p on backend %p 0x%" PRIx64 "-%d succ", pState, pMeta->streamBackend, pState->streamId, pState->taskId); return pState; -#else - - char cfgPath[1030]; - sprintf(cfgPath, "%s/cfg", statePath); - - szPage = szPage < 0 ? 4096 : szPage; - pages = pages < 0 ? 256 : pages; - char cfg[1024]; - memset(cfg, 0, 1024); - TdFilePtr pCfgFile = taosOpenFile(cfgPath, TD_FILE_READ); - if (pCfgFile != NULL) { - int64_t size = 0; - taosFStatFile(pCfgFile, &size, NULL); - if (size > 0) { - taosReadFile(pCfgFile, cfg, size); - sscanf(cfg, "%d\n%d\n", &szPage, &pages); - } - } else { - int32_t code = taosMulModeMkDir(statePath, 0755, false); - if (code == 0) { - pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE); - sprintf(cfg, "%d\n%d\n", szPage, pages); - taosWriteFile(pCfgFile, cfg, strlen(cfg)); - } +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - taosCloseFile(&pCfgFile); - - if (tdbOpen(statePath, szPage, pages, &pState->pTdbState->db, 1, 0, NULL) < 0) { - goto _err; - } - - // open state storage backend - if (tdbTbOpen("state.db", sizeof(SStateKey), -1, stateKeyCmpr, pState->pTdbState->db, &pState->pTdbState->pStateDb, - 0) < 0) { - goto _err; - } - - // todo refactor - if (tdbTbOpen("fill.state.db", sizeof(SWinKey), -1, winKeyCmpr, pState->pTdbState->db, - &pState->pTdbState->pFillStateDb, 0) < 0) { - goto _err; - } - - if (tdbTbOpen("session.state.db", sizeof(SStateSessionKey), -1, stateSessionKeyCmpr, pState->pTdbState->db, - &pState->pTdbState->pSessionStateDb, 0) < 0) { - goto _err; - } - - if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->pTdbState->db, - &pState->pTdbState->pFuncStateDb, 0) < 0) { - goto _err; - } - - if (tdbTbOpen("parname.state.db", sizeof(int64_t), TSDB_TABLE_NAME_LEN, NULL, pState->pTdbState->db, - &pState->pTdbState->pParNameDb, 0) < 0) { - goto _err; - } - - if (tdbTbOpen("partag.state.db", sizeof(int64_t), -1, NULL, pState->pTdbState->db, &pState->pTdbState->pParTagDb, 0) < - 0) { - goto _err; - } - - if (streamStateBegin(pState) < 0) { - goto _err; - } - - pState->pTdbState->pOwner = pTask; - - return pState; - -_err: - tdbTbClose(pState->pTdbState->pStateDb); - tdbTbClose(pState->pTdbState->pFuncStateDb); - tdbTbClose(pState->pTdbState->pFillStateDb); - tdbTbClose(pState->pTdbState->pSessionStateDb); - tdbTbClose(pState->pTdbState->pParNameDb); - tdbTbClose(pState->pTdbState->pParTagDb); - tdbClose(pState->pTdbState->db); - streamStateDestroy(pState, false); return NULL; -#endif } int32_t streamStateDelTaskDb(SStreamState* pState) { @@ -223,112 +151,68 @@ int32_t streamStateDelTaskDb(SStreamState* pState) { } void streamStateClose(SStreamState* pState, bool remove) { SStreamTask* pTask = pState->pTdbState->pOwner; -#ifdef USE_ROCKSDB streamStateDestroy(pState, remove); -#else - tdbCommit(pState->pTdbState->db, pState->pTdbState->txn); - tdbPostCommit(pState->pTdbState->db, pState->pTdbState->txn); - tdbTbClose(pState->pTdbState->pStateDb); - tdbTbClose(pState->pTdbState->pFuncStateDb); - tdbTbClose(pState->pTdbState->pFillStateDb); - tdbTbClose(pState->pTdbState->pSessionStateDb); - tdbTbClose(pState->pTdbState->pParNameDb); - tdbTbClose(pState->pTdbState->pParTagDb); - tdbClose(pState->pTdbState->db); -#endif } -int32_t streamStateBegin(SStreamState* pState) { -#ifdef USE_ROCKSDB - return 0; -#else - if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, NULL, NULL, NULL, - TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - tdbAbort(pState->pTdbState->db, pState->pTdbState->txn); - return -1; - } - return 0; -#endif -} +int32_t streamStateBegin(SStreamState* pState) { return 0; } -int32_t streamStateCommit(SStreamState* pState) { -#ifdef USE_ROCKSDB +void streamStateCommit(SStreamState* pState) { if (pState->pFileState) { SStreamSnapshot* pShot = getSnapshot(pState->pFileState); flushSnapshot(pState->pFileState, pShot, true); } - return 0; -#else - if (tdbCommit(pState->pTdbState->db, pState->pTdbState->txn) < 0) { - return -1; - } - if (tdbPostCommit(pState->pTdbState->db, pState->pTdbState->txn) < 0) { - return -1; - } - - if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, NULL, NULL, NULL, - TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - return -1; - } - return 0; -#endif } int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { -#ifdef USE_ROCKSDB - void* pVal = NULL; - int32_t len = getRowStateRowSize(pState->pFileState); - int32_t code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), &pVal, &len); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + void* pVal = NULL; + int32_t len = getRowStateRowSize(pState->pFileState); + code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), &pVal, &len); + TSDB_CHECK_CODE(code, lino, _end); + char* buf = ((SRowBuffPos*)pVal)->pRowBuff; uint32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); memcpy(buf + len - rowSize, value, vLen); - return TSDB_CODE_SUCCESS; -#else - return tdbTbUpsert(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, pState->pTdbState->txn); -#endif + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB - void* pVal = NULL; - int32_t len = getRowStateRowSize(pState->pFileState); - int32_t code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), (void**)(&pVal), &len); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + void* pVal = NULL; + int32_t len = getRowStateRowSize(pState->pFileState); + code = getFunctionRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), (void**)(&pVal), &len); + TSDB_CHECK_CODE(code, lino, _end); + char* buf = ((SRowBuffPos*)pVal)->pRowBuff; uint32_t rowSize = streamFileStateGetSelectRowSize(pState->pFileState); *ppVal = buf + len - rowSize; streamStateReleaseBuf(pState, pVal, false); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; -#else - return tdbTbGet(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), ppVal, pVLen); -#endif } // todo refactor int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { -#ifdef USE_ROCKSDB return 0; // return streamStatePut_rocksdb(pState, key, value, vLen); -#else - SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbUpsert(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, pState->pTdbState->txn); -#endif } -int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB - return getRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), pVal, pVLen); -#else - SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbGet(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), pVal, pVLen); -#endif +int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode) { + return getRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey), pVal, pVLen, pWinCode); } bool streamStateCheck(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB return hasRowBuff(pState->pFileState, (void*)key, sizeof(SWinKey)); -#else - SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbGet(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), NULL, NULL); -#endif } int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal) { @@ -338,330 +222,110 @@ int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal) { } // todo refactor -int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB - return deleteRowBuff(pState->pFileState, key, sizeof(SWinKey)); -#else - SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbDelete(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), pState->pTdbState->txn); -#endif +void streamStateDel(SStreamState* pState, const SWinKey* key) { + deleteRowBuff(pState->pFileState, key, sizeof(SWinKey)); } // todo refactor int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { -#ifdef USE_ROCKSDB return streamStateFillPut_rocksdb(pState, key, value, vLen); -#else - return tdbTbUpsert(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), value, vLen, pState->pTdbState->txn); -#endif } // todo refactor int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return streamStateFillGet_rocksdb(pState, key, pVal, pVLen); -#else - return tdbTbGet(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), pVal, pVLen); -#endif } // todo refactor -int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB - return streamStateFillDel_rocksdb(pState, key); -#else - return tdbTbDelete(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), pState->pTdbState->txn); -#endif +void streamStateFillDel(SStreamState* pState, const SWinKey* key) { + int32_t code = streamStateFillDel_rocksdb(pState, key); + qError("%s at line %d res %d", __func__, __LINE__, code); } -int32_t streamStateClear(SStreamState* pState) { -#ifdef USE_ROCKSDB - streamFileStateClear(pState->pFileState); - return 0; -#else - SWinKey key = {.ts = 0, .groupId = 0}; - streamStatePut(pState, &key, NULL, 0); - while (1) { - SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &key); - SWinKey delKey = {0}; - int32_t code = streamStateGetKVByCur(pCur, &delKey, NULL, 0); - streamStateFreeCur(pCur); - if (code == 0) { - streamStateDel(pState, &delKey); - } else { - break; - } - } - return 0; -#endif -} +void streamStateClear(SStreamState* pState) { streamFileStateClear(pState->pFileState); } void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex) { pState->number = number; pState->tsIndex = tsIdex; } -int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen) { -#ifdef USE_ROCKSDB - int32_t code = 0; +void streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; char* cfName = "default"; void* batch = streamStateCreateBatch(); code = streamStatePutBatch(pState, cfName, batch, pKey, pVal, vLen, 0); - if (code != 0) { - streamStateDestroyBatch(batch); - return code; - } + TSDB_CHECK_CODE(code, lino, _end); + code = streamStatePutBatch_rocksdb(pState, batch); + TSDB_CHECK_CODE(code, lino, _end); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } streamStateDestroyBatch(batch); - return code; -#else - return 0; -#endif } int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen) { -#ifdef USE_ROCKSDB - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; code = streamDefaultGet_rocksdb(pState, pKey, pVal, pLen); return code; -#else - return 0; -#endif } -int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB - return streamStateGet(pState, key, pVal, pVLen); -#else - // todo refactor - int32_t size = *pVLen; - if (streamStateGet(pState, key, pVal, pVLen) == 0) { - return 0; - } - *pVal = tdbRealloc(NULL, size); - memset(*pVal, 0, size); - return 0; -#endif +int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, + int32_t* pWinCode) { + return streamStateGet(pState, key, pVal, pVLen, pWinCode); } -int32_t streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used) { +void streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used) { // todo refactor if (!pVal) { - return 0; + return; } -#ifdef USE_ROCKSDB streamFileStateReleaseBuff(pState->pFileState, pVal, used); -#else - streamStateFreeVal(pVal); -#endif - return 0; } -int32_t streamStateClearBuff(SStreamState* pState, void* pVal) { - return streamFileStateClearBuff(pState->pFileState, pVal); -} +void streamStateClearBuff(SStreamState* pState, void* pVal) { streamFileStateClearBuff(pState->pFileState, pVal); } SStreamStateCur* streamStateFillGetCur(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB return streamStateFillGetCur_rocksdb(pState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) return NULL; - tdbTbcOpen(pState->pTdbState->pFillStateDb, &pCur->pCur, NULL); - - int32_t c = 0; - tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c); - if (c != 0) { - streamStateFreeCur(pCur); - return NULL; - } - return pCur; -#endif } SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key) { -#ifdef USE_ROCKSDB return streamStateGetAndCheckCur_rocksdb(pState, key); -#else - SStreamStateCur* pCur = streamStateFillGetCur(pState, key); - if (pCur) { - int32_t code = streamStateGetGroupKVByCur(pCur, key, NULL, 0); - if (code == 0) { - return pCur; - } - streamStateFreeCur(pCur); - } - return NULL; -#endif } int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return streamStateGetKVByCur_rocksdb(pCur, pKey, pVal, pVLen); -#else - if (!pCur) { - return -1; - } - const SStateKey* pKTmp = NULL; - int32_t kLen; - if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { - return -1; - } - if (pKTmp->opNum != pCur->number) { - return -1; - } - *pKey = pKTmp->key; - return 0; -#endif } int32_t streamStateFillGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return streamStateFillGetKVByCur_rocksdb(pCur, pKey, pVal, pVLen); -#else - if (!pCur) { - return -1; - } - const SWinKey* pKTmp = NULL; - int32_t kLen; - if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { - return -1; - } - *pKey = *pKTmp; - return 0; -#endif } int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return streamStateGetGroupKVByCur_rocksdb(pCur, pKey, pVal, pVLen); -#else - if (!pCur) { - return -1; - } - uint64_t groupId = pKey->groupId; - int32_t code = streamStateFillGetKVByCur(pCur, pKey, pVal, pVLen); - if (code == 0) { - if (pKey->groupId == groupId) { - return 0; - } - } - return -1; -#endif } SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB return streamStateSeekKeyNext_rocksdb(pState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return NULL; - } - pCur->number = pState->number; - if (tdbTbcOpen(pState->pTdbState->pStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - SStateKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - if (c > 0) return pCur; - - if (tdbTbcMoveToNext(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB return streamStateFillSeekKeyNext_rocksdb(pState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (!pCur) { - return NULL; - } - if (tdbTbcOpen(pState->pTdbState->pFillStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - if (c > 0) return pCur; - - if (tdbTbcMoveToNext(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* key) { -#ifdef USE_ROCKSDB return streamStateFillSeekKeyPrev_rocksdb(pState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return NULL; - } - if (tdbTbcOpen(pState->pTdbState->pFillStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - if (c < 0) return pCur; - - if (tdbTbcMoveToPrev(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } -int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur) { -#ifdef USE_ROCKSDB - return sessionWinStateMoveToNext(pCur); -#else - if (!pCur) { - return -1; - } - // - return tdbTbcMoveToNext(pCur->pCur); -#endif -} +void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur) { sessionWinStateMoveToNext(pCur); } -int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur) { -#ifdef USE_ROCKSDB +void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur) { qTrace("move cursor to next"); - return streamStateCurPrev_rocksdb(pCur); -#else - if (!pCur) { - return -1; - } - return tdbTbcMoveToPrev(pCur->pCur); -#endif + streamStateCurPrev_rocksdb(pCur); } void streamStateResetCur(SStreamStateCur* pCur) { @@ -672,8 +336,6 @@ void streamStateResetCur(SStreamStateCur* pCur) { if (pCur->snapshot) rocksdb_release_snapshot(pCur->db, pCur->snapshot); if (pCur->readOpt) rocksdb_readoptions_destroy(pCur->readOpt); - tdbTbcClose(pCur->pCur); - memset(pCur, 0, sizeof(SStreamStateCur)); pCur->buffIndex = -1; @@ -687,16 +349,9 @@ void streamStateFreeCur(SStreamStateCur* pCur) { taosMemoryFree(pCur); } -void streamStateFreeVal(void* val) { -#ifdef USE_ROCKSDB - taosMemoryFree(val); -#else - tdbFree(val); -#endif -} +void streamStateFreeVal(void* val) { taosMemoryFree(val); } int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen) { -#ifdef USE_ROCKSDB int32_t code = TSDB_CODE_SUCCESS; SRowBuffPos* pos = (SRowBuffPos*)value; if (pos->needFree) { @@ -715,438 +370,133 @@ int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void } } return code; -#else - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbUpsert(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, - pState->pTdbState->txn); -#endif } int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur, const SSessionKey* pKey, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return allocSessioncWinBuffByNextPosition(pState->pFileState, pCur, pKey, pVal, pVLen); -#else - return TSDB_CODE_FAILED; -#endif } int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return getSessionFlushedBuff(pState->pFileState, key, pVal, pVLen); -#else - - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, key); - SSessionKey resKey = *key; - void* tmp = NULL; - int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, &tmp, pVLen); - if (code == 0) { - if (key->win.skey != resKey.win.skey) { - code = -1; - } else { - *key = resKey; - *pVal = tdbRealloc(NULL, *pVLen); - memcpy(*pVal, tmp, *pVLen); - } - } - streamStateFreeCur(pCur); - return code; -#endif } -int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { -#ifdef USE_ROCKSDB +void streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { qDebug("===stream===delete skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, key->win.skey, key->win.ekey, key->groupId); - return deleteRowBuff(pState->pFileState, key, sizeof(SSessionKey)); -#else - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbDelete(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), pState->pTdbState->txn); -#endif + deleteRowBuff(pState->pFileState, key, sizeof(SSessionKey)); } -int32_t streamStateSessionReset(SStreamState* pState, void* pVal) { +void streamStateSessionReset(SStreamState* pState, void* pVal) { int32_t len = getRowStateRowSize(pState->pFileState); memset(pVal, 0, len); - return TSDB_CODE_SUCCESS; } SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key) { -#ifdef USE_ROCKSDB return sessionWinStateSeekKeyCurrentPrev(pState->pFileState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return NULL; - } - pCur->number = pState->number; - if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - if (c >= 0) return pCur; - - if (tdbTbcMoveToPrev(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key) { -#ifdef USE_ROCKSDB return sessionWinStateSeekKeyCurrentNext(pState->pFileState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return NULL; - } - pCur->number = pState->number; - if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - if (c <= 0) return pCur; - - if (tdbTbcMoveToNext(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key) { -#ifdef USE_ROCKSDB return sessionWinStateSeekKeyNext(pState->pFileState, key); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return NULL; - } - pCur->number = pState->number; - if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - if (c < 0) return pCur; - - if (tdbTbcMoveToNext(pCur->pCur) < 0) { - streamStateFreeCur(pCur); - return NULL; - } - - return pCur; -#endif } SStreamStateCur* streamStateCountSeekKeyPrev(SStreamState* pState, const SSessionKey* key, COUNT_TYPE count) { -#ifdef USE_ROCKSDB return countWinStateSeekKeyPrev(pState->pFileState, key, count); -#else -#endif } int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen) { -#ifdef USE_ROCKSDB return sessionWinStateGetKVByCur(pCur, pKey, pVal, pVLen); -#else - if (!pCur) { - return -1; - } - SStateSessionKey* pKTmp = NULL; - int32_t kLen; - if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, (const void**)pVal, pVLen) < 0) { - return -1; - } - if (pKTmp->opNum != pCur->number) { - return -1; - } - if (pKey->groupId != 0 && pKey->groupId != pKTmp->key.groupId) { - return -1; - } - *pKey = pKTmp->key; - return 0; -#endif } -int32_t streamStateSessionClear(SStreamState* pState) { -#ifdef USE_ROCKSDB +void streamStateSessionClear(SStreamState* pState) { sessionWinStateClear(pState->pFileState); - return streamStateSessionClear_rocksdb(pState); -#else - SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pState, &key); - while (1) { - SSessionKey delKey = {0}; - void* buf = NULL; - int32_t size = 0; - int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, &buf, &size); - if (code == 0 && size > 0) { - memset(buf, 0, size); - streamStateSessionPut(pState, &delKey, buf, size); - } else { - break; - } - streamStateCurNext(pState, pCur); - } - streamStateFreeCur(pCur); - return 0; -#endif + streamStateSessionClear_rocksdb(pState); } int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { -#ifdef USE_ROCKSDB return sessionWinStateGetKeyByRange(pState->pFileState, key, curKey, sessionRangeKeyCmpr); -#else - SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { - return -1; - } - pCur->number = pState->number; - if (tdbTbcOpen(pState->pTdbState->pSessionStateDb, &pCur->pCur, NULL) < 0) { - streamStateFreeCur(pCur); - return -1; - } - - SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c = 0; - if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { - streamStateFreeCur(pCur); - return -1; - } - - SSessionKey resKey = *key; - int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); - if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { - *curKey = resKey; - streamStateFreeCur(pCur); - return code; - } - - if (c > 0) { - streamStateCurNext(pState, pCur); - code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); - if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { - *curKey = resKey; - streamStateFreeCur(pCur); - return code; - } - } else if (c < 0) { - streamStateCurPrev(pState, pCur); - code = streamStateSessionGetKVByCur(pCur, &resKey, NULL, 0); - if (code == 0 && sessionRangeKeyCmpr(key, &resKey) == 0) { - *curKey = resKey; - streamStateFreeCur(pCur); - return code; - } - } - - streamStateFreeCur(pCur); - return -1; -#endif } int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* key, SSessionKey* curKey) { -#ifdef USE_ROCKSDB return sessionWinStateGetKeyByRange(pState->pFileState, key, curKey, countRangeKeyEqual); -#else -#endif } -int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, - int32_t* pVLen) { -#ifdef USE_ROCKSDB - return getSessionWinResultBuff(pState->pFileState, key, gap, pVal, pVLen); -#else - // todo refactor - int32_t res = 0; - SSessionKey originKey = *key; - SSessionKey searchKey = *key; - searchKey.win.skey = key->win.skey - gap; - searchKey.win.ekey = key->win.ekey + gap; - int32_t valSize = *pVLen; - void* tmp = tdbRealloc(NULL, valSize); - if (!tmp) { - return -1; - } - - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); - int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); - if (code == 0) { - if (sessionRangeKeyCmpr(&searchKey, key) == 0) { - memcpy(tmp, *pVal, valSize); - streamStateSessionDel(pState, key); - goto _end; - } - streamStateCurNext(pState, pCur); - } else { - *key = originKey; - streamStateFreeCur(pCur); - pCur = streamStateSessionSeekKeyNext(pState, key); - } - - code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); - if (code == 0) { - if (sessionRangeKeyCmpr(&searchKey, key) == 0) { - memcpy(tmp, *pVal, valSize); - streamStateSessionDel(pState, key); - goto _end; - } - } - - *key = originKey; - res = 1; - memset(tmp, 0, valSize); - -_end: - - *pVal = tmp; - streamStateFreeCur(pCur); - return res; - -#endif +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen, + int32_t* pWinCode) { + return getSessionWinResultBuff(pState->pFileState, key, gap, pVal, pVLen, pWinCode); } int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode) { // todo refactor - -#ifdef USE_ROCKSDB - return getStateWinResultBuff(pState->pFileState, key, pKeyData, keyDataLen, fn, pVal, pVLen); -#else - int32_t res = 0; - SSessionKey tmpKey = *key; - int32_t valSize = *pVLen; - void* tmp = tdbRealloc(NULL, valSize); - if (!tmp) { - return -1; - } - - SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pState, key); - int32_t code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); - if (code == 0) { - if (key->win.skey <= tmpKey.win.skey && tmpKey.win.ekey <= key->win.ekey) { - memcpy(tmp, *pVal, valSize); - streamStateSessionDel(pState, key); - goto _end; - } - - void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); - if (fn(pKeyData, stateKey) == true) { - memcpy(tmp, *pVal, valSize); - streamStateSessionDel(pState, key); - goto _end; - } - - streamStateCurNext(pState, pCur); - } else { - *key = tmpKey; - streamStateFreeCur(pCur); - pCur = streamStateSessionSeekKeyNext(pState, key); - } - - code = streamStateSessionGetKVByCur(pCur, key, pVal, pVLen); - if (code == 0) { - void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); - if (fn(pKeyData, stateKey) == true) { - memcpy(tmp, *pVal, valSize); - streamStateSessionDel(pState, key); - goto _end; - } - } - - *key = tmpKey; - res = 1; - memset(tmp, 0, valSize); - -_end: - - *pVal = tmp; - streamStateFreeCur(pCur); - return res; -#endif + return getStateWinResultBuff(pState->pFileState, key, pKeyData, keyDataLen, fn, pVal, pVLen, pWinCode); } int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { -#ifdef USE_ROCKSDB + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)) == NULL) { if (tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { - tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN); + code = tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN); + TSDB_CHECK_CODE(code, lino, _end); } - streamStatePutParName_rocksdb(pState, groupId, tbname); + code = streamStatePutParName_rocksdb(pState, groupId, tbname); + TSDB_CHECK_CODE(code, lino, _end); } - return TSDB_CODE_SUCCESS; -#else - return tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, - pState->pTdbState->txn); -#endif + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } -int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache) { -#ifdef USE_ROCKSDB - void* pStr = tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)); +int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + void* pStr = tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)); if (!pStr) { if (onlyCache && tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { - return TSDB_CODE_FAILED; + (*pWinCode) = TSDB_CODE_FAILED; } - int32_t code = streamStateGetParName_rocksdb(pState, groupId, pVal); - if (code == TSDB_CODE_SUCCESS && tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { - tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), *pVal, TSDB_TABLE_NAME_LEN); + (*pWinCode) = streamStateGetParName_rocksdb(pState, groupId, pVal); + if ((*pWinCode) == TSDB_CODE_SUCCESS && tSimpleHashGetSize(pState->parNameMap) < MAX_TABLE_NAME_NUM) { + code = tSimpleHashPut(pState->parNameMap, &groupId, sizeof(int64_t), *pVal, TSDB_TABLE_NAME_LEN); + TSDB_CHECK_CODE(code, lino, _end); } - return code; + goto _end; } *pVal = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); + if (!(*pVal)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + memcpy(*pVal, pStr, TSDB_TABLE_NAME_LEN); - return TSDB_CODE_SUCCESS; -#else - int32_t len; - return tdbTbGet(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), pVal, &len); -#endif + (*pWinCode) = TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void streamStateDestroy(SStreamState* pState, bool remove) { -#ifdef USE_ROCKSDB streamFileStateDestroy(pState->pFileState); // streamStateDestroy_rocksdb(pState, remove); tSimpleHashCleanup(pState->parNameMap); // do nothong -#endif taosMemoryFreeClear(pState->pTdbState); taosMemoryFreeClear(pState); } int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark) { -#ifdef USE_ROCKSDB return deleteExpiredCheckPoint(pState->pFileState, mark); -#else - return 0; -#endif } void streamStateReloadInfo(SStreamState* pState, TSKEY ts) { streamFileStateReloadInfo(pState->pFileState, ts); } @@ -1172,8 +522,9 @@ SStreamStateCur* createStreamStateCursor() { } // count window -int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, int32_t* pVLen) { - return getCountWinResultBuff(pState->pFileState, pKey, winCount, ppVal, pVLen); +int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, + int32_t* pVLen, int32_t* pWinCode) { + return getCountWinResultBuff(pState->pFileState, pKey, winCount, ppVal, pVLen, pWinCode); } int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 61b7e02a4c..e3e07136c6 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -183,7 +183,11 @@ int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, b TSKEY dumy = 0; for (uint64_t i = 0; i < DEFAULT_BUCKET_SIZE; ++i) { - taosArrayPush(pInfo->pTsBuckets, &dumy); + void* tmp = taosArrayPush(pInfo->pTsBuckets, &dumy); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; pInfo->pCloseWinSBF = NULL; @@ -251,7 +255,11 @@ static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) { code = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE, &res); TSDB_CHECK_CODE(code, lino, _end); - taosArrayPush(pInfo->pTsSBFs, &res); + void* tmp = taosArrayPush(pInfo->pTsSBFs, &res); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } (*ppSBf) = res; @@ -543,7 +551,11 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { TSKEY ts = INT64_MIN; for (int32_t i = 0; i < size; i++) { if (tDecodeI64(&decoder, &ts) < 0) return -1; - taosArrayPush(pInfo->pTsBuckets, &ts); + void* tmp = taosArrayPush(pInfo->pTsBuckets, &ts); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } if (tDecodeU64(&decoder, &pInfo->numBuckets) < 0) return -1; @@ -556,7 +568,11 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { code = tScalableBfDecode(&decoder, &pSBf); TSDB_CHECK_CODE(code, lino, _error); - taosArrayPush(pInfo->pTsSBFs, &pSBf); + void* tmp = taosArrayPush(pInfo->pTsSBFs, &pSBf); + if (!tmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } if (tDecodeU64(&decoder, &pInfo->numSBFs) < 0) return -1; diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index fb745f86cb..7f8892e961 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -57,7 +57,6 @@ struct SStreamFileState { _state_file_remove_fn stateFileRemoveFn; _state_file_get_fn stateFileGetFn; - _state_file_clear_fn stateFileClearFn; _state_fun_get_fn stateFunctionGetFn; }; @@ -157,7 +156,6 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ pFileState->stateFileRemoveFn = intervalFileRemoveFn; pFileState->stateFileGetFn = intervalFileGetFn; - pFileState->stateFileClearFn = streamStateClear_rocksdb; pFileState->cfName = taosStrdup("state"); pFileState->stateFunctionGetFn = getRowBuff; } else { @@ -169,7 +167,6 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ pFileState->stateFileRemoveFn = sessionFileRemoveFn; pFileState->stateFileGetFn = sessionFileGetFn; - pFileState->stateFileClearFn = streamStateSessionClear_rocksdb; pFileState->cfName = taosStrdup("sess"); pFileState->stateFunctionGetFn = getSessionRowBuff; } @@ -396,12 +393,10 @@ void* getFreeBuff(SStreamFileState* pFileState) { return ptr; } -int32_t streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { +void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { if (pPos->pRowBuff) { memset(pPos->pRowBuff, 0, pFileState->rowSize); - return TSDB_CODE_SUCCESS; } - return TSDB_CODE_FAILED; } SRowBuffPos* getNewRowPos(SStreamFileState* pFileState) { @@ -441,8 +436,11 @@ SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState) { return newPos; } -int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen) { +int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, + int32_t* pWinCode) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + (*pWinCode) = TSDB_CODE_SUCCESS; pFileState->maxTs = TMAX(pFileState->maxTs, pFileState->getTs(pKey)); SRowBuffPos** pos = tSimpleHashGet(pFileState->rowStateBuff, pKey, keyLen); if (pos) { @@ -450,40 +448,45 @@ int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, voi *pVal = *pos; (*pos)->beUsed = true; (*pos)->beFlushed = false; - return code; + goto _end; } SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); ASSERT(pNewPos->pRowBuff); memcpy(pNewPos->pKey, pKey, keyLen); - code = TSDB_CODE_FAILED; + (*pWinCode) = TSDB_CODE_FAILED; TSKEY ts = pFileState->getTs(pKey); if (!isDeteled(pFileState, ts) && isFlushedState(pFileState, ts, 0)) { int32_t len = 0; void* p = NULL; - code = streamStateGet_rocksdb(pFileState->pFileStore, pKey, &p, &len); - qDebug("===stream===get %" PRId64 " from disc, res %d", ts, code); - if (code == TSDB_CODE_SUCCESS) { + (*pWinCode) = streamStateGet_rocksdb(pFileState->pFileStore, pKey, &p, &len); + qDebug("===stream===get %" PRId64 " from disc, res %d", ts, (*pWinCode)); + if ((*pWinCode) == TSDB_CODE_SUCCESS) { memcpy(pNewPos->pRowBuff, p, len); } taosMemoryFree(p); } - tSimpleHashPut(pFileState->rowStateBuff, pKey, keyLen, &pNewPos, POINTER_BYTES); + code = tSimpleHashPut(pFileState->rowStateBuff, pKey, keyLen, &pNewPos, POINTER_BYTES); + TSDB_CHECK_CODE(code, lino, _end); + if (pVal) { *pVLen = pFileState->rowSize; *pVal = pNewPos; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } -int32_t deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { +void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { int32_t code_buff = pFileState->stateBuffRemoveFn(pFileState->rowStateBuff, pKey, keyLen); + qTrace("%s at line %d res:%s", __func__, __LINE__, code_buff); int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey); - if (code_buff == TSDB_CODE_SUCCESS || code_file == TSDB_CODE_SUCCESS) { - return TSDB_CODE_SUCCESS; - } - return TSDB_CODE_FAILED; + qTrace("%s at line %d res:%s", __func__, __LINE__, code_file); } int32_t resetRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { @@ -549,8 +552,9 @@ SStreamSnapshot* getSnapshot(SStreamFileState* pFileState) { return pFileState->usedBuffs; } -int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState) { +void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SListIter iter = {0}; tdListInitIter(pSnapshot, &iter, TD_LIST_FORWARD); @@ -563,8 +567,17 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, int32_t len = pFileState->rowSize + sizeof(uint64_t) + sizeof(int32_t) + 64; char* buf = taosMemoryCalloc(1, len); + if (!buf) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } void* batch = streamStateCreateBatch(); + if (!batch) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + while ((pNode = tdListNext(&iter)) != NULL && code == TSDB_CODE_SUCCESS) { SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; if (pPos->beFlushed || !pPos->pRowBuff) { @@ -575,14 +588,16 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, qDebug("===stream===flushed start:%" PRId64, pFileState->getTs(pPos->pKey)); if (streamStateGetBatchSize(batch) >= BATCH_LIMIT) { - streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); streamStateClearBatch(batch); + TSDB_CHECK_CODE(code, lino, _end); } void* pSKey = pFileState->stateBuffCreateStateKeyFn(pPos, ((SStreamState*)pFileState->pFileStore)->number); code = streamStatePutBatchOptimize(pFileState->pFileStore, idx, batch, pSKey, pPos->pRowBuff, pFileState->rowSize, 0, buf); taosMemoryFreeClear(pSKey); + TSDB_CHECK_CODE(code, lino, _end); // todo handle failure memset(buf, 0, len); } @@ -590,7 +605,8 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, int32_t numOfElems = streamStateGetBatchSize(batch); if (numOfElems > 0) { - streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + TSDB_CHECK_CODE(code, lino, _end); } else { goto _end; } @@ -606,14 +622,19 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, int32_t len = 0; streamFileStateEncode(&pFileState->flushMark, &valBuf, &len); qDebug("===stream===flushMark write:%" PRId64, pFileState->flushMark); - streamStatePutBatch(pFileState->pFileStore, "default", batch, STREAM_STATE_INFO_NAME, valBuf, len, 0); + code = streamStatePutBatch(pFileState->pFileStore, "default", batch, STREAM_STATE_INFO_NAME, valBuf, len, 0); taosMemoryFree(valBuf); - streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + TSDB_CHECK_CODE(code, lino, _end); + + code = streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); + TSDB_CHECK_CODE(code, lino, _end); } _end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } streamStateDestroyBatch(batch); - return code; } int32_t forceRemoveCheckpoint(SStreamFileState* pFileState, int64_t checkpointId) { @@ -660,27 +681,26 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { ts = atol((char*)buf); if (ts < mark) { // statekey winkey.ts < mark - forceRemoveCheckpoint(pFileState, i); + int32_t tmpRes = forceRemoveCheckpoint(pFileState, i); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); break; } } return code; } -int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { - int code = TSDB_CODE_SUCCESS; +void recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { + int32_t code = TSDB_CODE_SUCCESS; if (pFileState->maxTs != INT64_MIN) { int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) ? INT64_MIN : pFileState->maxTs - pFileState->deleteMark; - deleteExpiredCheckPoint(pFileState, mark); + int32_t tmpRes = deleteExpiredCheckPoint(pFileState, mark); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileState->pFileStore, INT64_MAX); - if (pCur == NULL) { - return -1; - } - int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); + int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); while (code == TSDB_CODE_SUCCESS) { if (pFileState->curRowCount >= recoverNum) { break; @@ -690,31 +710,32 @@ int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { int32_t vlen = 0; SSessionKey key = {0}; code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &vlen); - if (code != 0) { + if (code != TSDB_CODE_SUCCESS) { break; } SRowBuffPos* pPos = createSessionWinBuff(pFileState, &key, pVal, &vlen); - putSessionWinResultBuff(pFileState, pPos); + code = putSessionWinResultBuff(pFileState, pPos); + if (code != TSDB_CODE_SUCCESS) { + break; + } + code = streamStateSessionCurPrev_rocksdb(pCur); } streamStateFreeCur(pCur); - return code; } -int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { +void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { int32_t code = TSDB_CODE_SUCCESS; if (pFileState->maxTs != INT64_MIN) { int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) ? INT64_MIN : pFileState->maxTs - pFileState->deleteMark; - deleteExpiredCheckPoint(pFileState, mark); + int32_t tmpRes = deleteExpiredCheckPoint(pFileState, mark); + qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); } SStreamStateCur* pCur = streamStateSeekToLast_rocksdb(pFileState->pFileStore); - if (pCur == NULL) { - return -1; - } - int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); + int32_t recoverNum = TMIN(MIN_NUM_OF_RECOVER_ROW_BUFF, pFileState->maxRowCount); while (code == TSDB_CODE_SUCCESS) { if (pFileState->curRowCount >= recoverNum) { break; @@ -740,11 +761,9 @@ int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { destroyRowBuffPos(pNewPos); break; } - code = streamStateCurPrev_rocksdb(pCur); + streamStateCurPrev_rocksdb(pCur); } streamStateFreeCur(pCur); - - return TSDB_CODE_SUCCESS; } int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState) { return pFileState->selectivityRowSize; } @@ -759,7 +778,8 @@ void* getRowStateBuff(SStreamFileState* pFileState) { return pFileState->rowStat void* getStateFileStore(SStreamFileState* pFileState) { return pFileState->pFileStore; } bool isDeteled(SStreamFileState* pFileState, TSKEY ts) { - return pFileState->deleteMark != INT64_MAX && pFileState->maxTs > 0 && ts < (pFileState->maxTs - pFileState->deleteMark); + return pFileState->deleteMark != INT64_MAX && pFileState->maxTs > 0 && + ts < (pFileState->maxTs - pFileState->deleteMark); } bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap) { return ts <= (pFileState->flushMark + gap); } @@ -767,5 +787,6 @@ bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap) { return int32_t getRowStateRowSize(SStreamFileState* pFileState) { return pFileState->rowSize; } int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen) { - return pFileState->stateFunctionGetFn(pFileState, pKey, keyLen, pVal, pVLen); + int32_t winCode = TSDB_CODE_SUCCESS; + return pFileState->stateFunctionGetFn(pFileState, pKey, keyLen, pVal, pVLen, &winCode); } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 4a4dfd2653..2e6dfbc96d 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -233,7 +233,11 @@ int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) { SBloomFilter* pBF = NULL; code = tBloomFilterDecode(pDecoder, &pBF); TSDB_CHECK_CODE(code, lino, _error); - taosArrayPush(pSBf->bfArray, &pBF); + void* tmpRes = taosArrayPush(pSBf->bfArray, &pBF); + if (!tmpRes) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); + } } if (tDecodeU32(pDecoder, &pSBf->growth) < 0) { code = TSDB_CODE_FAILED; From d5873aa1b29cd8571a88185bad025545fa7fb8ec Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 09:01:35 +0800 Subject: [PATCH 08/37] adj log --- source/libs/executor/src/streamtimewindowoperator.c | 2 +- source/libs/stream/src/tstreamFileState.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 1ee6270677..60beea0876 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2352,7 +2352,7 @@ int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWi TSDB_CHECK_CODE(code, lino, _end); int32_t tmpRes = tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey)); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); if (pNextWin->isOutput && pStDeleted) { qDebug("===stream=== save delete window info %" PRId64 ", %" PRIu64, pNextWin->sessionWin.win.skey, diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 7f8892e961..f682673dc0 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -484,9 +484,9 @@ _end: void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { int32_t code_buff = pFileState->stateBuffRemoveFn(pFileState->rowStateBuff, pKey, keyLen); - qTrace("%s at line %d res:%s", __func__, __LINE__, code_buff); + qTrace("%s at line %d res:%d", __func__, __LINE__, code_buff); int32_t code_file = pFileState->stateFileRemoveFn(pFileState, pKey); - qTrace("%s at line %d res:%s", __func__, __LINE__, code_file); + qTrace("%s at line %d res:%d", __func__, __LINE__, code_file); } int32_t resetRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen) { @@ -682,7 +682,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { if (ts < mark) { // statekey winkey.ts < mark int32_t tmpRes = forceRemoveCheckpoint(pFileState, i); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); break; } } @@ -696,7 +696,7 @@ void recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { ? INT64_MIN : pFileState->maxTs - pFileState->deleteMark; int32_t tmpRes = deleteExpiredCheckPoint(pFileState, mark); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); } SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileState->pFileStore, INT64_MAX); @@ -731,7 +731,7 @@ void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { ? INT64_MIN : pFileState->maxTs - pFileState->deleteMark; int32_t tmpRes = deleteExpiredCheckPoint(pFileState, mark); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); } SStreamStateCur* pCur = streamStateSeekToLast_rocksdb(pFileState->pFileStore); From d0d73daf7e13b7211af7b808324e6704271f993c Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Thu, 18 Jul 2024 01:13:00 +0000 Subject: [PATCH 09/37] fix invalid read when taosc quit --- source/libs/transport/src/transCli.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 9818394a2a..cf466f3cd9 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -825,18 +825,19 @@ static int32_t allocConnRef(SCliConn* conn, bool update) { transRemoveExHandle(transGetRefMgt(), conn->refId); conn->refId = -1; } - SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); - exh->handle = conn; - exh->pThrd = conn->hostThrd; - QUEUE_INIT(&exh->q); - taosInitRWLatch(&exh->latch); + SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); exh->refId = transAddExHandle(transGetRefMgt(), exh); SExHandle* self = transAcquireExHandle(transGetRefMgt(), exh->refId); - ASSERT(exh == self); + if (self != exh) { + taosMemoryFree(exh); + return TSDB_CODE_REF_INVALID_ID; + } QUEUE_INIT(&exh->q); taosInitRWLatch(&exh->latch); + exh->handle = conn; + exh->pThrd = conn->hostThrd; conn->refId = exh->refId; if (conn->refId == -1) { @@ -2836,6 +2837,10 @@ int64_t transAllocHandle() { exh->refId = transAddExHandle(transGetRefMgt(), exh); SExHandle* self = transAcquireExHandle(transGetRefMgt(), exh->refId); ASSERT(exh == self); + if (exh != self) { + taosMemoryFree(exh); + return TSDB_CODE_REF_INVALID_ID; + } QUEUE_INIT(&exh->q); taosInitRWLatch(&exh->latch); From ab642b245c14876e3881893a5022db851853075e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 10:09:15 +0800 Subject: [PATCH 10/37] adj stream operator result --- include/libs/stream/tstreamFileState.h | 4 +- include/util/tlist.h | 2 +- .../executor/src/streamtimewindowoperator.c | 26 ++- source/libs/stream/src/streamSessionState.c | 98 ++++++++++- source/libs/stream/src/tstreamFileState.c | 154 ++++++++++++++---- source/util/src/tlist.c | 4 +- 6 files changed, 226 insertions(+), 62 deletions(-) diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 7d7001fdf9..699e761f5a 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -32,7 +32,7 @@ typedef SList SStreamSnapshot; typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen); typedef int32_t (*_state_buff_remove_fn)(void* pRowBuff, const void* pKey, size_t keyLen); -typedef int32_t (*_state_buff_remove_by_pos_fn)(SStreamFileState* pState, SRowBuffPos* pPos); +typedef void (*_state_buff_remove_by_pos_fn)(SStreamFileState* pState, SRowBuffPos* pPos); typedef void (*_state_buff_cleanup_fn)(void* pRowBuff); typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num); @@ -83,7 +83,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen); -int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos); +void deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos); int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen); diff --git a/include/util/tlist.h b/include/util/tlist.h index 866a37fee4..73b2ec79e3 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -224,7 +224,7 @@ void tdListInit(SList *list, int32_t eleSize); void tdListEmpty(SList *list); SList *tdListNew(int32_t eleSize); void *tdListFree(SList *list); -void *tdListFreeP(SList *list, FDelete fp); +void tdListFreeP(SList *list, FDelete fp); void tdListPrependNode(SList *list, SListNode *node); void tdListAppendNode(SList *list, SListNode *node); int32_t tdListPrepend(SList *list, void *data); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 60beea0876..d495e53d2e 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -183,7 +183,7 @@ static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) { void* value = *(void**)tmp; taosMemoryFree(value); int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, pW, sizeof(SWinKey)); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); } } } @@ -225,7 +225,7 @@ static void doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) SStreamIntervalOperatorInfo* pInfo = pOperator->info; SWinKey key = {.ts = ts, .groupId = groupId}; int32_t tmpRes = tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); pAPI->stateStore.streamStateDel(pInfo->pState, &key); } @@ -294,7 +294,7 @@ static int32_t doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, S } if (pUpdatedMap) { int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); } getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); } while (win.ekey <= endTsCols[i]); @@ -370,7 +370,7 @@ static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp TSDB_CHECK_CODE(code, lino, _end); } int32_t tmpRes = tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); } } @@ -744,14 +744,14 @@ static int32_t processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pF // pull data is over taosArrayDestroy(chArray); int32_t tmpRes = taosHashRemove(pMap, &winRes, sizeof(SWinKey)); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); res = true; qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts); void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey)); if (pFinalCh) { int32_t tmpRes = taosHashRemove(pFinalMap, &winRes, sizeof(SWinKey)); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); doDeleteWindow(pOperator, winRes.ts, winRes.groupId); STimeWindow nextWin = getFinalTimeWindow(winRes.ts, pInterval); SPullWindowInfo pull = {.window = nextWin, @@ -2215,7 +2215,7 @@ void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins) { SSessionKey key = {0}; getSessionHashKey(&pWin->sessionWin, &key); int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); } } @@ -2233,7 +2233,7 @@ void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SAr if (pVal) { releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore); int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); - qTrace("%s at line %d res:%s", __func__, __LINE__, tmpRes); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); } } } @@ -2883,14 +2883,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa } } - int32_t code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow); - if (code == TSDB_CODE_FAILED) { - // for history - qWarn("===stream===not found session result key:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64 "", - pKey->win.skey, pKey->win.ekey, pKey->groupId); - pGroupResInfo->index += 1; - continue; - } + code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow); + TSDB_CHECK_CODE(code, lino, _end); doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); // no results, continue to check the next one diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index bca9f5cdfb..c3bd2d2422 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -86,7 +86,11 @@ static int32_t addNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInf int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); - ASSERT(pNewPos->pRowBuff); + if (!pNewPos || !pNewPos->pRowBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); void* tmp = taosArrayPush(pWinInfos, &pNewPos); if (!tmp) { @@ -107,7 +111,11 @@ static int32_t insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWin int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); - ASSERT(pNewPos->pRowBuff); + if (!pNewPos || !pNewPos->pRowBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); void* tmp = taosArrayInsert(pWinInfos, index, &pNewPos); if (!tmp) { @@ -125,7 +133,14 @@ _end: } SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); + if (!pNewPos || !pNewPos->pRowBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); pNewPos->needFree = true; pNewPos->beFlushed = true; @@ -135,7 +150,13 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe int32_t len = getRowStateRowSize(pFileState); memset(pNewPos->pRowBuff, 0, len); } + +_end: taosMemoryFree(p); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + return NULL; + } return pNewPos; } @@ -169,6 +190,11 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); + if (!(*pVal)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + (*pWinCode) = code_file; qDebug("===stream===0 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file); } else { @@ -215,6 +241,11 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, int32_t code_file = streamStateSessionAddIfNotExist_rocksdb(pFileStore, pKey, gap, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pKey, p, pVLen); + if (!(*pVal)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + (*pWinCode) = code_file; qDebug("===stream===1 get session win:%" PRId64 ",%" PRId64 " from disc, res %d", startTs, endTs, code_file); goto _end; @@ -307,19 +338,30 @@ _end: } int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); + if (!pNewPos || !pNewPos->pRowBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } pNewPos->needFree = true; pNewPos->beFlushed = true; void* pBuff = NULL; - int32_t code = streamStateSessionGet_rocksdb(getStateFileStore(pFileState), pKey, &pBuff, pVLen); - if (code != TSDB_CODE_SUCCESS) { - return code; + int32_t winCode = streamStateSessionGet_rocksdb(getStateFileStore(pFileState), pKey, &pBuff, pVLen); + if (winCode != TSDB_CODE_SUCCESS) { + return winCode; } memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); memcpy(pNewPos->pRowBuff, pBuff, *pVLen); taosMemoryFreeClear(pBuff); (*pVal) = pNewPos; - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen) { @@ -343,12 +385,12 @@ int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen) return TSDB_CODE_SUCCESS; } -int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos) { +void deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos) { SSHashObj* pSessionBuff = getRowStateBuff(pFileState); SSessionKey* pWinKey = (SSessionKey*)pPos->pKey; void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); if (!ppBuff) { - return TSDB_CODE_SUCCESS; + return; } SArray* pWinStates = (SArray*)(*ppBuff); int32_t size = taosArrayGetSize(pWinStates); @@ -360,7 +402,6 @@ int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffP taosArrayRemove(pWinStates, index); } } - return TSDB_CODE_SUCCESS; } int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, @@ -416,6 +457,11 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream } } pNewPos = getNewRowPosForWrite(pFileState); + if (!pNewPos || !pNewPos->pRowBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + memcpy(pNewPos->pKey, pWinKey, sizeof(SSessionKey)); pNewPos->needFree = true; pNewPos->beFlushed = true; @@ -664,6 +710,11 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void code = TSDB_CODE_SUCCESS; } else if (code == TSDB_CODE_SUCCESS && pVal) { SRowBuffPos* pNewPos = getNewRowPosForWrite(pCur->pStreamFileState); + if (!pNewPos || !pNewPos->pRowBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFreeClear(pData); + return code; + } memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); pNewPos->needFree = true; pNewPos->beFlushed = true; @@ -758,6 +809,11 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch int32_t code_file = streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); + if (!(*pVal)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + (*pWinCode) = code_file; qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); @@ -810,6 +866,11 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch streamStateStateAddIfNotExist_rocksdb(pFileStore, pWinKey, pKeyData, keyDataLen, fn, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); + if (!(*pVal)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + (*pWinCode) = code_file; qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); @@ -896,6 +957,11 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)((char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE))); if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); + if (!(*pVal)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + goto _end; } } @@ -903,6 +969,10 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C pWinKey->win.ekey = endTs; (*pVal) = createSessionWinBuff(pFileState, pWinKey, NULL, NULL); taosMemoryFree(pRockVal); + if (!(*pVal)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } } else { code = addNewSessionWindow(pFileState, pWinStates, pWinKey, (SRowBuffPos**)pVal); TSDB_CHECK_CODE(code, lino, _end); @@ -942,6 +1012,11 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C if (tmpKey.win.ekey < pFirstWinKey->win.skey) { *pWinKey = tmpKey; (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); + if (!(*pVal)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + (*pWinCount) = code_file; qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); @@ -1001,6 +1076,11 @@ int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey int32_t code_file = getCountWinStateFromDisc(pFileStore, pWinKey, &p, pVLen); if (code_file == TSDB_CODE_SUCCESS && isFlushedState(pFileState, endTs, 0)) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); + if (!(*pVal)) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); } else { diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index f682673dc0..970f6e9bfc 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -71,15 +71,15 @@ int32_t stateHashBuffRemoveFn(void* pBuff, const void* pKey, size_t keyLen) { return tSimpleHashRemove(pBuff, pKey, keyLen); } -int32_t stateHashBuffRemoveByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos) { +void stateHashBuffRemoveByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos) { size_t keyLen = pFileState->keyLen; SRowBuffPos** ppPos = tSimpleHashGet(pFileState->rowStateBuff, pPos->pKey, keyLen); if (ppPos) { if ((*ppPos) == pPos) { - return tSimpleHashRemove(pFileState->rowStateBuff, pPos->pKey, keyLen); + int32_t tmpRes = tSimpleHashRemove(pFileState->rowStateBuff, pPos->pKey, keyLen); + qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes); } } - return TSDB_CODE_SUCCESS; } void stateHashBuffClearFn(void* pBuff) { tSimpleHashClear(pBuff); } @@ -276,13 +276,15 @@ void clearExpiredRowBuff(SStreamFileState* pFileState, TSKEY ts, bool all) { pFileState->stateBuffRemoveByPosFn(pFileState, pPos); } destroyRowBuffPos(pPos); - tdListPopNode(pFileState->usedBuffs, pNode); - taosMemoryFreeClear(pNode); + SListNode* tmp = tdListPopNode(pFileState->usedBuffs, pNode); + taosMemoryFreeClear(tmp); } } } -void clearFlushedRowBuff(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uint64_t max) { +int32_t clearFlushedRowBuff(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uint64_t max) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; uint64_t i = 0; SListIter iter = {0}; tdListInitIter(pFileState->usedBuffs, &iter, TD_LIST_FORWARD); @@ -291,16 +293,24 @@ void clearFlushedRowBuff(SStreamFileState* pFileState, SStreamSnapshot* pFlushLi while ((pNode = tdListNext(&iter)) != NULL && i < max) { SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; if (isFlushedState(pFileState, pFileState->getTs(pPos->pKey), 0) && !pPos->beUsed) { - tdListAppend(pFlushList, &pPos); + code = tdListAppend(pFlushList, &pPos); + TSDB_CHECK_CODE(code, lino, _end); + pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey)); pFileState->stateBuffRemoveByPosFn(pFileState, pPos); - tdListPopNode(pFileState->usedBuffs, pNode); - taosMemoryFreeClear(pNode); + SListNode* tmp = tdListPopNode(pFileState->usedBuffs, pNode); + taosMemoryFreeClear(tmp); if (pPos->pRowBuff) { i++; } } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void streamFileStateClear(SStreamFileState* pFileState) { @@ -314,7 +324,9 @@ bool needClearDiskBuff(SStreamFileState* pFileState) { return pFileState->flushM void streamFileStateReleaseBuff(SStreamFileState* pFileState, SRowBuffPos* pPos, bool used) { pPos->beUsed = used; } -void popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uint64_t max, bool used) { +int32_t popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uint64_t max, bool used) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; uint64_t i = 0; SListIter iter = {0}; tdListInitIter(pFileState->usedBuffs, &iter, TD_LIST_FORWARD); @@ -327,11 +339,13 @@ void popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uin ASSERT(pPos->needFree == true); continue; } - tdListAppend(pFlushList, &pPos); + code = tdListAppend(pFlushList, &pPos); + TSDB_CHECK_CODE(code, lino, _end); + pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey)); pFileState->stateBuffRemoveByPosFn(pFileState, pPos); - tdListPopNode(pFileState->usedBuffs, pNode); - taosMemoryFreeClear(pNode); + SListNode* tmp = tdListPopNode(pFileState->usedBuffs, pNode); + taosMemoryFreeClear(tmp); if (pPos->pRowBuff) { i++; } @@ -339,22 +353,35 @@ void popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uin } qInfo("stream state flush %d rows to disk. is used:%d", listNEles(pFlushList), used); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t flushRowBuff(SStreamFileState* pFileState) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamSnapshot* pFlushList = tdListNew(POINTER_BYTES); if (!pFlushList) { - return TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } uint64_t num = (uint64_t)(pFileState->curRowCount * FLUSH_RATIO); num = TMAX(num, FLUSH_NUM); - clearFlushedRowBuff(pFileState, pFlushList, num); + code = clearFlushedRowBuff(pFileState, pFlushList, num); + TSDB_CHECK_CODE(code, lino, _end); + if (isListEmpty(pFlushList)) { - popUsedBuffs(pFileState, pFlushList, num, false); + code = popUsedBuffs(pFileState, pFlushList, num, false); + TSDB_CHECK_CODE(code, lino, _end); if (isListEmpty(pFlushList)) { - popUsedBuffs(pFileState, pFlushList, num, true); + code = popUsedBuffs(pFileState, pFlushList, num, true); + TSDB_CHECK_CODE(code, lino, _end); } } @@ -369,7 +396,12 @@ int32_t flushRowBuff(SStreamFileState* pFileState) { } tdListFreeP(pFlushList, destroyRowBuffPosPtr); - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t clearRowBuff(SStreamFileState* pFileState) { @@ -400,8 +432,20 @@ void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { } SRowBuffPos* getNewRowPos(SStreamFileState* pFileState) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRowBuffPos* pPos = taosMemoryCalloc(1, sizeof(SRowBuffPos)); + if (!pPos) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + pPos->pKey = taosMemoryCalloc(1, pFileState->keyLen); + if (!pPos->pKey) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + void* pBuff = getFreeBuff(pFileState); if (pBuff) { pPos->pRowBuff = pBuff; @@ -417,18 +461,28 @@ SRowBuffPos* getNewRowPos(SStreamFileState* pFileState) { } } - int32_t code = clearRowBuff(pFileState); - ASSERT(code == 0); + code = clearRowBuff(pFileState); + TSDB_CHECK_CODE(code, lino, _end); + pPos->pRowBuff = getFreeBuff(pFileState); + code = tdListAppend(pFileState->usedBuffs, &pPos); + TSDB_CHECK_CODE(code, lino, _end); _end: - tdListAppend(pFileState->usedBuffs, &pPos); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + return NULL; + } + ASSERT(pPos->pRowBuff != NULL); return pPos; } SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState) { SRowBuffPos* newPos = getNewRowPos(pFileState); + if (!newPos) { + return NULL; + } newPos->beUsed = true; newPos->beFlushed = false; newPos->needFree = false; @@ -451,7 +505,11 @@ int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, voi goto _end; } SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); - ASSERT(pNewPos->pRowBuff); + if (!pNewPos || !pNewPos->pRowBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + memcpy(pNewPos->pKey, pKey, keyLen); (*pWinCode) = TSDB_CODE_FAILED; @@ -498,42 +556,65 @@ int32_t resetRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t key return TSDB_CODE_FAILED; } -static void recoverSessionRowBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { +static int32_t recoverSessionRowBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t len = 0; void* pBuff = NULL; - pFileState->stateFileGetFn(pFileState, pPos->pKey, &pBuff, &len); + code = pFileState->stateFileGetFn(pFileState, pPos->pKey, &pBuff, &len); + TSDB_CHECK_CODE(code, lino, _end); memcpy(pPos->pRowBuff, pBuff, len); taosMemoryFree(pBuff); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pPos->pRowBuff) { if (pPos->needFree) { - recoverSessionRowBuff(pFileState, pPos); + code = recoverSessionRowBuff(pFileState, pPos); + TSDB_CHECK_CODE(code, lino, _end); } (*pVal) = pPos->pRowBuff; - return TSDB_CODE_SUCCESS; + goto _end; } pPos->pRowBuff = getFreeBuff(pFileState); if (!pPos->pRowBuff) { if (pFileState->curRowCount < pFileState->maxRowCount) { pPos->pRowBuff = taosMemoryCalloc(1, pFileState->rowSize); + if (!pPos->pRowBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } pFileState->curRowCount++; } else { - int32_t code = clearRowBuff(pFileState); - ASSERT(code == 0); + code = clearRowBuff(pFileState); + TSDB_CHECK_CODE(code, lino, _end); pPos->pRowBuff = getFreeBuff(pFileState); } ASSERT(pPos->pRowBuff); } - recoverSessionRowBuff(pFileState, pPos); + code = recoverSessionRowBuff(pFileState, pPos); + TSDB_CHECK_CODE(code, lino, _end); + (*pVal) = pPos->pRowBuff; if (!pPos->needFree) { - tdListPrepend(pFileState->usedBuffs, &pPos); + code = tdListPrepend(pFileState->usedBuffs, &pPos); } - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen) { @@ -726,6 +807,7 @@ void recoverSesssion(SStreamFileState* pFileState, int64_t ckId) { void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pFileState->maxTs != INT64_MIN) { int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) ? INT64_MIN @@ -744,6 +826,11 @@ void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { void* pVal = NULL; int32_t vlen = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); + if (!pNewPos || !pNewPos->pRowBuff) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); + } + code = streamStateGetKVByCur_rocksdb(pCur, pNewPos->pKey, (const void**)&pVal, &vlen); if (code != TSDB_CODE_SUCCESS || pFileState->getTs(pNewPos->pKey) < pFileState->flushMark) { destroyRowBuffPos(pNewPos); @@ -763,6 +850,11 @@ void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId) { } streamStateCurPrev_rocksdb(pCur); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } streamStateFreeCur(pCur); } diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index 90a2b1baab..d4b7335832 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -55,13 +55,11 @@ void tdListEmptyP(SList *list, FDelete fp) { } } -void *tdListFreeP(SList *list, FDelete fp) { +void tdListFreeP(SList *list, FDelete fp) { if (list) { tdListEmptyP(list, fp); taosMemoryFree(list); } - - return NULL; } void tdListPrependNode(SList *list, SListNode *node) { TD_DLIST_PREPEND(list, node); } From 4e6b898c9a0f31286498c3a7fdc6efb11e77b5a5 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 10:37:14 +0800 Subject: [PATCH 11/37] adj stream operator result --- include/libs/executor/storageapi.h | 2 +- include/libs/stream/streamState.h | 2 +- include/libs/stream/tstreamFileState.h | 5 +- source/libs/executor/inc/executorInt.h | 2 +- .../executor/src/streameventwindowoperator.c | 4 +- .../executor/src/streamtimewindowoperator.c | 53 +++++++++++++------ source/libs/stream/src/streamSessionState.c | 8 +-- source/libs/stream/src/streamState.c | 19 +++++-- source/libs/stream/src/tstreamFileState.c | 29 ++++++++-- 9 files changed, 90 insertions(+), 34 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index e91b7c71ea..8afe55b0a1 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -367,7 +367,7 @@ typedef struct SStateStore { int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen, int32_t* pWinCode); int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); - int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode); void (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); void (*streamStateSessionReset)(SStreamState* pState, void* pVal); void (*streamStateSessionClear)(SStreamState* pState); diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 06da578749..46874b7c65 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -54,7 +54,7 @@ int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, voi int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen, int32_t* pWinCode); int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); -int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode); void streamStateSessionDel(SStreamState* pState, const SSessionKey* key); void streamStateSessionReset(SStreamState* pState, void* pVal); void streamStateSessionClear(SStreamState* pState); diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index 699e761f5a..8c005fa994 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -59,7 +59,7 @@ int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, voi void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen); int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal); bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen); -void putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); +int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); SStreamSnapshot* getSnapshot(SStreamFileState* pFileState); void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState); @@ -81,7 +81,8 @@ int32_t getRowStateRowSize(SStreamFileState* pFileState); int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen, int32_t* pWinCode); int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); -int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); +int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen, + int32_t* pWinCode); int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen); void deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos); int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index c9acead357..ec095a481f 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -955,7 +955,7 @@ void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, void doBuildDeleteDataBlock(struct SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite); void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock); -void getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo); +int32_t getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo); void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin); int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index f79f29dc55..e5f76d4544 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -744,7 +744,9 @@ void streamEventReloadState(SOperatorInfo* pOperator) { SEventWindowInfo curInfo = {0}; qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, i); - getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); + code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); + TSDB_CHECK_CODE(code, lino, _end); + // event window has been deleted if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) { continue; diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index d495e53d2e..5a90207125 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -3389,11 +3389,16 @@ void resetWinRange(STimeWindow* winRange) { winRange->ekey = INT64_MAX; } -void getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo) { +int32_t getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t rowSize = pAggSup->resultRowSize; - int32_t code = - pAggSup->stateStore.streamStateSessionGet(pAggSup->pState, pKey, (void**)&pWinInfo->pStatePos, &rowSize); - if (code == TSDB_CODE_SUCCESS) { + int32_t winCode = TSDB_CODE_SUCCESS; + code = pAggSup->stateStore.streamStateSessionGet(pAggSup->pState, pKey, (void**)&pWinInfo->pStatePos, &rowSize, + &winCode); + TSDB_CHECK_CODE(code, lino, _end); + + if (winCode == TSDB_CODE_SUCCESS) { pWinInfo->sessionWin = *pKey; pWinInfo->isOutput = true; if (pWinInfo->pStatePos->needFree) { @@ -3402,6 +3407,12 @@ void getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, } else { SET_SESSION_WIN_INVALID((*pWinInfo)); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void reloadAggSupFromDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup) { @@ -3435,7 +3446,8 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); for (int32_t i = 0; i < num; i++) { SResultWindowInfo winInfo = {0}; - getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo); + code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); if (!IS_VALID_SESSION_WIN(winInfo)) { continue; } @@ -3488,7 +3500,8 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { } for (int32_t i = 0; i < num; i++) { SResultWindowInfo winInfo = {0}; - getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo); + code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo); + TSDB_CHECK_CODE(code, lino, _end); if (!IS_VALID_SESSION_WIN(winInfo)) { continue; } @@ -3946,14 +3959,19 @@ bool compareWinStateKey(SStateKeys* left, SStateKeys* right) { return compareVal(left->pData, right); } -void getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SStateWindowInfo* pCurWin, - SStateWindowInfo* pNextWin) { - int32_t size = pAggSup->resultRowSize; +int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SStateWindowInfo* pCurWin, + SStateWindowInfo* pNextWin) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SStreamStateCur* pCur = NULL; + int32_t size = pAggSup->resultRowSize; pCurWin->winInfo.sessionWin.groupId = pKey->groupId; pCurWin->winInfo.sessionWin.win.skey = pKey->win.skey; pCurWin->winInfo.sessionWin.win.ekey = pKey->win.ekey; - getSessionWindowInfoByKey(pAggSup, pKey, &pCurWin->winInfo); + code = getSessionWindowInfoByKey(pAggSup, pKey, &pCurWin->winInfo); + TSDB_CHECK_CODE(code, lino, _end); ASSERT(IS_VALID_SESSION_WIN(pCurWin->winInfo)); + pCurWin->pStateKey = (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize)); pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys); @@ -3969,12 +3987,11 @@ void getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SS pCurWin->winInfo.sessionWin.win.ekey); pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin; - SStreamStateCur* pCur = - pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin); + pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin); int32_t nextSize = pAggSup->resultRowSize; - int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, - (void**)&pNextWin->winInfo.pStatePos, &nextSize); - if (code != TSDB_CODE_SUCCESS) { + int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, + (void**)&pNextWin->winInfo.pStatePos, &nextSize); + if (winCode != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_INVALID(pNextWin->winInfo); } else { pNextWin->pStateKey = @@ -3985,9 +4002,15 @@ void getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SS pNextWin->pStateKey->isNull = false; pNextWin->winInfo.isOutput = true; } + +_end: pAggSup->stateStore.streamStateFreeCur(pCur); qDebug("===stream===get state next win buff. skey:%" PRId64 ", endkey:%" PRId64, pNextWin->winInfo.sessionWin.win.skey, pNextWin->winInfo.sessionWin.win.ekey); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index c3bd2d2422..2c30eff832 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -337,7 +337,7 @@ _end: return code; } -int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { +int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen, int32_t* pWinCode) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); @@ -348,9 +348,9 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v pNewPos->needFree = true; pNewPos->beFlushed = true; void* pBuff = NULL; - int32_t winCode = streamStateSessionGet_rocksdb(getStateFileStore(pFileState), pKey, &pBuff, pVLen); - if (winCode != TSDB_CODE_SUCCESS) { - return winCode; + (*pWinCode) = streamStateSessionGet_rocksdb(getStateFileStore(pFileState), pKey, &pBuff, pVLen); + if ((*pWinCode) != TSDB_CODE_SUCCESS) { + goto _end; } memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); memcpy(pNewPos->pRowBuff, pBuff, *pVLen); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 7c1d3a8baa..ce4b1800fb 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -353,22 +353,33 @@ void streamStateFreeVal(void* val) { taosMemoryFree(val); } int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRowBuffPos* pos = (SRowBuffPos*)value; if (pos->needFree) { if (isFlushedState(pState->pFileState, key->win.ekey, 0)) { if (!pos->pRowBuff) { - return code; + goto _end; } code = streamStateSessionPut_rocksdb(pState, key, pos->pRowBuff, vLen); + TSDB_CHECK_CODE(code, lino, _end); + streamStateReleaseBuf(pState, pos, true); - putFreeBuff(pState->pFileState, pos); + code = putFreeBuff(pState->pFileState, pos); + TSDB_CHECK_CODE(code, lino, _end); + stDebug("===stream===save skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64 ".code:%d", key->win.skey, key->win.ekey, key->groupId, code); } else { pos->beFlushed = false; code = putSessionWinResultBuff(pState->pFileState, value); + TSDB_CHECK_CODE(code, lino, _end); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -377,8 +388,8 @@ int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStre return allocSessioncWinBuffByNextPosition(pState->pFileState, pCur, pKey, pVal, pVLen); } -int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { - return getSessionFlushedBuff(pState->pFileState, key, pVal, pVLen); +int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode) { + return getSessionFlushedBuff(pState->pFileState, key, pVal, pVLen, pWinCode); } void streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 970f6e9bfc..76d3f5e795 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -124,7 +124,8 @@ static void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) { *pLen = sizeof(TSKEY); (*pVal) = taosMemoryCalloc(1, *pLen); void* buff = *pVal; - taosEncodeFixedI64(&buff, *pKey); + int32_t tmp = taosEncodeFixedI64(&buff, *pKey); + ASSERT(tmp == sizeof(TSKEY)); } SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, @@ -255,14 +256,25 @@ void streamFileStateDestroy(SStreamFileState* pFileState) { taosMemoryFree(pFileState); } -void putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { +int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pPos->pRowBuff) { - tdListAppend(pFileState->freeBuffs, &(pPos->pRowBuff)); + code = tdListAppend(pFileState->freeBuffs, &(pPos->pRowBuff)); + TSDB_CHECK_CODE(code, lino, _end); pPos->pRowBuff = NULL; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void clearExpiredRowBuff(SStreamFileState* pFileState, TSKEY ts, bool all) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SListIter iter = {0}; tdListInitIter(pFileState->usedBuffs, &iter, TD_LIST_FORWARD); @@ -270,7 +282,8 @@ void clearExpiredRowBuff(SStreamFileState* pFileState, TSKEY ts, bool all) { while ((pNode = tdListNext(&iter)) != NULL) { SRowBuffPos* pPos = *(SRowBuffPos**)(pNode->data); if (all || (pFileState->getTs(pPos->pKey) < ts && !pPos->beUsed)) { - putFreeBuff(pFileState, pPos); + code = putFreeBuff(pFileState, pPos); + TSDB_CHECK_CODE(code, lino, _end); if (!all) { pFileState->stateBuffRemoveByPosFn(pFileState, pPos); @@ -280,6 +293,11 @@ void clearExpiredRowBuff(SStreamFileState* pFileState, TSKEY ts, bool all) { taosMemoryFreeClear(tmp); } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } int32_t clearFlushedRowBuff(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uint64_t max) { @@ -392,7 +410,8 @@ int32_t flushRowBuff(SStreamFileState* pFileState) { SListNode* pNode = NULL; while ((pNode = tdListNext(&fIter)) != NULL) { SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; - putFreeBuff(pFileState, pPos); + code = putFreeBuff(pFileState, pPos); + TSDB_CHECK_CODE(code, lino, _end); } tdListFreeP(pFlushList, destroyRowBuffPosPtr); From 99f4484328a0880d2407883fe6c33be98d41fd73 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 11:26:50 +0800 Subject: [PATCH 12/37] adj stream operator result --- include/libs/executor/storageapi.h | 7 +- include/libs/stream/tstreamUpdate.h | 9 ++- include/util/tscalablebf.h | 2 +- source/libs/executor/src/scanoperator.c | 3 +- .../executor/src/streamtimewindowoperator.c | 4 +- source/libs/stream/src/streamUpdate.c | 69 +++++++++++++------ source/util/src/tscalablebf.c | 13 ++-- source/util/test/bloomFilterTest.cpp | 24 +++++-- 8 files changed, 89 insertions(+), 42 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 8afe55b0a1..a746fa8443 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -367,7 +367,8 @@ typedef struct SStateStore { int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen, int32_t* pWinCode); int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); - int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode); + int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen, + int32_t* pWinCode); void (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); void (*streamStateSessionReset)(SStreamState* pState, void* pVal); void (*streamStateSessionClear)(SStreamState* pState); @@ -385,8 +386,8 @@ typedef struct SStateStore { int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); - TSKEY(*updateInfoFillBlockData) - (SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); + int32_t (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, + int32_t primaryKeyCol, TSKEY* pMaxResTs); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index e1b33de429..06465e79e5 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -25,9 +25,12 @@ extern "C" { #endif -int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); -int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); -TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); +int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo); +int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, + SUpdateInfo** ppInfo); +int32_t updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol, + TSKEY* pMaxResTs); bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid); void updateInfoDestroy(SUpdateInfo* pInfo); diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h index f82493aca2..3f0f6d6e68 100644 --- a/include/util/tscalablebf.h +++ b/include/util/tscalablebf.h @@ -34,7 +34,7 @@ typedef struct SScalableBf { int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf); int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len); -int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len); +int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len, int32_t* winRes); int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len); void tScalableBfDestroy(SScalableBf* pSBf); int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index cf464ad918..885a21d5e4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2659,7 +2659,8 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (pInfo->pRecoverRes != NULL) { calBlockTbName(pInfo, pInfo->pRecoverRes, 0); if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { - TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex, pInfo->primaryKeyIndex); + TSKEY maxTs = INT64_MIN; + pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex, pInfo->primaryKeyIndex, &maxTs); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); } if (pInfo->pCreateTbRes->info.rows > 0) { diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 5a90207125..3feb3156d8 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -4565,7 +4565,9 @@ void streamStateReloadState(SOperatorInfo* pOperator) { SStateWindowInfo nextInfo = {0}; qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, pSeKeyBuf[i].groupId, i); - getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo); + code = getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo); + TSDB_CHECK_CODE(code, lino, _end); + bool cpRes = compareWinStateKey(curInfo.pStateKey, nextInfo.pStateKey); qDebug("===stream=== reload state. next window info %" PRId64 ", %" PRIu64 ", compare:%d", nextInfo.winInfo.sessionWin.win.skey, nextInfo.winInfo.sessionWin.groupId, cpRes); diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index e3e07136c6..68e48bb378 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -172,7 +172,8 @@ int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, b code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _end); } - windowSBfAdd(pInfo, bfSize); + code = windowSBfAdd(pInfo, bfSize); + TSDB_CHECK_CODE(code, lino, _end); pInfo->pTsBuckets = taosArrayInit(DEFAULT_BUCKET_SIZE, sizeof(TSKEY)); if (pInfo->pTsBuckets == NULL) { @@ -246,7 +247,9 @@ static int32_t getSBf(SUpdateInfo* pInfo, TSKEY ts, SScalableBf** ppSBf) { if (index >= pInfo->numSBFs) { uint64_t count = index + 1 - pInfo->numSBFs; windowSBfDelete(pInfo, count); - windowSBfAdd(pInfo, count); + code = windowSBfAdd(pInfo, count); + TSDB_CHECK_CODE(code, lino, _end); + index = pInfo->numSBFs - 1; } SScalableBf* res = taosArrayGetP(pInfo->pTsSBFs, index); @@ -276,9 +279,13 @@ bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) { return false; } -TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol) { +int32_t updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol, TSKEY* pMaxResTs) { int32_t code = TSDB_CODE_SUCCESS; - if (pBlock == NULL || pBlock->info.rows == 0) return INT64_MIN; + int32_t lino = 0; + if (pBlock == NULL || pBlock->info.rows == 0) { + (*pMaxResTs) = INT64_MIN; + goto _end; + } TSKEY maxTs = INT64_MIN; void* pPkVal = NULL; void* pMaxPkVal = NULL; @@ -302,10 +309,9 @@ TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t p } } SScalableBf* pSBf = NULL; - int32_t code = getSBf(pInfo, ts, &pSBf); - if (code != TSDB_CODE_SUCCESS) { - uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + code = getSBf(pInfo, ts, &pSBf); + TSDB_CHECK_CODE(code, lino, _end); + if (pSBf) { if (primaryKeyCol >= 0) { pPkVal = colDataGetData(pPkDataInfo, i); @@ -313,18 +319,29 @@ TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t p } int32_t buffLen = getKeyBuff(ts, tbUid, pPkVal, len, pInfo->pKeyBuff); // we don't care whether the data is updated or not - tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen); + int32_t winRes = 0; + code = tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen, &winRes); + TSDB_CHECK_CODE(code, lino, _end); } } void* pMaxTs = taosHashGet(pInfo->pMap, &tbUid, sizeof(int64_t)); if (pMaxTs == NULL || pInfo->comparePkRowFn(pMaxTs, &maxTs, pMaxPkVal, pInfo->comparePkCol) == -1) { int32_t valueLen = getValueBuff(maxTs, pMaxPkVal, maxLen, pInfo->pValueBuff); - taosHashPut(pInfo->pMap, &tbUid, sizeof(int64_t), pInfo->pValueBuff, valueLen); + code = taosHashPut(pInfo->pMap, &tbUid, sizeof(int64_t), pInfo->pValueBuff, valueLen); + TSDB_CHECK_CODE(code, lino, _end); } - return maxTs; + (*pMaxResTs) = maxTs; + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t res = TSDB_CODE_FAILED; int32_t buffLen = 0; @@ -335,7 +352,8 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p if (ts < maxTs - pInfo->watermark) { // this window has been closed. if (pInfo->pCloseWinSBF) { - res = tScalableBfPut(pInfo->pCloseWinSBF, pInfo->pKeyBuff, buffLen); + code = tScalableBfPut(pInfo->pCloseWinSBF, pInfo->pKeyBuff, buffLen, &res); + TSDB_CHECK_CODE(code, lino, _end); if (res == TSDB_CODE_SUCCESS) { return false; } else { @@ -346,16 +364,16 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p } SScalableBf* pSBf = NULL; - int32_t code = getSBf(pInfo, ts, &pSBf); - if (code != TSDB_CODE_SUCCESS) { - uWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + code = getSBf(pInfo, ts, &pSBf); + TSDB_CHECK_CODE(code, lino, _end); int32_t size = taosHashGetSize(pInfo->pMap); if ((!pMapMaxTs && size < DEFAULT_MAP_SIZE) || (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == -1)) { int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff); - taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); + code = taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); + TSDB_CHECK_CODE(code, lino, _end); + // pSBf may be a null pointer if (pSBf) { res = tScalableBfPutNoCheck(pSBf, pInfo->pKeyBuff, buffLen); @@ -365,7 +383,8 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p // pSBf may be a null pointer if (pSBf) { - res = tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen); + code = tScalableBfPut(pSBf, pInfo->pKeyBuff, buffLen, &res); + TSDB_CHECK_CODE(code, lino, _end); } if (!pMapMaxTs && maxTs < ts) { @@ -378,6 +397,11 @@ bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* p } else if (res == TSDB_CODE_SUCCESS) { return false; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } // check from tsdb api return true; } @@ -596,7 +620,8 @@ int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo) { for (int32_t i = 0; i < mapSize; i++) { if (tDecodeU64(&decoder, &uid) < 0) return -1; if (tDecodeBinary(&decoder, (uint8_t**)&pVal, &valSize) < 0) return -1; - taosHashPut(pInfo->pMap, &uid, sizeof(uint64_t), pVal, valSize); + code = taosHashPut(pInfo->pMap, &uid, sizeof(uint64_t), pVal, valSize); + TSDB_CHECK_CODE(code, lino, _error); } ASSERT(mapSize == taosHashGetSize(pInfo->pMap)); if (tDecodeU64(&decoder, &pInfo->maxDataVersion) < 0) return -1; @@ -633,7 +658,11 @@ bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void res = false; } else { int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff); - taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); + int32_t code = taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); + if (code != TSDB_CODE_SUCCESS) { + res = false; + uError("%s failed at line %d since %d", __func__, __LINE__, code); + } } return res; } diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c index 2e6dfbc96d..95f67d364a 100644 --- a/source/util/src/tscalablebf.c +++ b/source/util/src/tscalablebf.c @@ -97,19 +97,20 @@ _error: return code; } -int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len) { +int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len, int32_t* winRes) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (pSBf->status == SBF_INVALID) { code = TSDB_CODE_FAILED; - TSDB_CHECK_CODE(code, lino, _error); + TSDB_CHECK_CODE(code, lino, _end); } uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len); uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len); int32_t size = taosArrayGetSize(pSBf->bfArray); for (int32_t i = size - 2; i >= 0; --i) { if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_FAILED; + (*winRes) = TSDB_CODE_FAILED; + goto _end; } } @@ -120,12 +121,12 @@ int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len) { pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf); if (code != TSDB_CODE_SUCCESS) { pSBf->status = SBF_INVALID; - TSDB_CHECK_CODE(code, lino, _error); + TSDB_CHECK_CODE(code, lino, _end); } } - return tBloomFilterPutHash(pNormalBf, h1, h2); + (*winRes) = tBloomFilterPutHash(pNormalBf, h1, h2); -_error: +_end: if (code != TSDB_CODE_SUCCESS) { uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } diff --git a/source/util/test/bloomFilterTest.cpp b/source/util/test/bloomFilterTest.cpp index 9d83cbf086..4a48677809 100644 --- a/source/util/test/bloomFilterTest.cpp +++ b/source/util/test/bloomFilterTest.cpp @@ -14,8 +14,8 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, normal_bloomFilter) { GTEST_ASSERT_NE(0, tBloomFilterInit(100, -0.1, &pBFTmp)); GTEST_ASSERT_NE(0, tBloomFilterInit(0, 0.01, &pBFTmp)); - SBloomFilter *pBF1 = NULL; - int32_t code = tBloomFilterInit(100, 0.005, &pBF1); + SBloomFilter* pBF1 = NULL; + int32_t code = tBloomFilterInit(100, 0.005, &pBF1); GTEST_ASSERT_EQ(0, code); GTEST_ASSERT_EQ(pBF1->numBits, 1152); GTEST_ASSERT_EQ(pBF1->numUnits, 1152 / 64); @@ -83,7 +83,9 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { int64_t index = 0; for (; count < 100; index++) { int64_t ts = index + ts1; - if (tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS) { + int32_t res = TSDB_CODE_SUCCESS; + int32_t code = tScalableBfPut(pSBF1, &ts, sizeof(int64_t), &res); + if (res == TSDB_CODE_SUCCESS) { count++; } } @@ -91,7 +93,9 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { for (; count < 300; index++) { int64_t ts = index + ts1; - if (tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS) { + int32_t res = TSDB_CODE_SUCCESS; + int32_t code = tScalableBfPut(pSBF1, &ts, sizeof(int64_t), &res); + if (res == TSDB_CODE_SUCCESS) { count++; } } @@ -99,7 +103,9 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { for (; count < 700; index++) { int64_t ts = index + ts1; - if (tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS) { + int32_t res = TSDB_CODE_SUCCESS; + int32_t code = tScalableBfPut(pSBF1, &ts, sizeof(int64_t), &res); + if (res == TSDB_CODE_SUCCESS) { count++; } } @@ -107,7 +113,9 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { for (; count < 1500; index++) { int64_t ts = index + ts1; - if (tScalableBfPut(pSBF1, &ts, sizeof(int64_t)) == TSDB_CODE_SUCCESS) { + int32_t res = TSDB_CODE_SUCCESS; + int32_t code = tScalableBfPut(pSBF1, &ts, sizeof(int64_t), &res); + if (res == TSDB_CODE_SUCCESS) { count++; } } @@ -132,7 +140,9 @@ TEST(TD_UTIL_BLOOMFILTER_TEST, scalable_bloomFilter) { GTEST_ASSERT_EQ(0, tScalableBfInit(size, 0.001, &pSBF4)); for (int64_t i = 0; i < 1000; i++) { int64_t ts = i + ts1; - GTEST_ASSERT_EQ(tScalableBfPut(pSBF4, &ts, sizeof(int64_t)), TSDB_CODE_SUCCESS); + int32_t res = TSDB_CODE_SUCCESS; + int32_t code = tScalableBfPut(pSBF4, &ts, sizeof(int64_t), &res); + GTEST_ASSERT_EQ(res, TSDB_CODE_SUCCESS); } for (int64_t i = 0; i < 1000; i++) { From 455c9d32c42a5519e6ae21306dab9ed1d8c38c05 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 11:44:15 +0800 Subject: [PATCH 13/37] adj stream operator result --- .../executor/src/streamtimewindowoperator.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 3feb3156d8..93b8e0af8b 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -911,10 +911,10 @@ void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBl pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (pCtx[j].fpSet.finalize) { - int32_t code1 = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); - if (TAOS_FAILED(code1)) { - qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code1)); - T_LONG_JMP(pTaskInfo->env, code1); + int32_t tmpRes = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (TAOS_FAILED(tmpRes)) { + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(tmpRes)); + TSDB_CHECK_CODE(code, lino, _end); } } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { // do nothing, todo refactor @@ -2299,7 +2299,7 @@ void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { getSessionHashKey(pKey, &hashKey); int32_t code = tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey)); if (code != TSDB_CODE_SUCCESS) { - qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + qTrace("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); } } @@ -2905,8 +2905,11 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (pCtx[j].fpSet.finalize) { - code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); - TSDB_CHECK_CODE(code, lino, _end); + int32_t tmpRes = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (TAOS_FAILED(tmpRes)) { + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(tmpRes)); + TSDB_CHECK_CODE(code, lino, _end); + } } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { // do nothing, todo refactor } else { From 952908f0c9769633e9993e1454489381755c21c2 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 14:01:24 +0800 Subject: [PATCH 14/37] adj stream operator result --- source/libs/stream/src/tstreamFileState.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 76d3f5e795..5e3e3a4654 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -123,7 +123,7 @@ static void streamFileStateDecode(TSKEY* pKey, void* pBuff, int32_t len) { pBuff static void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) { *pLen = sizeof(TSKEY); (*pVal) = taosMemoryCalloc(1, *pLen); - void* buff = *pVal; + void* buff = *pVal; int32_t tmp = taosEncodeFixedI64(&buff, *pKey); ASSERT(tmp == sizeof(TSKEY)); } @@ -456,13 +456,13 @@ SRowBuffPos* getNewRowPos(SStreamFileState* pFileState) { SRowBuffPos* pPos = taosMemoryCalloc(1, sizeof(SRowBuffPos)); if (!pPos) { code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _end); + TSDB_CHECK_CODE(code, lino, _error); } pPos->pKey = taosMemoryCalloc(1, pFileState->keyLen); if (!pPos->pKey) { code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _end); + TSDB_CHECK_CODE(code, lino, _error); } void* pBuff = getFreeBuff(pFileState); @@ -481,13 +481,15 @@ SRowBuffPos* getNewRowPos(SStreamFileState* pFileState) { } code = clearRowBuff(pFileState); - TSDB_CHECK_CODE(code, lino, _end); + TSDB_CHECK_CODE(code, lino, _error); pPos->pRowBuff = getFreeBuff(pFileState); - code = tdListAppend(pFileState->usedBuffs, &pPos); - TSDB_CHECK_CODE(code, lino, _end); _end: + code = tdListAppend(pFileState->usedBuffs, &pPos); + TSDB_CHECK_CODE(code, lino, _error); + +_error: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return NULL; @@ -500,6 +502,7 @@ _end: SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState) { SRowBuffPos* newPos = getNewRowPos(pFileState); if (!newPos) { + qError("%s failed at line %d since newPos is null", __func__, __LINE__); return NULL; } newPos->beUsed = true; From cc9233fdb3acfcfc1c1a9a13fdaf1d0bd4c22e58 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 16:08:01 +0800 Subject: [PATCH 15/37] adj stream operator result --- source/libs/executor/inc/executorInt.h | 4 +- .../executor/src/streamcountwindowoperator.c | 31 ++- .../executor/src/streameventwindowoperator.c | 40 ++-- source/libs/executor/src/streamfilloperator.c | 130 ++++++---- .../executor/src/streamtimewindowoperator.c | 225 ++++++++++-------- source/libs/stream/src/streamSessionState.c | 4 +- source/libs/stream/src/streamState.c | 2 +- source/libs/stream/src/streamUpdate.c | 22 +- source/libs/stream/src/tstreamFileState.c | 12 +- 9 files changed, 277 insertions(+), 193 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index ec095a481f..c6266c1612 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -939,7 +939,7 @@ void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pAr void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey); int32_t deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, SSHashObj* pMapDelete, SSHashObj* pPkDelete, bool needAdd); -void getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated); +int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated); int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed); int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar); int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2); @@ -991,7 +991,7 @@ void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey); int32_t saveDeleteInfo(SArray* pWins, SSessionKey key); void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins); int32_t copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted); -void copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest); +int32_t copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest); bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo); bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType); diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index a1702885c8..68ff0057a5 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -140,21 +140,21 @@ static void removeCountResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionK getSessionHashKey(pKey, &key); int32_t code = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); if (code != TSDB_CODE_SUCCESS) { - qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + qInfo("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); } code = tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); if (code != TSDB_CODE_SUCCESS) { - qWarn("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + qInfo("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); } } static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs, int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, - SSHashObj* pStDeleted, bool* pRebuild) { - SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; + SSHashObj* pStDeleted, bool* pRebuild, int32_t* pWinRows) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; + SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; int32_t num = 0; for (int32_t i = start; i < rows; i++) { if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) { @@ -173,9 +173,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI needDelState = true; if (pStDeleted && pWinInfo->winInfo.isOutput) { code = saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + TSDB_CHECK_CODE(code, lino, _end); } pWinInfo->winInfo.sessionWin.win.skey = pTs[start]; @@ -194,7 +192,13 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI } } - return maxNum; + (*pWinRows) = maxNum; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, EStreamType mode, SSessionKey* pDelRange) { @@ -292,8 +296,9 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl setSessionWinOutputInfo(pStUpdated, &curWin.winInfo); slidingRows = *curWin.pWindowCount; if (!buffInfo.rebuildWindow) { - winRows = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStUpdated, - pStDeleted, &buffInfo.rebuildWindow); + code = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStUpdated, + pStDeleted, &buffInfo.rebuildWindow, &winRows); + TSDB_CHECK_CODE(code, lino, _end); } if (buffInfo.rebuildWindow) { SSessionKey range = {0}; @@ -655,7 +660,8 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { continue; } else if (pBlock->info.type == STREAM_GET_ALL) { pInfo->recvGetAll = true; - getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated); + code = getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; @@ -696,7 +702,8 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { TSDB_CHECK_CODE(code, lino, _end); if (pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator)) { - copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted); + code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted); + TSDB_CHECK_CODE(code, lino, _end); } SSDataBlock* opRes = buildCountResult(pOperator); diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index e5f76d4544..2e4ea29437 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -185,12 +185,14 @@ _error: int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey, TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, - SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { + SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild, + int32_t* pWinRow) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; *pRebuild = false; if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) { - return 1; + (*pWinRow) = 1; + goto _end; } TSKEY maxTs = INT64_MAX; @@ -204,15 +206,14 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW for (int32_t i = start; i < rows; ++i) { if (pTsData[i] >= maxTs) { - return i - start; + (*pWinRow) = i - start; + goto _end; } if (pWin->skey > pTsData[i]) { if (pStDeleted && pWinInfo->winInfo.isOutput) { code = saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + TSDB_CHECK_CODE(code, lino, _end); } removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->winInfo.sessionWin); pWin->skey = pTsData[i]; @@ -229,7 +230,8 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW } else { *pRebuild = true; pWinInfo->pWinFlag->endFlag |= ends[i]; - return i + 1 - start; + (*pWinRow) = i + 1 - start; + goto _end; } memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); @@ -238,10 +240,17 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey) { *pRebuild = true; } - return i + 1 - start; + (*pWinRow) = i + 1 - start; + goto _end; } } - return rows - start; + (*pWinRow) = rows - start; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated, @@ -362,9 +371,10 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); bool rebuild = false; - winRows = - updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, - rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); + code = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, + rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild, &winRows); + TSDB_CHECK_CODE(code, lino, _end); + ASSERT(winRows >= 1); if (rebuild) { uint64_t uid = 0; @@ -616,7 +626,8 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { continue; } else if (pBlock->info.type == STREAM_GET_ALL) { pInfo->recvGetAll = true; - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated); + code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; @@ -668,7 +679,8 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { taosArrayDestroy(pHisWins); } if (pInfo->destHasPrimaryKey && IS_NORMAL_EVENT_OP(pOperator)) { - copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted); + code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted); + TSDB_CHECK_CODE(code, lino, _end); } initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 518f7a33fb..5969bd3bf4 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -150,22 +150,6 @@ void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStora resetFillWindow(&pFillSup->nextNext); } -void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - - void* pState = pOperator->pTaskInfo->streamInfo.pState; - resetPrevAndNextWindow(pFillSup, pState, pAPI); - - SWinKey key = {.ts = ts, .groupId = groupId}; - int32_t curVLen = 0; - - int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } - pFillSup->cur.key = key.ts; -} - void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; void* pState = pOperator->pTaskInfo->streamInfo.pState; @@ -477,25 +461,40 @@ void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillS ASSERT(pFillInfo->pos != FILL_POS_INVALID); } -static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId) { +static int32_t checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId, bool* pRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SWinKey key = {.groupId = groupId, .ts = ts}; if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { - return false; + (*pRes) = false; } - int32_t code = tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + code = tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); + TSDB_CHECK_CODE(code, lino, _end); + (*pRes) = true; + +_end: if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - return true; + return code; } -static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) { +static int32_t buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock, + bool* pRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (pBlock->info.rows >= pBlock->info.capacity) { - return false; + (*pRes) = false; + goto _end; } uint64_t groupId = pBlock->info.id.groupId; - if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) { - return true; + bool ckRes = true; + code = checkResult(pFillSup, ts, groupId, &ckRes); + TSDB_CHECK_CODE(code, lino, _end); + + if (pFillSup->hasDelete && !ckRes) { + (*pRes) = true; + goto _end; } for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) { SFillColInfo* pFillCol = pFillSup->pAllColInfo + i; @@ -509,14 +508,18 @@ static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFill bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows); if (!filled) { SResultCellData* pCell = getResultCell(pResRow, slotId); - int32_t code = setRowCell(pColData, pBlock->info.rows, pCell); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + code = setRowCell(pColData, pBlock->info.rows, pCell); + TSDB_CHECK_CODE(code, lino, _end); } } pBlock->info.rows++; - return true; + (*pRes) = true; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { @@ -527,26 +530,37 @@ static bool hasRemainCalc(SStreamFillInfo* pFillInfo) { } static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; if (inWinRange(&pFillSup->winRange, &st)) { - bool res = buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock); - if (!res) { - int32_t code = TSDB_CODE_FAILED; - qError("%s failed at line %d since block is full", __func__, __LINE__); - } + bool res = true; + code = buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock, &res); + TSDB_CHECK_CODE(code, lino, _end); } pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillSup->interval.precision); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) { uint64_t groupId = pBlock->info.id.groupId; SWinKey key = {.groupId = groupId, .ts = pFillInfo->current}; STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current}; - if ((pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId)) || + bool ckRes = true; + code = checkResult(pFillSup, pFillInfo->current, groupId, &ckRes); + TSDB_CHECK_CODE(code, lino, _end); + + if ((pFillSup->hasDelete && !ckRes) || !inWinRange(&pFillSup->winRange, &st)) { pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, pFillSup->interval.precision); @@ -570,10 +584,8 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* if (pFillCol->notFillCol) { bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index); if (!filled) { - int32_t code = setRowCell(pColData, index, pCell); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + code = setRowCell(pColData, index, pCell); + TSDB_CHECK_CODE(code, lino, _end); } } else { if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) { @@ -590,10 +602,8 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* cur.key = pFillInfo->current; cur.val = taosMemoryCalloc(1, pCell->bytes); taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type); - int32_t code = colDataSetVal(pColData, index, (const char*)cur.val, false); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + code = colDataSetVal(pColData, index, (const char*)cur.val, false); + TSDB_CHECK_CODE(code, lino, _end); destroySPoint(&cur); } } @@ -601,6 +611,11 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillSup->interval.precision); pBlock->info.rows++; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) { @@ -613,17 +628,19 @@ static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SRes } static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + bool res = false; if (pFillInfo->needFill == false) { - bool res = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes); - if (!res) { - int32_t code = TSDB_CODE_FAILED; - qError("%s failed at line %d since block is full", __func__, __LINE__); - } + code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res); + TSDB_CHECK_CODE(code, lino, _end); return; } if (pFillInfo->pos == FILL_POS_START) { - if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { + code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res); + TSDB_CHECK_CODE(code, lino, _end); + if (res) { pFillInfo->pos = FILL_POS_INVALID; } } @@ -633,7 +650,9 @@ static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* doStreamFillLinear(pFillSup, pFillInfo, pRes); if (pFillInfo->pos == FILL_POS_MID) { - if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { + code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res); + TSDB_CHECK_CODE(code, lino, _end); + if (res) { pFillInfo->pos = FILL_POS_INVALID; } } @@ -648,10 +667,17 @@ static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* } } if (pFillInfo->pos == FILL_POS_END) { - if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) { + code = buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes, &res); + TSDB_CHECK_CODE(code, lino, _end); + if (res) { pFillInfo->pos = FILL_POS_INVALID; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } } void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol, diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 93b8e0af8b..da68c0e2b5 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -514,23 +514,22 @@ void reloadFromDownStream(SOperatorInfo* downstream, SStreamIntervalOperatorInfo pInfo->pUpdateInfo = pScanInfo->pUpdateInfo; } -void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo) { +int32_t initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore; if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - initIntervalDownStream(downstream->pDownstream[0], type, pInfo); - return; + return initIntervalDownStream(downstream->pDownstream[0], type, pInfo); } SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; if (!pScanInfo->pUpdateInfo) { - int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, - pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at %d since %s", __func__, __LINE__, tstrerror(code)); - } + code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, + pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo); + TSDB_CHECK_CODE(code, lino, _end); } pScanInfo->interval = pInfo->interval; @@ -538,6 +537,12 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt pScanInfo->pState = pInfo->pState; pInfo->pUpdateInfo = pScanInfo->pUpdateInfo; pInfo->basic.primaryPkIndex = pScanInfo->primaryKeyIndex; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput, @@ -1795,12 +1800,14 @@ _end: SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild, SReadHandle* pHandle) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - int32_t code = 0; if (pInfo == NULL || pOperator == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } pOperator->exprSupp.hasWindowOrGroup = true; @@ -1830,9 +1837,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); } int32_t numOfCols = 0; @@ -1850,9 +1855,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, pInfo->pState, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); initResultRowInfo(&pInfo->binfo.resultRowInfo); @@ -1912,12 +1915,11 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { - initIntervalDownStream(downstream, pPhyNode->type, pInfo); + code = initIntervalDownStream(downstream, pPhyNode->type, pInfo); + TSDB_CHECK_CODE(code, lino, _error); } code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); // for stream void* buff = NULL; @@ -2162,13 +2164,19 @@ void getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResu } int32_t saveDeleteInfo(SArray* pWins, SSessionKey key) { - // key.win.ekey = key.win.skey; - void* res = taosArrayPush(pWins, &key); + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + void* res = taosArrayPush(pWins, &key); if (!res) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _end); } - return TSDB_CODE_SUCCESS; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) { @@ -2240,17 +2248,19 @@ void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SAr int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId, int32_t rows, int32_t start, int64_t gap, - SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted) { + SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, + int32_t* pWinRos) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; for (int32_t i = start; i < rows; ++i) { if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) { - return i - start; + (*pWinRos) = i - start; + goto _end; } if (pWinInfo->sessionWin.win.skey > pStartTs[i]) { if (pStDeleted && pWinInfo->isOutput) { - int32_t code = saveDeleteRes(pStDeleted, pWinInfo->sessionWin); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + code = saveDeleteRes(pStDeleted, pWinInfo->sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->sessionWin); pWinInfo->sessionWin.win.skey = pStartTs[i]; @@ -2261,7 +2271,13 @@ int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* } memcpy(pWinInfo->pStatePos->pKey, &pWinInfo->sessionWin, sizeof(SSessionKey)); } - return rows - start; + (*pWinRos) = rows - start; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx, @@ -2357,10 +2373,8 @@ int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWi if (pNextWin->isOutput && pStDeleted) { qDebug("===stream=== save delete window info %" PRId64 ", %" PRIu64, pNextWin->sessionWin.win.skey, pNextWin->sessionWin.groupId); - int32_t code = saveDeleteRes(pStDeleted, pNextWin->sessionWin); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + code = saveDeleteRes(pStDeleted, pNextWin->sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } removeSessionResult(pAggSup, pStUpdated, pAggSup->pResultRows, &pNextWin->sessionWin); doDeleteSessionWindow(pAggSup, &pNextWin->sessionWin); @@ -2499,8 +2513,9 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData continue; } setSessionWinOutputInfo(pStUpdated, &winInfo); - winRows = updateSessionWindowInfo(pAggSup, &winInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap, - pAggSup->pResultRows, pStUpdated, pStDeleted); + code = updateSessionWindowInfo(pAggSup, &winInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap, + pAggSup->pResultRows, pStUpdated, pStDeleted, &winRows); + TSDB_CHECK_CODE(code, lino, _end); int64_t winDelta = 0; if (addGap) { @@ -2749,7 +2764,7 @@ static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, _end: if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } return code; } @@ -2797,7 +2812,9 @@ _end: return code; } -void getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { +int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { @@ -2806,12 +2823,15 @@ void getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { continue; } pWinInfo->pStatePos->beUpdated = false; - int32_t code = saveResult(*pWinInfo, pStUpdated); - if (code != TSDB_CODE_SUCCESS) { - pWinInfo->pStatePos->beUpdated = true; - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + code = saveResult(*pWinInfo, pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) { @@ -3186,21 +3206,27 @@ void resetUnCloseSessionWinInfo(SSHashObj* winMap) { } } -void copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest) { +int32_t copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (tSimpleHashGetSize(source) == 0) { - return; + goto _end; } void* pIte = NULL; int32_t iter = 0; size_t keyLen = 0; while ((pIte = tSimpleHashIterate(source, pIte, &iter)) != NULL) { SSessionKey* pKey = tSimpleHashGetKey(pIte, &keyLen); - int32_t code = saveDeleteRes(dest, *pKey); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + code = saveDeleteRes(dest, *pKey); + TSDB_CHECK_CODE(code, lino, _end); } tSimpleHashClear(source); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { @@ -3281,7 +3307,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { continue; } else if (pBlock->info.type == STREAM_GET_ALL) { pInfo->recvGetAll = true; - getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated); + code = getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; @@ -3350,7 +3377,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) { - copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted); + code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted); + TSDB_CHECK_CODE(code, lino, _end); } initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); pInfo->pUpdated = NULL; @@ -3518,22 +3546,16 @@ void streamSessionReloadState(SOperatorInfo* pOperator) { winInfo.sessionWin.groupId); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { code = saveResult(winInfo, pInfo->pStUpdated); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + TSDB_CHECK_CODE(code, lino, _end); } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { if (!isCloseWindow(&winInfo.sessionWin.win, &pInfo->twAggSup)) { code = saveDeleteRes(pInfo->pStDeleted, winInfo.sessionWin); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + TSDB_CHECK_CODE(code, lino, _end); } SSessionKey key = {0}; getSessionHashKey(&winInfo.sessionWin, &key); code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + TSDB_CHECK_CODE(code, lino, _end); } } code = saveSessionOutputBuf(pAggSup, &winInfo); @@ -3775,7 +3797,8 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { pInfo->clearState = true; break; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pStUpdated); + code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pStUpdated); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; @@ -4098,7 +4121,10 @@ _end: int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId, SColumnInfoData* pKeyCol, int32_t rows, int32_t start, - bool* allEqual, SSHashObj* pResultRows, SSHashObj* pSeUpdated, SSHashObj* pSeDeleted) { + bool* allEqual, SSHashObj* pResultRows, SSHashObj* pSeUpdated, SSHashObj* pSeDeleted, + int32_t* pWinRows) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; *allEqual = true; for (int32_t i = start; i < rows; ++i) { char* pKeyData = colDataGetData(pKeyCol, i); @@ -4107,20 +4133,20 @@ int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pW if (IS_VALID_SESSION_WIN(pNextWin->winInfo)) { // ts belongs to the next window if (pTs[i] >= pNextWin->winInfo.sessionWin.win.skey) { - return i - start; + (*pWinRows) = i - start; + goto _end; } } } else { - return i - start; + (*pWinRows) = i - start; + goto _end; } } if (pWinInfo->winInfo.sessionWin.win.skey > pTs[i]) { if (pSeDeleted && pWinInfo->winInfo.isOutput) { - int32_t code = saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + code = saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin); + TSDB_CHECK_CODE(code, lino, _end); } removeSessionResult(pAggSup, pSeUpdated, pResultRows, &pWinInfo->winInfo.sessionWin); pWinInfo->winInfo.sessionWin.win.skey = pTs[i]; @@ -4131,7 +4157,13 @@ int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pW *allEqual = false; } } - return rows - start; + (*pWinRows) = rows - start; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, @@ -4189,8 +4221,10 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore); setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); - winRows = updateStateWindowInfo(pAggSup, &curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual, - pAggSup->pResultRows, pSeUpdated, pStDeleted); + code = updateStateWindowInfo(pAggSup, &curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual, + pAggSup->pResultRows, pSeUpdated, pStDeleted, &winRows); + TSDB_CHECK_CODE(code, lino, _end); + if (!allEqual) { uint64_t uid = 0; appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, @@ -4442,7 +4476,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { continue; } else if (pBlock->info.type == STREAM_GET_ALL) { pInfo->recvGetAll = true; - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated); + code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated); + TSDB_CHECK_CODE(code, lino, _end); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; @@ -4483,7 +4518,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { TSDB_CHECK_CODE(code, lino, _end); } if (pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator)) { - copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted); + code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted); + TSDB_CHECK_CODE(code, lino, _end); } initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); @@ -4582,23 +4618,17 @@ void streamStateReloadState(SOperatorInfo* pOperator) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { code = saveResult(curInfo.winInfo, pInfo->pSeUpdated); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + TSDB_CHECK_CODE(code, lino, _end); } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) { code = saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + TSDB_CHECK_CODE(code, lino, _end); } SSessionKey key = {0}; getSessionHashKey(&curInfo.winInfo.sessionWin, &key); code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + TSDB_CHECK_CODE(code, lino, _end); } } else if (IS_VALID_SESSION_WIN(nextInfo.winInfo)) { releaseOutputBuf(pAggSup->pState, nextInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); @@ -4896,16 +4926,17 @@ _end: SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - goto _error; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode; - - int32_t code = TSDB_CODE_SUCCESS; - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); pInfo->interval = (SInterval){ @@ -4948,17 +4979,13 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, pInfo->pState, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); if (pIntervalPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); } pInfo->invertible = false; @@ -5010,11 +5037,11 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys taosMemoryFree(buff); } - initIntervalDownStream(downstream, pPhyNode->type, pInfo); + code = initIntervalDownStream(downstream, pPhyNode->type, pInfo); + TSDB_CHECK_CODE(code, lino, _error); + code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + TSDB_CHECK_CODE(code, lino, _error); return pOperator; diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 2c30eff832..dd63dc6115 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -936,9 +936,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C } code = tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); - if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - } + TSDB_CHECK_CODE(code, lino, _end); } TSKEY startTs = pWinKey->win.skey; diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index ce4b1800fb..56a74e149d 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -239,7 +239,7 @@ int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal // todo refactor void streamStateFillDel(SStreamState* pState, const SWinKey* key) { int32_t code = streamStateFillDel_rocksdb(pState, key); - qError("%s at line %d res %d", __func__, __LINE__, code); + qTrace("%s at line %d res %d", __func__, __LINE__, code); } void streamStateClear(SStreamState* pState) { streamFileStateClear(pState->pFileState); } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 68e48bb378..e395b3066b 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -279,7 +279,8 @@ bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid) { return false; } -int32_t updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol, TSKEY* pMaxResTs) { +int32_t updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol, + TSKEY* pMaxResTs) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; if (pBlock == NULL || pBlock->info.rows == 0) { @@ -652,17 +653,22 @@ _error: } bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len) { - TSKEY* pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); - bool res = true; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + TSKEY* pMapMaxTs = taosHashGet(pInfo->pMap, &tableId, sizeof(uint64_t)); + bool res = true; if (pMapMaxTs && pInfo->comparePkRowFn(pMapMaxTs, &ts, pPkVal, pInfo->comparePkCol) == 1) { res = false; } else { int32_t valueLen = getValueBuff(ts, pPkVal, len, pInfo->pValueBuff); - int32_t code = taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); - if (code != TSDB_CODE_SUCCESS) { - res = false; - uError("%s failed at line %d since %d", __func__, __LINE__, code); - } + code = taosHashPut(pInfo->pMap, &tableId, sizeof(uint64_t), pInfo->pValueBuff, valueLen); + TSDB_CHECK_CODE(code, lino, _error); } return res; + +_error: + if (code != TSDB_CODE_SUCCESS) { + uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return false; } diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 5e3e3a4654..2f656d9b41 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -500,16 +500,24 @@ _error: } SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SRowBuffPos* newPos = getNewRowPos(pFileState); if (!newPos) { - qError("%s failed at line %d since newPos is null", __func__, __LINE__); - return NULL; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _error); } newPos->beUsed = true; newPos->beFlushed = false; newPos->needFree = false; newPos->beUpdated = true; return newPos; + +_error: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return NULL; } int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen, From afa7d04aaf67d9d0305d13ab4be0edac63b4efa1 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 16:39:01 +0800 Subject: [PATCH 16/37] adj stream operator result --- source/libs/executor/src/groupoperator.c | 2 +- source/libs/stream/src/streamSessionState.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 2e65ca2a7a..58228cb2e5 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -1374,7 +1374,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, NULL); code = buildStreamCreateTableResult(pOperator); TSDB_CHECK_CODE(code, lino, _end); - if (pInfo->pCreateTbRes->info.rows > 0) { + if (pInfo->pCreateTbRes && pInfo->pCreateTbRes->info.rows > 0) { return pInfo->pCreateTbRes; } return buildStreamPartitionResult(pOperator); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index dd63dc6115..c20f6b0577 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -713,6 +713,7 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void if (!pNewPos || !pNewPos->pRowBuff) { code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFreeClear(pData); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); return code; } memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); From 3e262250d73ffd01093c44445e1b6e363c94a87b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jul 2024 17:17:02 +0800 Subject: [PATCH 17/37] enh: refact errno --- source/dnode/vnode/src/tsdb/tsdbFS.c | 45 +-- source/dnode/vnode/src/tsdb/tsdbFS2.c | 124 ++++--- source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 4 +- source/dnode/vnode/src/tsdb/tsdbFSetRW.c | 7 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 342 ++++++++---------- 5 files changed, 234 insertions(+), 288 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index a796f5121c..b262912f9b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -90,12 +90,14 @@ _exit: } static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) { - int32_t code = 0; - int32_t lino = 0; + int32_t code = 0; + int32_t lino = 0; + TdFilePtr pFD = NULL; + uint8_t *pData = NULL; // encode to binary - int32_t size = tsdbFSToBinary(NULL, pFS) + sizeof(TSCKSUM); - uint8_t *pData = taosMemoryMalloc(size); + int32_t size = tsdbFSToBinary(NULL, pFS) + sizeof(TSCKSUM); + pData = taosMemoryMalloc(size); if (pData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); @@ -104,32 +106,26 @@ static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) { taosCalcChecksumAppend(0, pData, size); // save to file - TdFilePtr pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); + pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - TSDB_CHECK_CODE(code, lino, _exit); + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } int64_t n = taosWriteFile(pFD, pData, size); if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } if (taosFsyncFile(pFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(&pFD); - TSDB_CHECK_CODE(code, lino, _exit); + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } - taosCloseFile(&pFD); - _exit: - if (pData) taosMemoryFree(pData); if (code) { tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); } + taosMemoryFree(pData); + taosCloseFile(&pFD); return code; } @@ -180,11 +176,6 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); } - - // if (size != tsdbLogicToFileSize(pTsdb->fs.pDelFile->size, pTsdb->pVnode->config.tsdbPageSize)) { - // code = TSDB_CODE_FILE_CORRUPTED; - // TSDB_CHECK_CODE(code, lino, _exit); - // } } // SArray @@ -327,17 +318,15 @@ static int32_t load_fs(const char *fname, STsdbFS *pFS) { TSDB_CHECK_CODE(code, lino, _exit); } - taosCloseFile(&pFD); - - // decode binary code = tsdbBinaryToFS(pData, size, pFS); TSDB_CHECK_CODE(code, lino, _exit); _exit: - if (pData) taosMemoryFree(pData); if (code) { tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); } + taosMemoryFree(pData); + taosCloseFile(&pFD); return code; } @@ -383,7 +372,11 @@ static int32_t tsdbNewFileSet(STsdb *pTsdb, SDFileSet *pSetTo, SDFileSet *pSetFr int32_t code = 0; int32_t lino = 0; - *pSetTo = (SDFileSet){.diskId = pSetFrom->diskId, .fid = pSetFrom->fid, .nSttF = 0}; + *pSetTo = (SDFileSet){ + .diskId = pSetFrom->diskId, + .fid = pSetFrom->fid, + .nSttF = 0, + }; // head pSetTo->pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 9b8a979d01..0d620827fb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -41,7 +41,9 @@ static const char *gCurrentFname[] = { static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) { fs[0] = taosMemoryCalloc(1, sizeof(*fs[0])); - if (fs[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (fs[0] == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } fs[0]->tsdb = pTsdb; tsem_init(&fs[0]->canEdit, 0, 1); @@ -75,69 +77,75 @@ int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype) { } static int32_t save_json(const cJSON *json, const char *fname) { - int32_t code = 0; + int32_t code = 0; + int32_t lino; + char *data = NULL; + TdFilePtr fp = NULL; - char *data = cJSON_PrintUnformatted(json); - if (data == NULL) return TSDB_CODE_OUT_OF_MEMORY; + data = cJSON_PrintUnformatted(json); + if (data == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } - TdFilePtr fp = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); + fp = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (fp == NULL) { - code = TAOS_SYSTEM_ERROR(code); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); } if (taosWriteFile(fp, data, strlen(data)) < 0) { - code = TAOS_SYSTEM_ERROR(code); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); } if (taosFsyncFile(fp) < 0) { - code = TAOS_SYSTEM_ERROR(code); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); } - taosCloseFile(&fp); - _exit: + if (code) { + tsdbError("%s failed at %s:%d since %s", __func__, fname, __LINE__, tstrerror(code)); + } taosMemoryFree(data); + taosCloseFile(&fp); return code; } static int32_t load_json(const char *fname, cJSON **json) { int32_t code = 0; + int32_t lino; char *data = NULL; TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ); - if (fp == NULL) return TAOS_SYSTEM_ERROR(code); + if (fp == NULL) { + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); + } int64_t size; if (taosFStatFile(fp, &size, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(code); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); } data = taosMemoryMalloc(size + 1); if (data == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } if (taosReadFile(fp, data, size) < 0) { - code = TAOS_SYSTEM_ERROR(code); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(code), lino, _exit); } data[size] = '\0'; json[0] = cJSON_Parse(data); if (json[0] == NULL) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); } _exit: + if (code) { + tsdbError("%s failed at %s:%d since %s", __func__, fname, __LINE__, tstrerror(code)); + json[0] = NULL; + } taosCloseFile(&fp); - if (data) taosMemoryFree(data); - if (code) json[0] = NULL; + taosMemoryFree(data); return code; } @@ -146,26 +154,23 @@ int32_t save_fs(const TFileSetArray *arr, const char *fname) { int32_t lino = 0; cJSON *json = cJSON_CreateObject(); - if (!json) return TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); // fmtv if (cJSON_AddNumberToObject(json, "fmtv", 1) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } // fset cJSON *ajson = cJSON_AddArrayToObject(json, "fset"); if (!ajson) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } const STFileSet *fset; TARRAY2_FOREACH(arr, fset) { cJSON *item = cJSON_CreateObject(); if (!item) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } cJSON_AddItemToArray(ajson, item); @@ -226,14 +231,17 @@ static int32_t load_fs(STsdb *pTsdb, const char *fname, TFileSetArray *arr) { _exit: if (code) { - tsdbError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); + tsdbError("%s failed at %sP%d since %s, fname:%s", __func__, __FILE__, lino, tstrerror(code), fname); + } + if (json) { + cJSON_Delete(json); } - if (json) cJSON_Delete(json); return code; } static int32_t apply_commit(STFileSystem *fs) { int32_t code = 0; + int32_t lino; TFileSetArray *fsetArray1 = fs->fSetArr; TFileSetArray *fsetArray2 = fs->fSetArrTmp; int32_t i1 = 0, i2 = 0; @@ -250,15 +258,15 @@ static int32_t apply_commit(STFileSystem *fs) { } else if (fset1->fid > fset2->fid) { // create new file set with fid of fset2->fid code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1); - if (code) return code; + TSDB_CHECK_CODE(code, lino, _exit); code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); - if (code) return code; + TSDB_CHECK_CODE(code, lino, _exit); i1++; i2++; } else { // edit code = tsdbTFileSetApplyEdit(fs->tsdb, fset2, fset1); - if (code) return code; + TSDB_CHECK_CODE(code, lino, _exit); i1++; i2++; } @@ -269,15 +277,19 @@ static int32_t apply_commit(STFileSystem *fs) { } else { // create new file set with fid of fset2->fid code = tsdbTFileSetInitCopy(fs->tsdb, fset2, &fset1); - if (code) return code; + TSDB_CHECK_CODE(code, lino, _exit); code = TARRAY2_SORT_INSERT(fsetArray1, fset1, tsdbTFileSetCmprFn); - if (code) return code; + TSDB_CHECK_CODE(code, lino, _exit); i1++; i2++; } } - return 0; +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(fs->tsdb->pVnode), lino, code); + } + return code; } static int32_t commit_edit(STFileSystem *fs) { @@ -303,7 +315,8 @@ static int32_t commit_edit(STFileSystem *fs) { _exit: if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(fs->tsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(fs->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } else { tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); } @@ -335,7 +348,8 @@ static int32_t abort_edit(STFileSystem *fs) { _exit: if (code) { - tsdbError("vgId:%d %s failed since %s", TD_VID(fs->tsdb->pVnode), __func__, tstrerror(code)); + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(fs->tsdb->pVnode), __func__, __FILE__, lino, + tstrerror(code)); } else { tsdbInfo("vgId:%d %s success, etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); } @@ -360,13 +374,6 @@ static int32_t tsdbFSDoScanAndFixFile(STFileSystem *fs, const STFileObj *fobj) { } found = true; - /* - const char *object_name = taosDirEntryBaseName((char *)fobj->fname); - long s3_size = s3Size(object_name); - if (s3_size > 0) { - return 0; - } - */ } if (!found) { @@ -376,16 +383,6 @@ static int32_t tsdbFSDoScanAndFixFile(STFileSystem *fs, const STFileObj *fobj) { } } - { // TODO: check file size - // int64_t fsize; - // if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) { - // code = TAOS_SYSTEM_ERROR(terrno); - // tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(fs->tsdb->pVnode), __func__, - // fobj->fname, tstrerror(code)); - // return code; - // } - } - return 0; } @@ -408,6 +405,7 @@ static int32_t tsdbFSAddEntryToFileObjHash(STFileHash *hash, const char *fname) static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) { int32_t code = 0; + int32_t lino; char fname[TSDB_FILENAME_LEN]; // init hash table @@ -415,14 +413,13 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) { hash->numBucket = 4096; hash->buckets = taosMemoryCalloc(hash->numBucket, sizeof(STFileHashEntry *)); if (hash->buckets == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } // vnode.json current_fname(fs->tsdb, fname, TSDB_FCURRENT); code = tsdbFSAddEntryToFileObjHash(hash, fname); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); // other STFileSet *fset = NULL; @@ -431,7 +428,7 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) { for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) { if (fset->farr[i] != NULL) { code = tsdbFSAddEntryToFileObjHash(hash, fset->farr[i]->fname); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } } @@ -441,13 +438,14 @@ static int32_t tsdbFSCreateFileObjHash(STFileSystem *fs, STFileHash *hash) { STFileObj *fobj; TARRAY2_FOREACH(lvl->fobjArr, fobj) { code = tsdbFSAddEntryToFileObjHash(hash, fobj->fname); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } } } _exit: if (code) { + TSDB_ERROR_LOG(TD_VID(fs->tsdb->pVnode), lino, code); tsdbFSDestroyFileObjHash(hash); } return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c index b180f9d099..1adca31347 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c @@ -35,7 +35,9 @@ int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **wri int32_t lino = 0; writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter)); - if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (writer[0] == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } writer[0]->config[0] = config[0]; diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c index f6ca515d1e..77d8998aa7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRW.c @@ -44,9 +44,10 @@ static int32_t tsdbFSetWriteTableDataBegin(SFSetWriter *writer, const TABLEID *t writer->ctx->tbid->uid = tbid->uid; code = tsdbUpdateSkmTb(writer->config->tsdb, writer->ctx->tbid, writer->skmTb); + TSDB_CHECK_CODE(code , lino, _exit); code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, tbid->suid ? tbid->suid : tbid->uid, &writer->pColCmprObj); - // TSDB_CHECK_CODE(code, lino, _exit); + // TODO: TSDB_CHECK_CODE(code, lino, _exit); writer->blockDataIdx = 0; for (int32_t i = 0; i < ARRAY_SIZE(writer->blockData); i++) { @@ -136,7 +137,9 @@ int32_t tsdbFSetWriterOpen(SFSetWriterConfig *config, SFSetWriter **writer) { int32_t lino = 0; writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); - if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (writer[0] == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } writer[0]->config[0] = config[0]; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 932bf2d92c..eb4fee3024 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -16,10 +16,12 @@ #include "cos.h" #include "crypt.h" #include "tsdb.h" +#include "tsdbDef.h" #include "vnd.h" static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { int32_t code = 0; + int32_t lino; const char *path = pFD->path; int32_t szPage = pFD->szPage; int32_t flag = pFD->flag; @@ -38,70 +40,27 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { char *dot = strrchr(lc_path, '.'); if (!dot) { tsdbError("unexpected path: %s", lc_path); - code = TAOS_SYSTEM_ERROR(ENOENT); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(ENOENT), lino, _exit); } snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", pFD->lcn); pFD->pFD = taosOpenFile(lc_path, flag); if (pFD->pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - // taosMemoryFree(pFD); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } if (taosStatFile(lc_path, &lc_size, NULL, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - // taosCloseFile(&pFD->pFD); - // taosMemoryFree(pFD); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } } else { tsdbInfo("no file: %s", path); - code = TAOS_SYSTEM_ERROR(errno); - // taosMemoryFree(pFD); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } pFD->s3File = 1; - /* - const char *object_name = taosDirEntryBaseName((char *)path); - long s3_size = 0; - if (tsS3Enabled) { - long size = s3Size(object_name); - if (size < 0) { - code = terrno = TSDB_CODE_FAILED_TO_CONNECT_S3; - goto _exit; - } - - s3_size = size; - } - - if (tsS3Enabled && !strncmp(path + strlen(path) - 5, ".data", 5) && s3_size > 0) { -#ifndef S3_BLOCK_CACHE - s3EvictCache(path, s3_size); - s3Get(object_name, path); - - pFD->pFD = taosOpenFile(path, flag); - if (pFD->pFD == NULL) { - code = TAOS_SYSTEM_ERROR(ENOENT); - // taosMemoryFree(pFD); - goto _exit; - } -#else - pFD->pFD = (TdFilePtr)&pFD->s3File; - int32_t vid = 0; - sscanf(object_name, "v%df%dver%" PRId64 ".data", &vid, &pFD->fid, &pFD->cid); - pFD->objName = object_name; - // pFD->szFile = s3_size; -#endif - */ } pFD->pBuf = taosMemoryCalloc(1, szPage); if (pFD->pBuf == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - // taosCloseFile(&pFD->pFD); - // taosMemoryFree(pFD); - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } if (lc_size > 0) { @@ -114,11 +73,7 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { // not check file size when reading data files. if (flag != TD_FILE_READ /* && !pFD->s3File*/) { if (!lc_size && taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - // taosMemoryFree(pFD->pBuf); - // taosCloseFile(&pFD->pFD); - // taosMemoryFree(pFD); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } } @@ -126,12 +81,16 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { pFD->szFile = pFD->szFile / szPage; _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code); + } return code; } // =============== PAGE-WISE FILE =============== int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn) { int32_t code = 0; + int32_t lino; STsdbFD *pFD = NULL; int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; @@ -139,8 +98,7 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF pFD = (STsdbFD *)taosMemoryCalloc(1, sizeof(*pFD) + strlen(path) + 1); if (pFD == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } pFD->path = (char *)&pFD[1]; @@ -155,6 +113,9 @@ int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppF *ppFD = pFD; _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pTsdb->pVnode), lino, code); + } return code; } @@ -172,12 +133,11 @@ void tsdbCloseFile(STsdbFD **ppFD) { static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) { int32_t code = 0; + int32_t lino; if (!pFD->pFD) { code = tsdbOpenFileImpl(pFD); - if (code) { - goto _exit; - } + TSDB_CHECK_CODE(code, lino, _exit); } if (pFD->pgno > 0) { @@ -193,8 +153,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); @@ -223,8 +182,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } if (pFD->szFile < pFD->pgno) { @@ -234,18 +192,20 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD, int32_t encryptAlgorithm, char *e pFD->pgno = 0; _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code); + } return code; } static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgorithm, char *encryptKey) { int32_t code = 0; + int32_t lino; // ASSERT(pgno <= pFD->szFile); if (!pFD->pFD) { code = tsdbOpenFileImpl(pFD); - if (code) { - goto _exit; - } + TSDB_CHECK_CODE(code, lino, _exit); } int64_t offset = PAGE_OFFSET(pgno, pFD->szPage); @@ -257,43 +217,19 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor offset -= chunkoffset; } ASSERT(offset >= 0); - /* - if (pFD->s3File) { - LRUHandle *handle = NULL; - pFD->blkno = (pgno + tsS3BlockSize - 1) / tsS3BlockSize; - code = tsdbCacheGetBlockS3(pFD->pTsdb->bCache, pFD, &handle); - if (code != TSDB_CODE_SUCCESS || handle == NULL) { - tsdbCacheRelease(pFD->pTsdb->bCache, handle); - if (code == TSDB_CODE_SUCCESS && !handle) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - goto _exit; - } - - uint8_t *pBlock = (uint8_t *)taosLRUCacheValue(pFD->pTsdb->bCache, handle); - - int64_t blk_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage; - memcpy(pFD->pBuf, pBlock + (offset - blk_offset), pFD->szPage); - - tsdbCacheRelease(pFD->pTsdb->bCache, handle); - } else { - */ // seek int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } // read n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } else if (n < pFD->szPage) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); } //} @@ -322,19 +258,22 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno, int32_t encryptAlgor // check if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); } pFD->pgno = pgno; _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code); + } return code; } int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, char *encryptKey) { int32_t code = 0; + int32_t lino; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int64_t bOffset = fOffset % pFD->szPage; @@ -343,11 +282,11 @@ int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t do { if (pFD->pgno != pgno) { code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); if (pgno <= pFD->szFile) { code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } else { pFD->pgno = pgno; } @@ -362,12 +301,16 @@ int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t } while (n < size); _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code); + } return code; } static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm, char *encryptKey) { int32_t code = 0; + int32_t lino; int64_t n = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); @@ -380,7 +323,7 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6 while (n < size) { if (pFD->pgno != pgno) { code = tsdbReadFilePage(pFD, pgno, encryptAlgorithm, encryptKey); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } int64_t nRead = TMIN(szPgCont - bOffset, size - n); @@ -392,15 +335,20 @@ static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int6 } _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code); + } return code; } static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { int32_t code = 0; + int32_t lino; SVnodeCfg *pCfg = &pFD->pTsdb->pVnode->config; int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; int64_t cOffset = offset % chunksize; int64_t n = 0; + char *buf = NULL; char *object_name = taosDirEntryBaseName(pFD->path); char object_name_prefix[TSDB_FILENAME_LEN]; @@ -410,11 +358,13 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo char *dot = strrchr(object_name_prefix, '.'); if (!dot) { tsdbError("unexpected path: %s", object_name_prefix); - code = TAOS_SYSTEM_ERROR(ENOENT); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(ENOENT), lino, _exit); } - char *buf = taosMemoryCalloc(1, size); + buf = taosMemoryCalloc(1, size); + if (buf == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } for (int32_t chunkno = offset / chunksize + 1; n < size; ++chunkno) { int64_t nRead = TMIN(chunksize - cOffset, size - n); @@ -423,20 +373,14 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo // read last chunk int64_t ret = taosLSeekFile(pFD->pFD, chunksize * (chunkno - pFD->lcn) + cOffset, SEEK_SET); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosMemoryFree(buf); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } ret = taosReadFile(pFD->pFD, buf + n, nRead); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); - taosMemoryFree(buf); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } else if (ret < nRead) { - code = TSDB_CODE_FILE_CORRUPTED; - taosMemoryFree(buf); - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); } } else { uint8_t *pBlock = NULL; @@ -444,10 +388,7 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", chunkno); code = s3GetObjectBlock(object_name_prefix, cOffset, nRead, check, &pBlock); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(buf); - goto _exit; - } + TSDB_CHECK_CODE(code, lino, _exit); memcpy(buf + n, pBlock, nRead); taosMemoryFree(pBlock); @@ -457,14 +398,19 @@ static int32_t tsdbReadFileBlock(STsdbFD *pFD, int64_t offset, int64_t size, boo cOffset = 0; } - *ppBlock = buf; - _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code); + taosMemoryFree(buf); + } else { + *ppBlock = (uint8_t *)buf; + } return code; } static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint) { int32_t code = 0; + int32_t lino; int64_t n = 0; int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); @@ -486,7 +432,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64 if (handle) { tsdbCacheRelease(pFD->pTsdb->pgCache, handle); } - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } if (!handle) { @@ -499,8 +445,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64 // check if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); } pFD->pgno = pgno; @@ -525,16 +470,9 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64 } int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage; - /* - code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } - */ + code = tsdbReadFileBlock(pFD, retrieve_offset, retrieve_size, 1, &pBlock); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } + TSDB_CHECK_CODE(code, lino, _exit); // 3, Store Pages in Cache int nPage = pgnoEnd - pgno + 1; for (int i = 0; i < nPage; ++i) { @@ -550,8 +488,7 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64 // check if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit); } pFD->pgno = pgno; @@ -568,59 +505,72 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64 } _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code); + } return code; } int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint, int32_t encryptAlgorithm, char *encryptKey) { int32_t code = 0; + int32_t lino; + if (!pFD->pFD) { code = tsdbOpenFileImpl(pFD); - if (code) { - goto _exit; - } + TSDB_CHECK_CODE(code, lino, _exit); } if (pFD->s3File && pFD->lcn > 1 /* && tsS3BlockSize < 0*/) { - return tsdbReadFileS3(pFD, offset, pBuf, size, szHint); + code = tsdbReadFileS3(pFD, offset, pBuf, size, szHint); + TSDB_CHECK_CODE(code, lino, _exit); } else { - return tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey); + code = tsdbReadFileImp(pFD, offset, pBuf, size, encryptAlgorithm, encryptKey); + TSDB_CHECK_CODE(code, lino, _exit); } _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code); + } return code; } int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint, int32_t encryptAlgorithm, char *encryptKey) { int32_t code; + int32_t lino; code = tBufferEnsureCapacity(buffer, buffer->size + size); - if (code) return code; + TSDB_CHECK_CODE(code, lino, _exit); + code = tsdbReadFile(pFD, offset, (uint8_t *)tBufferGetDataEnd(buffer), size, szHint, encryptAlgorithm, encryptKey); - if (code) return code; + TSDB_CHECK_CODE(code, lino, _exit); + buffer->size += size; +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code); + } return code; } int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey) { int32_t code = 0; - /* - if (pFD->s3File) { - tsdbWarn("%s file: %s", __func__, pFD->path); - return code; - } - */ + int32_t lino; + code = tsdbWriteFilePage(pFD, encryptAlgorithm, encryptKey); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); if (taosFsyncFile(pFD->pFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; + TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit); } _exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pFD->pTsdb->pVnode), lino, code); + } return code; } @@ -635,8 +585,7 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS // alloc pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } pReader->pTsdb = pTsdb; pReader->pSet = pSet; @@ -711,22 +660,25 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { int32_t code = 0; + int32_t lino; SHeadFile *pHeadFile = pReader->pSet->pHeadF; int64_t offset = pHeadFile->offset; int64_t size = pHeadFile->size - offset; taosArrayClear(aBlockIdx); - if (size == 0) return code; + if (size == 0) { + return code; + } // alloc code = tRealloc(&pReader->aBuf[0], size); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // read int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // decode int64_t n = 0; @@ -735,37 +687,39 @@ int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx); if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } } ASSERT(n == size); - return code; - -_err: - tsdbError("vgId:%d, read block idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code); + } return code; } int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) { int32_t code = 0; + int32_t lino; SSttFile *pSttFile = pReader->pSet->aSttF[iStt]; int64_t offset = pSttFile->offset; int64_t size = pSttFile->size - offset; taosArrayClear(aSttBlk); - if (size == 0) return code; + if (size == 0) { + return 0; + } // alloc code = tRealloc(&pReader->aBuf[0], size); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // read int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // decode int64_t n = 0; @@ -774,46 +728,45 @@ int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) { n += tGetSttBlk(pReader->aBuf[0] + n, &sttBlk); if (taosArrayPush(aSttBlk, &sttBlk) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } } ASSERT(n == size); - return code; - -_err: - tsdbError("vgId:%d, read stt blk failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code); + } return code; } int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk) { int32_t code = 0; + int32_t lino; int64_t offset = pBlockIdx->offset; int64_t size = pBlockIdx->size; // alloc code = tRealloc(&pReader->aBuf[0], size); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // read int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // decode int64_t n = tGetMapData(pReader->aBuf[0], mDataBlk); if (n < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } ASSERT(n == size); - return code; - -_err: - tsdbError("vgId:%d, read block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code); + } return code; } @@ -834,8 +787,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb // alloc pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader)); if (pDelFReader == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } // open impl @@ -844,15 +796,13 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); code = tsdbOpenFile(fname, pTsdb, TD_FILE_READ, &pDelFReader->pReadH, 0); - if (code) { - taosMemoryFree(pDelFReader); - goto _exit; - } + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { *ppReader = NULL; - tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + TSDB_ERROR_LOG(TD_VID(pTsdb->pVnode), lino, code); + taosMemoryFree(pDelFReader); } else { *ppReader = pDelFReader; } @@ -881,6 +831,7 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { int32_t code = 0; + int32_t lino; int64_t offset = pDelIdx->offset; int64_t size = pDelIdx->size; int64_t n; @@ -889,13 +840,13 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa // alloc code = tRealloc(&pReader->aBuf[0], size); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // read int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // // decode n = 0; @@ -907,22 +858,22 @@ int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelDa continue; } if (taosArrayPush(aDelData, &delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } } ASSERT(n == size); - return code; - -_err: - tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code); + } return code; } int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { int32_t code = 0; + int32_t lino; int32_t n; int64_t offset = pReader->fDel.offset; int64_t size = pReader->fDel.size - offset; @@ -931,13 +882,13 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { // alloc code = tRealloc(&pReader->aBuf[0], size); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // read int32_t encryptAlgorithm = pReader->pTsdb->pVnode->config.tsdbCfg.encryptAlgorithm; char *encryptKey = pReader->pTsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size, 0, encryptAlgorithm, encryptKey); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // decode n = 0; @@ -947,16 +898,15 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx); if (taosArrayPush(aDelIdx, &delIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } } ASSERT(n == size); - return code; - -_err: - tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(pReader->pTsdb->pVnode), lino, code); + } return code; } From f161514c190d927ed354840b8a62427aff5b2cec Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 17:24:36 +0800 Subject: [PATCH 18/37] adj stream operator result --- source/libs/executor/src/groupoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 58228cb2e5..cacb0cb20b 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -1319,7 +1319,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { if (hasRemainTbName(pInfo)) { code = buildStreamCreateTableResult(pOperator); TSDB_CHECK_CODE(code, lino, _end); - if (pInfo->pCreateTbRes->info.rows > 0) { + if (pInfo->pCreateTbRes && pInfo->pCreateTbRes->info.rows > 0) { return pInfo->pCreateTbRes; } } From dfff9b19ad262ef84a19ccb4a9bc4aa9ee6860a7 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 18:08:31 +0800 Subject: [PATCH 19/37] adj ci --- tests/script/tsim/stream/checkTaskStatus.sim | 30 +++++++++++++++++++ .../tsim/stream/partitionbyColumnInterval.sim | 17 +++++++---- tests/script/tsim/stream/sliding.sim | 15 ++++------ 3 files changed, 47 insertions(+), 15 deletions(-) create mode 100644 tests/script/tsim/stream/checkTaskStatus.sim diff --git a/tests/script/tsim/stream/checkTaskStatus.sim b/tests/script/tsim/stream/checkTaskStatus.sim new file mode 100644 index 0000000000..13570c2a86 --- /dev/null +++ b/tests/script/tsim/stream/checkTaskStatus.sim @@ -0,0 +1,30 @@ +print ====check task status start + +$loop_count = 0 + +loopCheck0: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 30 then + return -1 +endi + +print 1 select * from information_schema.ins_stream_tasks; +sql select * from information_schema.ins_stream_tasks; + +if $rows == 0 then + print rows=$rows + goto loopCheck0 +endi + +print 1 select * from information_schema.ins_stream_tasks where status != "ready"; +sql select * from information_schema.ins_stream_tasks where status != "ready"; + +if $rows != 0 then + print rows=$rows + goto loopCheck0 +endi + +print ====check task status end \ No newline at end of file diff --git a/tests/script/tsim/stream/partitionbyColumnInterval.sim b/tests/script/tsim/stream/partitionbyColumnInterval.sim index cc70242172..e70b25dd04 100644 --- a/tests/script/tsim/stream/partitionbyColumnInterval.sim +++ b/tests/script/tsim/stream/partitionbyColumnInterval.sim @@ -14,7 +14,8 @@ sql create database test vgroups 1; sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a interval(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -196,7 +197,8 @@ sql create database test1 vgroups 1; sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b interval(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -284,7 +286,8 @@ sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -482,7 +485,8 @@ sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(2,2,2); sql create table t4 using st tags(2,2,2); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt4 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,2,2,3,1.0); sql insert into t2 values(1648791213000,2,2,3,1.0); @@ -573,7 +577,8 @@ sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(2,2,2); sql create table t4 using st tags(2,2,2); sql create stream streams5 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt5 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t2 values(1648791213000,2,2,3,1.0); @@ -664,7 +669,7 @@ sql use test6; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams6 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt6 subtable("aaa-a") as select _wstart, count(*) from t1 partition by a interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,0,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0); diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index cc4d14b6fb..acf68e0286 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -21,9 +21,8 @@ sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 in sql create stream streams2 trigger at_once watermark 1d IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s) sliding (5s); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s) sliding (5s); sql create stream stream_t2 trigger at_once watermark 1d IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s) sliding (5s); -sleep 1000 -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,2,3,1.0); sql insert into t1 values(1648791216000,2,2,3,1.1); @@ -314,9 +313,8 @@ sql create table t2 using st tags(2,2,2); sql create stream streams11 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); sql create stream streams12 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); -sleep 1000 -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223001,2,2,3,1.1); @@ -450,9 +448,8 @@ sql create table t2 using st tags(2,2,2); sql create stream streams21 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt21 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); sql create stream streams22 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt22 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); -sleep 1000 -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791223001,2,2,2,1.1); @@ -591,7 +588,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams23 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt23 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(20s) sliding(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791223001,2,2,2,1.1); @@ -716,9 +714,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart as ts, count(*),min(a) c1 from st interval(10s) sliding(5s); -sleep 1000 -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791243000,2,1,1,1.0); From 8d769c97ecf9ce9b8881737ff2de661e3e4afc43 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 18:48:52 +0800 Subject: [PATCH 20/37] adj ci --- tests/script/tsim/stream/basic0.sim | 2 + tests/script/tsim/stream/basic1.sim | 14 +++++ tests/script/tsim/stream/basic2.sim | 2 + tests/script/tsim/stream/basic3.sim | 1 + tests/script/tsim/stream/basic4.sim | 7 ++- tests/script/tsim/stream/basic5.sim | 8 +-- .../script/tsim/stream/checkStreamSTable.sim | 21 ++++--- .../script/tsim/stream/checkStreamSTable1.sim | 60 +----------------- .../tsim/stream/checkpointInterval0.sim | 3 + .../tsim/stream/checkpointInterval1.sim | 4 +- .../script/tsim/stream/checkpointSession0.sim | 3 + .../script/tsim/stream/checkpointSession1.sim | 3 +- tests/script/tsim/stream/checkpointState0.sim | 3 + tests/script/tsim/stream/count0.sim | 12 +++- tests/script/tsim/stream/count2.sim | 8 ++- tests/script/tsim/stream/count3.sim | 4 +- tests/script/tsim/stream/countSliding0.sim | 8 ++- tests/script/tsim/stream/countSliding1.sim | 4 +- tests/script/tsim/stream/countSliding2.sim | 4 +- tests/script/tsim/stream/deleteInterval.sim | 12 +++- tests/script/tsim/stream/deleteScalar.sim | 9 ++- tests/script/tsim/stream/deleteSession.sim | 14 +++-- tests/script/tsim/stream/deleteState.sim | 4 ++ .../tsim/stream/distributeInterval0.sim | 11 +++- .../stream/distributeIntervalRetrive0.sim | 5 +- .../stream/distributeMultiLevelInterval0.sim | 31 +--------- .../script/tsim/stream/distributeSession0.sim | 62 +------------------ tests/script/tsim/stream/event0.sim | 12 +++- tests/script/tsim/stream/event1.sim | 12 +++- tests/script/tsim/stream/event2.sim | 3 +- tests/script/tsim/stream/eventtmp.sim | 3 +- .../script/tsim/stream/fillHistoryBasic1.sim | 4 +- .../script/tsim/stream/fillHistoryBasic2.sim | 8 ++- .../script/tsim/stream/fillHistoryBasic3.sim | 3 +- .../script/tsim/stream/fillHistoryBasic4.sim | 2 + .../tsim/stream/fillHistoryTransform.sim | 6 ++ .../tsim/stream/fillIntervalDelete0.sim | 5 ++ .../tsim/stream/fillIntervalDelete1.sim | 5 ++ .../script/tsim/stream/fillIntervalLinear.sim | 9 +++ .../tsim/stream/fillIntervalPartitionBy.sim | 2 + .../tsim/stream/fillIntervalPrevNext.sim | 6 ++ .../tsim/stream/fillIntervalPrevNext1.sim | 3 + .../script/tsim/stream/fillIntervalRange.sim | 5 ++ .../script/tsim/stream/fillIntervalValue.sim | 11 ++++ .../script/tsim/stream/ignoreCheckUpdate.sim | 8 ++- .../script/tsim/stream/ignoreExpiredData.sim | 9 ++- tests/script/tsim/stream/partitionby.sim | 7 ++- tests/script/tsim/stream/partitionby1.sim | 6 +- .../tsim/stream/partitionbyColumnOther.sim | 4 +- .../tsim/stream/partitionbyColumnSession.sim | 12 +++- .../tsim/stream/partitionbyColumnState.sim | 6 +- tests/script/tsim/stream/pauseAndResume.sim | 7 ++- tests/script/tsim/stream/scalar.sim | 2 + tests/script/tsim/stream/schedSnode.sim | 2 +- .../script/tsim/stream/streamPrimaryKey0.sim | 6 +- .../script/tsim/stream/streamPrimaryKey1.sim | 4 +- .../script/tsim/stream/streamPrimaryKey2.sim | 4 +- .../script/tsim/stream/streamPrimaryKey3.sim | 6 +- 58 files changed, 280 insertions(+), 221 deletions(-) diff --git a/tests/script/tsim/stream/basic0.sim b/tests/script/tsim/stream/basic0.sim index 5a99aadf36..1661904c98 100644 --- a/tests/script/tsim/stream/basic0.sim +++ b/tests/script/tsim/stream/basic0.sim @@ -37,6 +37,8 @@ sleep 1000 sql create stream s1 trigger at_once into outstb as select _wstart, min(k), max(k), sum(k) as sum_alias from ct1 interval(10m) +run tsim/stream/checkTaskStatus.sim + sql show stables if $rows != 2 then return -1 diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index 4b5377d928..0d89f528d1 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -18,6 +18,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223001,2,2,3,1.1); sql insert into t1 values(1648791233002,3,2,3,2.1); @@ -548,6 +551,8 @@ sql create table t5 using st tags(2,2,2); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) c3,max(b) c4 from st partition by tbname interval(10s); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1, sum(a) c3,max(b) c4, now c5 from st partition by tbname interval(10s); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,1,1,1.0) t2 values(1648791213000,2,2,2,2.0) t3 values(1648791213000,3,3,3,3.0) t4 values(1648791213000,4,4,4,4.0); $loop_count = 0 @@ -669,6 +674,8 @@ sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc i sql create table ts1 using st tags(1,1,1); sql create stream stream_t3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from ts1 interval(10s) ; +run tsim/stream/checkTaskStatus.sim + sql insert into ts1 values(1648791211000,1,2,3); sleep 50 sql insert into ts1 values(1648791222001,2,2,3); @@ -702,6 +709,9 @@ sql create database test4 vgroups 1; sql use test4; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt__4 as select _wstart, count(*) c1 from t1 where a > 5 interval(10s); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0); $loop_count = 0 @@ -800,6 +810,8 @@ sql create table ts1 using st tags(1,1,1); sql create stream streams5 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt5 as select count(*), _wstart, _wend, max(a) from ts1 interval(10s) ; sql create stream streams6 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt6 as select count(*), _wstart, _wend, max(a), _wstart as ts from ts1 interval(10s) ; +run tsim/stream/checkTaskStatus.sim + sql_error create stream streams7 trigger at_once into streamt7 as select _wstart, count(*), _wstart, _wend, max(a) from ts1 interval(10s) ; sql_error create stream streams8 trigger at_once into streamt8 as select count(*), _wstart, _wstart, _wend, max(a) from ts1 interval(10s) ; sql_error create stream streams9 trigger at_once into streamt9 as select _wstart as ts, count(*), _wstart as ts, _wend, max(a) from ts1 interval(10s) ; @@ -842,6 +854,8 @@ sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc i sql create table ts1 using st tags(1,1,1); sql create stream streams7 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt7 as select _wstart, count(*) from ts1 interval(10s) ; +run tsim/stream/checkTaskStatus.sim + sql insert into ts1 values(1648791211000,1,2,3); sql_error insert into ts1 values(-1648791211000,1,2,3); diff --git a/tests/script/tsim/stream/basic2.sim b/tests/script/tsim/stream/basic2.sim index 0ad82b8b05..ad655f2d16 100644 --- a/tests/script/tsim/stream/basic2.sim +++ b/tests/script/tsim/stream/basic2.sim @@ -35,6 +35,8 @@ endi sql create stream s1 trigger at_once into outstb as select _wstart, min(k), max(k), sum(k) as sum_alias from ct1 interval(10m) +run tsim/stream/checkTaskStatus.sim + sql show stables if $rows != 2 then return -1 diff --git a/tests/script/tsim/stream/basic3.sim b/tests/script/tsim/stream/basic3.sim index f18061a6df..010578d4d2 100644 --- a/tests/script/tsim/stream/basic3.sim +++ b/tests/script/tsim/stream/basic3.sim @@ -28,6 +28,7 @@ sql create stream streamd4 into streamt4 as select tbname, _wstart,_wend, count( sql create stream streamd5 into streamt5 as select tbname, _wstart,_wend, count(*), max(ca), min(cb) from st where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by tbname state_window(cc); sql create stream streamd6 into streamt6 as select ca, _wstart,_wend, count(*), max(ca), min(cb) from t1 where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca state_window(cc); +run tsim/stream/checkTaskStatus.sim sql alter local 'keepColumnName' '1' sql CREATE STABLE `meters_test_data` (`ts` TIMESTAMP, `close` FLOAT, `parttime` TIMESTAMP, `parttime_str` VARCHAR(32)) TAGS (`id` VARCHAR(32)); diff --git a/tests/script/tsim/stream/basic4.sim b/tests/script/tsim/stream/basic4.sim index d2bf321ad5..8868c3fd60 100644 --- a/tests/script/tsim/stream/basic4.sim +++ b/tests/script/tsim/stream/basic4.sim @@ -15,6 +15,8 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once ignore expired 0 ignore update 0 into streamt as select _wstart, count(*) c1 from t1 interval(1s); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791212001,2,2,3,1.1); sql insert into t1 values(1648791213002,3,2,3,2.1); @@ -80,7 +82,8 @@ sql use test2; sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create stream streams2 trigger at_once ignore expired 0 ignore update 0 waterMark 200s into streamt2 as select _wstart, count(*) c1 from t1 interval(1s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791212001,2,2,3,1.1); @@ -147,6 +150,8 @@ sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once ignore expired 0 ignore update 0 into streamt1 as select _wstart, count(*) c1 from t1 session(ts, 1s); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791213000,1,2,3,1.1); sql insert into t1 values(1648791215000,1,2,3,1.1); diff --git a/tests/script/tsim/stream/basic5.sim b/tests/script/tsim/stream/basic5.sim index 7b5f587feb..d7dd603d3c 100644 --- a/tests/script/tsim/stream/basic5.sim +++ b/tests/script/tsim/stream/basic5.sim @@ -15,7 +15,7 @@ sql use test3; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams3 trigger at_once ignore expired 0 ignore update 0 into streamt3 as select _wstart, count(*) c1 from t1 state_window(a); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791213000,2,2,3,1.1); @@ -225,7 +225,7 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams4 trigger at_once ignore expired 0 ignore update 0 into streamt4 as select _wstart, first(a), b, c, ta, tb from st interval(1s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791213000,2,3,4,1.1); @@ -301,7 +301,7 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams5 trigger at_once ignore expired 0 ignore update 0 into streamt5 as select _wstart, b, c, ta, tb, max(b) from t1 interval(1s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791211000,1,2,3,1.0); sql insert into t1 values(1648791213000,2,3,4,1.1); @@ -382,7 +382,7 @@ sql create stream streams9 trigger at_once ignore expired 0 ignore update 0 into sql create stream streams10 trigger at_once ignore expired 0 ignore update 0 into streamt10 as select ts, b, c, last(c), ta, tb from st partition by tbname event_window start with d = 0 end with d = 9; sql create stream streams11 trigger at_once ignore expired 1 ignore update 0 watermark 100s into streamt11 as select ts, b, c, last(c), ta, tb from st partition by tbname count_window(2); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791211000,1,2,3,0); sql insert into t1 values(1648791213000,2,3,4,0); diff --git a/tests/script/tsim/stream/checkStreamSTable.sim b/tests/script/tsim/stream/checkStreamSTable.sim index 873fb3f060..d3fc77fbf0 100644 --- a/tests/script/tsim/stream/checkStreamSTable.sim +++ b/tests/script/tsim/stream/checkStreamSTable.sim @@ -22,7 +22,8 @@ sql create table t2 using st tags(2,2,2); sql create stable result.streamt0(ts timestamp,a int,b int) tags(ta int,tb varchar(100),tc int); sql create stream streams0 trigger at_once into result.streamt0 tags(tb) as select _wstart, count(*) c1, max(a) c2 from st partition by tbname tb interval(10s); -sleep 500 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,2,2,3); @@ -108,7 +109,8 @@ sql create table t2 using st tags(2,2,2); sql create stable result1.streamt1(ts timestamp,a int,b int,c int) tags(ta varchar(100),tb int,tc int); sql create stream streams1 trigger at_once into result1.streamt1(ts,c,a,b) tags(ta) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by tbname as ta interval(10s); -sleep 500 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,10,20,30); sql insert into t2 values(1648791213000,40,50,60); @@ -198,7 +200,8 @@ sql_error create stream streams2 trigger at_once into result2.streamt2 as selec # column dest 3, source 2 sql create stream streams2 trigger at_once into result2.streamt2(ts, a) tags(ta) as select _wstart, count(*) c1 from st partition by tbname as ta interval(10s); -sleep 500 + +run tsim/stream/checkTaskStatus.sim print ===== step5 @@ -215,7 +218,8 @@ sql create table t2 using st tags(4,5,6); sql create stable result3.streamt3(ts timestamp,a int,b int,c int, d int) tags(ta int,tb int,tc int); sql create stream streams3 trigger at_once into result3.streamt3(ts,c,a,b) as select _wstart, count(*) c1, max(a),min(b) c2 from st interval(10s); -sleep 500 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,10,20,30); sql insert into t2 values(1648791213000,40,50,60); @@ -295,7 +299,8 @@ sql create table t2 using st tags(4,5,6); sql create stable result4.streamt4(ts timestamp,a int,b int,c int, d int) tags(tg1 int,tg2 int,tg3 int); sql create stream streams4 trigger at_once into result4.streamt4(ts,c,a,b) tags(tg2, tg3, tg1) subtable( concat("tbl-", cast(tg1 as varchar(10)) ) ) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, tc as tg3 interval(10s); -sleep 500 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,10,20,30); sql insert into t2 values(1648791213000,40,50,60); @@ -380,7 +385,8 @@ sql create table t2 using st tags(4,5,6); sql create stable result5.streamt5(ts timestamp,a int,b int,c int, d int) tags(tg1 int,tg2 int,tg3 int); sql create stream streams5 trigger at_once into result5.streamt5(ts,c,a,b) tags(tg2, tg3, tg1) subtable( concat("tbl-", cast(tg3 as varchar(10)) ) ) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, a as tg3 session(ts, 10s); -sleep 500 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,NULL,NULL,NULL); @@ -465,7 +471,8 @@ sql create stream streams8 trigger at_once into streamt8 as select _wstart as sql drop stream streams8; sql create stream streams71 trigger at_once into streamt8(ts, c2) tags(group_id)as select _wstart, count(*) from t1 partition by tbname as group_id interval(10s); -sleep 500 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791233000,1,2,3,1.0); diff --git a/tests/script/tsim/stream/checkStreamSTable1.sim b/tests/script/tsim/stream/checkStreamSTable1.sim index b25e582104..94991f72d9 100644 --- a/tests/script/tsim/stream/checkStreamSTable1.sim +++ b/tests/script/tsim/stream/checkStreamSTable1.sim @@ -16,36 +16,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2 from st interval(1s) ; -print ====check task status start +run tsim/stream/checkTaskStatus.sim -$loop_count = 0 - -loopCheck: - -sleep 1000 - -$loop_count = $loop_count + 1 -if $loop_count == 30 then - return -1 -endi - -print 1 select * from information_schema.ins_stream_tasks; -sql select * from information_schema.ins_stream_tasks; - -if $rows == 0 then - print rows=$rows - goto loopCheck -endi - -print 1 select * from information_schema.ins_stream_tasks where status != "ready"; -sql select * from information_schema.ins_stream_tasks where status != "ready"; - -if $rows != 0 then - print rows=$rows - goto loopCheck -endi - -print ====check task status end sql insert into t1 values(1648791211000,1,2,3); sql insert into t1 values(1648791212000,2,2,3); @@ -77,36 +49,8 @@ sql alter table streamt1 add column c3 double; print create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2, avg(b) c3 from st interval(1s) ; sql create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2, avg(b) c3 from st interval(1s) ; -print ====check task status start +run tsim/stream/checkTaskStatus.sim -$loop_count = 0 - -loopCheck1: - -sleep 1000 - -$loop_count = $loop_count + 1 -if $loop_count == 30 then - return -1 -endi - -print 1 select * from information_schema.ins_stream_tasks; -sql select * from information_schema.ins_stream_tasks; - -if $rows == 0 then - print rows=$rows - goto loopCheck1 -endi - -print 1 select * from information_schema.ins_stream_tasks where status != "ready"; -sql select * from information_schema.ins_stream_tasks where status != "ready"; - -if $rows != 0 then - print rows=$rows - goto loopCheck1 -endi - -print ====check task status end sql insert into t2 values(1648791213000,1,2,3); sql insert into t1 values(1648791214000,1,2,3); diff --git a/tests/script/tsim/stream/checkpointInterval0.sim b/tests/script/tsim/stream/checkpointInterval0.sim index 1c212eb2a7..a548f05c82 100644 --- a/tests/script/tsim/stream/checkpointInterval0.sim +++ b/tests/script/tsim/stream/checkpointInterval0.sim @@ -15,6 +15,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from t1 interval(10s); sql create stream streams1 trigger window_close IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1, sum(a) from t1 interval(10s); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213001,2,2,3,1.1); diff --git a/tests/script/tsim/stream/checkpointInterval1.sim b/tests/script/tsim/stream/checkpointInterval1.sim index 36f361ad64..c7426411cd 100644 --- a/tests/script/tsim/stream/checkpointInterval1.sim +++ b/tests/script/tsim/stream/checkpointInterval1.sim @@ -15,7 +15,7 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from st interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3,1.0); @@ -59,6 +59,8 @@ system sh/stop_dnodes.sh system sh/exec.sh -n dnode1 -s start +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213002,3,2,3,1.1); sql insert into t2 values(1648791223003,4,2,3,1.1); diff --git a/tests/script/tsim/stream/checkpointSession0.sim b/tests/script/tsim/stream/checkpointSession0.sim index 1d503806c5..c5e1cecf93 100644 --- a/tests/script/tsim/stream/checkpointSession0.sim +++ b/tests/script/tsim/stream/checkpointSession0.sim @@ -14,6 +14,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from t1 session(ts, 10s); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213001,2,2,3,1.1); diff --git a/tests/script/tsim/stream/checkpointSession1.sim b/tests/script/tsim/stream/checkpointSession1.sim index 9720803145..0427585122 100644 --- a/tests/script/tsim/stream/checkpointSession1.sim +++ b/tests/script/tsim/stream/checkpointSession1.sim @@ -15,7 +15,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from st session(ts, 10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0); diff --git a/tests/script/tsim/stream/checkpointState0.sim b/tests/script/tsim/stream/checkpointState0.sim index 3836721212..f81486e3b9 100644 --- a/tests/script/tsim/stream/checkpointState0.sim +++ b/tests/script/tsim/stream/checkpointState0.sim @@ -14,6 +14,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from t1 state_window(b); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213001,2,2,3,1.1); diff --git a/tests/script/tsim/stream/count0.sim b/tests/script/tsim/stream/count0.sim index 5f5ec72275..b392363533 100644 --- a/tests/script/tsim/stream/count0.sim +++ b/tests/script/tsim/stream/count0.sim @@ -11,7 +11,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791213001,9,2,2,1.1); @@ -80,7 +82,9 @@ sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,t sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(3) -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791213001,9,2,2,1.1); @@ -192,7 +196,9 @@ sql insert into t1 values(1648791213009,0,3,3,1.0); sleep 500 sql create stream streams3 trigger at_once FILL_HISTORY 1 IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt3 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791223000,0,1,1,1.0); sql insert into t1 values(1648791223001,9,2,2,1.1); diff --git a/tests/script/tsim/stream/count2.sim b/tests/script/tsim/stream/count2.sim index 2558bd1072..3a413442a5 100644 --- a/tests/script/tsim/stream/count2.sim +++ b/tests/script/tsim/stream/count2.sim @@ -11,7 +11,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213001,9,2,2,1.1); sql insert into t1 values(1648791213009,0,3,3,1.0); @@ -140,7 +142,9 @@ sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,t sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(3) -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213001,9,2,2,1.1); diff --git a/tests/script/tsim/stream/count3.sim b/tests/script/tsim/stream/count3.sim index f04996cdaa..c034bd6495 100644 --- a/tests/script/tsim/stream/count3.sim +++ b/tests/script/tsim/stream/count3.sim @@ -11,7 +11,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(3); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791213001,9,2,2,1.1); diff --git a/tests/script/tsim/stream/countSliding0.sim b/tests/script/tsim/stream/countSliding0.sim index 82c54649b2..794a8ce347 100644 --- a/tests/script/tsim/stream/countSliding0.sim +++ b/tests/script/tsim/stream/countSliding0.sim @@ -11,7 +11,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,1,1,1.0); sleep 100 @@ -241,7 +243,9 @@ sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,t sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams2 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname count_window(4, 2); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,1,1,1.0); sleep 100 diff --git a/tests/script/tsim/stream/countSliding1.sim b/tests/script/tsim/stream/countSliding1.sim index 6759ab7abd..43f5a44cb6 100644 --- a/tests/script/tsim/stream/countSliding1.sim +++ b/tests/script/tsim/stream/countSliding1.sim @@ -11,7 +11,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,1,1,1.0); sleep 100 diff --git a/tests/script/tsim/stream/countSliding2.sim b/tests/script/tsim/stream/countSliding2.sim index 8841283c81..6c6a51ac76 100644 --- a/tests/script/tsim/stream/countSliding2.sim +++ b/tests/script/tsim/stream/countSliding2.sim @@ -11,7 +11,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(4, 2); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,1,1,1.0); sleep 100 diff --git a/tests/script/tsim/stream/deleteInterval.sim b/tests/script/tsim/stream/deleteInterval.sim index 11e5ee39d2..9fa706d188 100644 --- a/tests/script/tsim/stream/deleteInterval.sim +++ b/tests/script/tsim/stream/deleteInterval.sim @@ -17,7 +17,9 @@ sql create database test vgroups 1; sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 interval(10s); -sleep 500 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sleep 1000 @@ -195,7 +197,9 @@ sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); -sleep 500 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); @@ -422,7 +426,9 @@ sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); -sleep 500 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); diff --git a/tests/script/tsim/stream/deleteScalar.sim b/tests/script/tsim/stream/deleteScalar.sim index 9a634a5c2d..45ebacba84 100644 --- a/tests/script/tsim/stream/deleteScalar.sim +++ b/tests/script/tsim/stream/deleteScalar.sim @@ -10,7 +10,8 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select ts, a, b from t1 partition by a; -sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0); @@ -94,7 +95,8 @@ sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 subtable(concat("aaa-", cast( a as varchar(10) ))) as select ts, a, b from t1 partition by a; -sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0); @@ -178,7 +180,8 @@ sql use test2; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 subtable("aaa-a") as select ts, a, b from t1; -sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0); diff --git a/tests/script/tsim/stream/deleteSession.sim b/tests/script/tsim/stream/deleteSession.sim index 18ff56ee3b..cdb43c7672 100644 --- a/tests/script/tsim/stream/deleteSession.sim +++ b/tests/script/tsim/stream/deleteSession.sim @@ -18,7 +18,8 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 session(ts, 5s); -sleep 2000 +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sleep 1000 @@ -195,7 +196,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); -sleep 2000 +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); @@ -426,7 +428,9 @@ sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); -sleep 2000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791210000,1,1,1,NULL); sql insert into t1 values(1648791210001,2,2,2,NULL); @@ -538,7 +542,9 @@ sql create table t2 using st tags(2,2,2); print create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname session(ts, 2s); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname session(ts, 2s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791210000,1,2,3); sql insert into t1 values(1648791220000,2,2,3); diff --git a/tests/script/tsim/stream/deleteState.sim b/tests/script/tsim/stream/deleteState.sim index 8ca5e0d507..04c57c6483 100644 --- a/tests/script/tsim/stream/deleteState.sim +++ b/tests/script/tsim/stream/deleteState.sim @@ -18,6 +18,8 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(a); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sleep 1000 sql delete from t1 where ts = 1648791213000; @@ -195,6 +197,8 @@ sql create table t2 using st tags(2,2,2); print create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname state_window(c); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart, count(*) c1 from st partition by tbname state_window(c); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791210000,1,2,1); sql insert into t1 values(1648791220000,2,2,2); sql insert into t1 values(1648791221000,2,2,2); diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index cc8b7227d7..7f206b78d8 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -19,7 +19,8 @@ sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); @@ -439,7 +440,8 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create stream stream_t2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 20s into streamtST1 as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; -sleep 2000 +run tsim/stream/checkTaskStatus.sim + sql insert into ts1 values(1648791211000,1,2,3); sql insert into ts1 values(1648791222001,2,2,3); sql insert into ts2 values(1648791211000,1,2,3); @@ -475,7 +477,8 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ; -sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql insert into ts1 values(1648791211000,1,2,3); sleep 50 @@ -527,6 +530,8 @@ sql create table ddd using st tags(4,2,2); sql create stream streams1 IGNORE EXPIRED 0 IGNORE UPDATE 0 fill_history 0 watermark 3s into streamst subtable(c) as select _wstart, c , count(*) c1, last_row(b) c2 from st partition by c interval(1s) ; +run tsim/stream/checkTaskStatus.sim + sql insert into aaa values(1648791221001,2,2,"/a1/aa/aa"); sql insert into bbb values(1648791221001,2,2,"/a1/aa/aa"); sql insert into ccc values(1648791221001,2,2,"/a1/aa/aa"); diff --git a/tests/script/tsim/stream/distributeIntervalRetrive0.sim b/tests/script/tsim/stream/distributeIntervalRetrive0.sim index 39c3353429..5569f4267b 100644 --- a/tests/script/tsim/stream/distributeIntervalRetrive0.sim +++ b/tests/script/tsim/stream/distributeIntervalRetrive0.sim @@ -45,7 +45,7 @@ sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 delete_mark 10s into streamtST1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); @@ -267,7 +267,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 delete_mark 20s into streamt1 as select _wstart as c0, count(*) c1, count(a) c2 from st interval(10s) ; -sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791211000,1,2,3); sql insert into t1 values(1262275200000,2,2,3); diff --git a/tests/script/tsim/stream/distributeMultiLevelInterval0.sim b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim index 3ac1fbdd2b..15cc0d2484 100644 --- a/tests/script/tsim/stream/distributeMultiLevelInterval0.sim +++ b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim @@ -21,36 +21,7 @@ sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4 from st interval(10s); -print ====check task status start - -$loop_count = 0 - -loopCheck: - -sleep 1000 - -$loop_count = $loop_count + 1 -if $loop_count == 30 then - return -1 -endi - -print 1 select * from information_schema.ins_stream_tasks; -sql select * from information_schema.ins_stream_tasks; - -if $rows == 0 then - print rows=$rows - goto loopCheck -endi - -print 1 select * from information_schema.ins_stream_tasks where status != "ready"; -sql select * from information_schema.ins_stream_tasks where status != "ready"; - -if $rows != 0 then - print rows=$rows - goto loopCheck -endi - -print ====check task status end +run tsim/stream/checkTaskStatus.sim sql insert into ts1 values(1648791213000,1,1,3,4.1); sql insert into ts1 values(1648791223000,2,2,3,1.1); diff --git a/tests/script/tsim/stream/distributeSession0.sim b/tests/script/tsim/stream/distributeSession0.sim index 9a112c2cdf..749b9bd273 100644 --- a/tests/script/tsim/stream/distributeSession0.sim +++ b/tests/script/tsim/stream/distributeSession0.sim @@ -41,36 +41,7 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST as select _wstart, count(*) c1, sum(a) c2 , max(b) c3 from st session(ts, 10s) ; -print ====check task status start - -$loop_count = 0 - -loopCheck: - -sleep 1000 - -$loop_count = $loop_count + 1 -if $loop_count == 30 then - return -1 -endi - -print 1 select * from information_schema.ins_stream_tasks; -sql select * from information_schema.ins_stream_tasks; - -if $rows == 0 then - print rows=$rows - goto loopCheck -endi - -print 1 select * from information_schema.ins_stream_tasks where status != "ready"; -sql select * from information_schema.ins_stream_tasks where status != "ready"; - -if $rows != 0 then - print rows=$rows - goto loopCheck -endi - -print ====check task status end +run tsim/stream/checkTaskStatus.sim sql insert into ts1 values(1648791211000,1,1,1) (1648791211005,1,1,1); sql insert into ts2 values(1648791221004,1,2,3) (1648791221008,2,2,3); @@ -117,36 +88,7 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create stream stream_t2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST2 as select _wstart, count(*) c1, sum(a) c2 , max(b) c3 from st partition by a session(ts, 10s) ; -print ====check task status start - -$loop_count = 0 - -loopCheck1: - -sleep 1000 - -$loop_count = $loop_count + 1 -if $loop_count == 30 then - return -1 -endi - -print 1 select * from information_schema.ins_stream_tasks; -sql select * from information_schema.ins_stream_tasks; - -if $rows == 0 then - print rows=$rows - goto loopCheck1 -endi - -print 1 select * from information_schema.ins_stream_tasks where status != "ready"; -sql select * from information_schema.ins_stream_tasks where status != "ready"; - -if $rows != 0 then - print rows=$rows - goto loopCheck1 -endi - -print ====check task status end +run tsim/stream/checkTaskStatus.sim sql insert into ts1 values(1648791201000,1,1,1) (1648791210000,1,1,1); sql insert into ts1 values(1648791211000,2,1,1) (1648791212000,2,1,1); diff --git a/tests/script/tsim/stream/event0.sim b/tests/script/tsim/stream/event0.sim index 57116bbc79..bcb8fac27d 100644 --- a/tests/script/tsim/stream/event0.sim +++ b/tests/script/tsim/stream/event0.sim @@ -11,7 +11,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 event_window start with a = 0 end with a = 9; -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791223001,9,2,2,1.1); sql insert into t1 values(1648791213009,0,3,3,1.0); @@ -157,7 +159,9 @@ sql use test2; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 event_window start with a = 0 end with b = 9; -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,0,1,1,1.0); sql insert into t1 values(1648791213009,1,2,2,2.1); sql insert into t1 values(1648791223000,0,9,9,9.0); @@ -205,7 +209,9 @@ sql use test3; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 event_window start with a = 0 end with b = 9; -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791233009,1,2,2,2.1); diff --git a/tests/script/tsim/stream/event1.sim b/tests/script/tsim/stream/event1.sim index f3b0e4eb01..19d77abc82 100644 --- a/tests/script/tsim/stream/event1.sim +++ b/tests/script/tsim/stream/event1.sim @@ -11,7 +11,9 @@ sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 event_window start with a = 0 end with b = 9; -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791233000,0,1,1,1.0); sql insert into t1 values(1648791243000,1,9,9,9.0); @@ -54,7 +56,9 @@ sql use test2; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 event_window start with a = 0 end with b = 9; -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791223000,0,3,3,3.0); @@ -240,7 +244,9 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname event_window start with a = 0 end with b = 9; -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791223000,0,3,3,3.0); diff --git a/tests/script/tsim/stream/event2.sim b/tests/script/tsim/stream/event2.sim index 9fc7615fb8..2b21ac84c5 100644 --- a/tests/script/tsim/stream/event2.sim +++ b/tests/script/tsim/stream/event2.sim @@ -35,7 +35,8 @@ sleep 1000 sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 fill_history 1 into streamt0 as select _wstart as s, count(*) c1, sum(b), max(c), _wend as e from st partition by tbname event_window start with a = 0 end with b = 9; -sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791253000,1,9,13,2.0); sql insert into t2 values(1648791253000,1,9,14,2.0); diff --git a/tests/script/tsim/stream/eventtmp.sim b/tests/script/tsim/stream/eventtmp.sim index 52ca4fd9ef..392fb23615 100644 --- a/tests/script/tsim/stream/eventtmp.sim +++ b/tests/script/tsim/stream/eventtmp.sim @@ -22,7 +22,8 @@ sql insert into t4 values(1648791243000,1,9,12,2.0); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 fill_history 1 into streamt0 as select _wstart as s, count(*) c1, sum(b), max(c), _wend as e from st partition by tbname event_window start with a = 0 end with b = 9; -sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql insert into t4 values(1648791253000,1,9,16,2.0); diff --git a/tests/script/tsim/stream/fillHistoryBasic1.sim b/tests/script/tsim/stream/fillHistoryBasic1.sim index d2417a73ab..57193012b0 100644 --- a/tests/script/tsim/stream/fillHistoryBasic1.sim +++ b/tests/script/tsim/stream/fillHistoryBasic1.sim @@ -18,7 +18,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream stream1 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223001,2,2,3,1.1); diff --git a/tests/script/tsim/stream/fillHistoryBasic2.sim b/tests/script/tsim/stream/fillHistoryBasic2.sim index b3dd947eb9..2fffa1e3e5 100644 --- a/tests/script/tsim/stream/fillHistoryBasic2.sim +++ b/tests/script/tsim/stream/fillHistoryBasic2.sim @@ -81,7 +81,8 @@ sql insert into ts3 values(1648791243006,4,2,3,3.1) (1648791213001,1,52,13,1.0) sql create stream stream_t1 trigger at_once fill_history 1 watermark 1d IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim + $loop_count = 0 loop1: @@ -213,6 +214,8 @@ sql insert into ts2 values(1648791222001,2,2,3); sql create stream stream_t2 trigger at_once fill_history 1 watermark 20s IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST1 as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; +run tsim/stream/checkTaskStatus.sim + $loop_count = 0 loop2: sql select * from streamtST1; @@ -243,7 +246,8 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create stream stream_t3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ; -sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql insert into ts1 values(1648791211000,1,2,3); sleep 50 diff --git a/tests/script/tsim/stream/fillHistoryBasic3.sim b/tests/script/tsim/stream/fillHistoryBasic3.sim index fb3776fbd3..ca109843c0 100644 --- a/tests/script/tsim/stream/fillHistoryBasic3.sim +++ b/tests/script/tsim/stream/fillHistoryBasic3.sim @@ -19,7 +19,8 @@ sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); sql create stream streams2 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); -sleep 3000 +run tsim/stream/checkTaskStatus.sim + $loop_count = 0 diff --git a/tests/script/tsim/stream/fillHistoryBasic4.sim b/tests/script/tsim/stream/fillHistoryBasic4.sim index 3e1ea68b4f..9fbf45bbeb 100644 --- a/tests/script/tsim/stream/fillHistoryBasic4.sim +++ b/tests/script/tsim/stream/fillHistoryBasic4.sim @@ -31,6 +31,8 @@ sql insert into t1 values(1648791213004,4,2,3,4.1); print create stream stream2 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s) sql create stream stream2 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); +run tsim/stream/checkTaskStatus.sim + $loop_count = 0 loop0: diff --git a/tests/script/tsim/stream/fillHistoryTransform.sim b/tests/script/tsim/stream/fillHistoryTransform.sim index fe58b76b78..5933f7754e 100644 --- a/tests/script/tsim/stream/fillHistoryTransform.sim +++ b/tests/script/tsim/stream/fillHistoryTransform.sim @@ -23,6 +23,8 @@ sql insert into t1 values(1648791213000,10,2,3,1.0); sql create stream stream0 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, sum(a) from t1 interval(10s); +run tsim/stream/checkTaskStatus.sim + $loop_count = 0 loop00: @@ -153,6 +155,8 @@ sql insert into t1 values(1648791213000,10,2,3,1.0); sql create stream stream1 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, sum(a) from st interval(10s); +run tsim/stream/checkTaskStatus.sim + $loop_count = 0 loop00: @@ -283,6 +287,8 @@ sql insert into t1 values(1648791213000,10,2,3,1.0); sql create stream stream2 trigger at_once fill_history 1 IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, sum(a) from st partition by ta interval(10s); +run tsim/stream/checkTaskStatus.sim + $loop_count = 0 loop00: diff --git a/tests/script/tsim/stream/fillIntervalDelete0.sim b/tests/script/tsim/stream/fillIntervalDelete0.sim index f63417411a..6ba3cc31e1 100644 --- a/tests/script/tsim/stream/fillIntervalDelete0.sim +++ b/tests/script/tsim/stream/fillIntervalDelete0.sim @@ -21,6 +21,9 @@ sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 in sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); sql create stream streams5 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt5 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,1,1,1.0,'aaa'); sleep 1000 @@ -262,6 +265,8 @@ sql create stream streams8 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 in sql create stream streams9 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt9 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); sql create stream streams10 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt10 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791210000,1,1,1,1.0,'aaa'); sql insert into t1 values(1648791217000,1,1,1,1.0,'aaa'); diff --git a/tests/script/tsim/stream/fillIntervalDelete1.sim b/tests/script/tsim/stream/fillIntervalDelete1.sim index 088dd9bdb7..11fe67bbe7 100644 --- a/tests/script/tsim/stream/fillIntervalDelete1.sim +++ b/tests/script/tsim/stream/fillIntervalDelete1.sim @@ -24,6 +24,8 @@ sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 in sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); sql create stream streams5 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt5 as select _wstart as ts, max(a), sum(b), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791210000,0,0,0,0.0,'aaa'); sql insert into t1 values(1648791213000,1,1,1,1.0,'bbb'); sql insert into t1 values(1648791215000,5,5,5,5.0,'ccc'); @@ -227,6 +229,8 @@ sql create stream streams8 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 in sql create stream streams9 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt9 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); sql create stream streams10 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt10 as select _wstart as ts, max(a), sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791210000,1,1,1,1.0,'aaa'); sql insert into t1 values(1648791215000,6,8,8,8.0,'bbb'); sql insert into t1 values(1648791220000,11,10,10,10.0,'ccc'); @@ -359,6 +363,7 @@ sql create stream streams13 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 i sql create stream streams14 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt14 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(prev); sql create stream streams15 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt15 as select _wstart as ts, avg(a), count(*), timezone(), to_iso8601(1) from t1 where ts >= 1648791210000 and ts < 1648791240000 interval(1s) fill(linear); +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,1,1,1.0,'aaa'); sql insert into t1 values(1648791210001,1,1,1,1.0,'aaa'); diff --git a/tests/script/tsim/stream/fillIntervalLinear.sim b/tests/script/tsim/stream/fillIntervalLinear.sim index 94a401ae8f..2d866b1e0a 100644 --- a/tests/script/tsim/stream/fillIntervalLinear.sim +++ b/tests/script/tsim/stream/fillIntervalLinear.sim @@ -17,6 +17,9 @@ sql create database test1 vgroups 1; sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart as ts, max(a)+sum(c), avg(b), first(s), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791216000,5,5,5,5.0,'bbb'); sql insert into t1 values(1648791210000,1,1,1,1.0,'ccc') (1648791219000,2,2,2,2.0,'ddd') (1648791222000,3,3,3,3.0,'eee'); @@ -206,6 +209,9 @@ sql create database test2 vgroups 1; sql use test2; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart as ts, max(a)+sum(c), avg(b), first(s), count(*) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(linear); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791210000,1,1,1,1.0,'ccc') (1648791219000,2,2,2,2.0,'ddd') (1648791222000,3,3,3,3.0,'eee'); sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791216000,5,5,5,5.0,'bbb'); @@ -394,6 +400,9 @@ sql create database test3 vgroups 1; sql use test3; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart as ts, max(a), b+c, s, b+1, 1 from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(linear); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791215000,1,1,1,1.0,'aaa'); sql insert into t1 values(1648791217000,2,2,2,2.0,'bbb'); sql insert into t1 values(1648791211000,3,3,3,3.0,'ccc'); diff --git a/tests/script/tsim/stream/fillIntervalPartitionBy.sim b/tests/script/tsim/stream/fillIntervalPartitionBy.sim index 193b34f743..4e19d79408 100644 --- a/tests/script/tsim/stream/fillIntervalPartitionBy.sim +++ b/tests/script/tsim/stream/fillIntervalPartitionBy.sim @@ -24,6 +24,8 @@ sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 in sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(prev); sql create stream streams5 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt5 as select _wstart as ts, max(a) c1, sum(b), count(*) from st where ts >= 1648791210000 and ts < 1648791261000 partition by ta interval(1s) fill(linear); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791210000,0,0,0,0.0,'aaa'); sql insert into t1 values(1648791213000,1,1,1,1.0,'bbb'); sql insert into t1 values(1648791215000,5,5,5,5.0,'ccc'); diff --git a/tests/script/tsim/stream/fillIntervalPrevNext.sim b/tests/script/tsim/stream/fillIntervalPrevNext.sim index 0bbdd78f3c..4e752c6a5c 100644 --- a/tests/script/tsim/stream/fillIntervalPrevNext.sim +++ b/tests/script/tsim/stream/fillIntervalPrevNext.sim @@ -17,6 +17,9 @@ sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart as ts, count(*) c1, max(a)+min(c), avg(b) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791215000,5,5,5,5.0,'aaa'); sql insert into t1 values(1648791211000,1,1,1,1.0,'aaa') (1648791217000,2,2,2,2.0,'aaa') (1648791220000,3,3,3,3.0,'aaa'); @@ -265,6 +268,9 @@ sql use test5; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); sql create stream streams5 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt5 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(prev); sql create stream streams6 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt6 as select _wstart as ts, count(*) c1, max(a)+min(c), avg(b) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(next); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791211000,1,1,1,1.0,'aaa') (1648791217000,2,2,2,2.0,'aaa') (1648791220000,3,3,3,3.0,'aaa'); sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791215000,5,5,5,5.0,'aaa'); diff --git a/tests/script/tsim/stream/fillIntervalPrevNext1.sim b/tests/script/tsim/stream/fillIntervalPrevNext1.sim index 91b2e64ad9..b6c24f7a0f 100644 --- a/tests/script/tsim/stream/fillIntervalPrevNext1.sim +++ b/tests/script/tsim/stream/fillIntervalPrevNext1.sim @@ -18,6 +18,9 @@ sql use test7; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); sql create stream streams7 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt7 as select _wstart as ts, max(a), b+c, s from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(prev); sql create stream streams8 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt8 as select _wstart as ts, max(a), 1, b+1 from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(next); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791215000,1,1,1,1.0,'aaa'); sql insert into t1 values(1648791217000,2,2,2,2.0,'bbb'); sql insert into t1 values(1648791211000,3,3,3,3.0,'ccc'); diff --git a/tests/script/tsim/stream/fillIntervalRange.sim b/tests/script/tsim/stream/fillIntervalRange.sim index e5316e6a1e..1b3a2d3e2a 100644 --- a/tests/script/tsim/stream/fillIntervalRange.sim +++ b/tests/script/tsim/stream/fillIntervalRange.sim @@ -14,6 +14,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20));; sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart ts, count(*) c1 from t1 interval(1s) fill(NULL); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791211000,1,2,3,1.0,'aaa'); sleep 100 sql insert into t1 values(1648795308000,1,2,3,1.0,'aaa'); @@ -131,6 +134,8 @@ sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 in print create stream streams2 trigger at_once into streamt2 as select _wstart ts, max(a) c1 from t1 interval(1s) fill(prev); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart ts, max(a) c1 from t1 interval(1s) fill(prev); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791211000,1,2,3,1.0,'aaa'); sleep 100 sql insert into t1 values(1648795308000,1,2,3,1.0,'aaa'); diff --git a/tests/script/tsim/stream/fillIntervalValue.sim b/tests/script/tsim/stream/fillIntervalValue.sim index dac73129e9..321e1bb708 100644 --- a/tests/script/tsim/stream/fillIntervalValue.sim +++ b/tests/script/tsim/stream/fillIntervalValue.sim @@ -15,6 +15,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20));; sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart ts, count(*) c1 from t1 where ts > 1648791210000 and ts < 1648791413000 interval(10s) fill(value, 100); sql create stream streams1a trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamta as select _wstart ts, count(*) c1 from t1 where ts > 1648791210000 and ts < 1648791413000 interval(10s) fill(value_f, 100); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0,'aaa'); sleep 100 sql insert into t1 values(1648791233000,1,2,3,1.0,'aaa'); @@ -147,6 +150,9 @@ sql create database test2 vgroups 1; sql use test2; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart as ts, count(*) c1, max(b)+sum(a) from t1 where ts >= 1648791210000 and ts < 1648791261000 interval(1s) fill(value, 100,200); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791211000,1,1,1,1.0,'aaa') (1648791217000,2,2,2,2.0,'aaa') (1648791220000,3,3,3,3.0,'aaa'); sql insert into t1 values(1648791213000,4,4,4,4.0,'aaa') (1648791215000,5,5,5,5.0,'aaa'); @@ -282,6 +288,8 @@ sql use test3; sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart as ts, max(b), a+b, c from t1 where ts >= 1648791200000 and ts < 1648791261000 interval(10s) sliding(3s) fill(value, 100,200,300); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791220000,1,1,1,1.0,'aaa'); sleep 100 sql insert into t1 values(1648791260000,1,1,1,1.0,'aaa'); @@ -473,6 +481,9 @@ sql create table t2 using st tags(2,2,2); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart ts, count(*) c1, concat(tbname, 'aaa') as pname, timezone() from st where ts > 1648791000000 and ts < 1648793000000 partition by tbname interval(10s) fill(NULL); sql create stream streams4a trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4a as select _wstart ts, count(*) c1, concat(tbname, 'aaa') as pname, timezone() from st where ts > 1648791000000 and ts < 1648793000000 partition by tbname interval(10s) fill(NULL_F); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0,'aaa'); sql insert into t1 values(1648791233000,1,2,3,1.0,'aaa'); sql insert into t1 values(1648791273000,1,2,3,1.0,'aaa'); diff --git a/tests/script/tsim/stream/ignoreCheckUpdate.sim b/tests/script/tsim/stream/ignoreCheckUpdate.sim index 108c845e4d..d73c42d3d7 100644 --- a/tests/script/tsim/stream/ignoreCheckUpdate.sim +++ b/tests/script/tsim/stream/ignoreCheckUpdate.sim @@ -16,6 +16,8 @@ print create stream streams0 trigger at_once IGNORE EXPIRED 0 ignore update 1 in sql create stream streams0 trigger at_once IGNORE EXPIRED 0 ignore update 1 into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 interval(10s); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,1,1); sql insert into t1 values(1648791213000,2,2,2); @@ -78,6 +80,8 @@ print create stream streams1 trigger at_once ignore update 1 into streamt1 as se sql create stream streams1 trigger at_once ignore update 1 into streamt1 as select _wstart c1, count(*) c2, max(b) c3 from t1 session(ts, 10s); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,1,1); sql insert into t1 values(1648791213000,2,2,2); @@ -141,6 +145,8 @@ print create stream streams2 trigger at_once ignore update 1 into streamt2 as se sql create stream streams2 trigger at_once ignore update 1 into streamt2 as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(c); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,1,1); sql insert into t1 values(1648791213000,2,2,1); @@ -206,7 +212,7 @@ print create stream streams3 trigger at_once ignore update 1 into streamt3 as se sql create stream streams3 trigger at_once ignore update 1 into streamt3 as select _wstart c1, count(*) c2, max(b) c3 from st interval(10s); -sleep 2000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,1,1); sql insert into t1 values(1648791213000,2,2,2); diff --git a/tests/script/tsim/stream/ignoreExpiredData.sim b/tests/script/tsim/stream/ignoreExpiredData.sim index 864f8caea5..3624a8af32 100644 --- a/tests/script/tsim/stream/ignoreExpiredData.sim +++ b/tests/script/tsim/stream/ignoreExpiredData.sim @@ -48,6 +48,9 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 1 into streamt1 as select _wstart, count(*) c1, sum(a) c3 from t1 interval(10s); sql create stream streams2 trigger at_once IGNORE EXPIRED 1 into streamt2 as select _wstart, count(*) c1, sum(a) c3 from t1 session(ts,10s); sql create stream streams3 trigger at_once IGNORE EXPIRED 1 into streamt3 as select _wstart, count(*) c1, sum(a) c3 from t1 state_window(a); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223001,1,2,3,1.1); sql insert into t1 values(1648791233002,2,2,3,2.1); @@ -114,6 +117,9 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 1 into streamtST1 as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; sql create stream stream_t2 trigger at_once IGNORE EXPIRED 1 into streamtST2 as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st session(ts, 10s) ; + +run tsim/stream/checkTaskStatus.sim + sql insert into ts1 values(1648791211000,1,2,3); sleep 1000 sql insert into ts1 values(1648791222001,2,2,3); @@ -197,7 +203,8 @@ sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,3,3); sql create table ts4 using st tags(4,4,4); sql create stream streams_21 trigger at_once IGNORE EXPIRED 1 into streamt_21 as select _wstart, count(*) c1 from st interval(10s) ; -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into ts1 values(1648791211000,1,2,3); sql insert into ts1 values(1648791211001,2,2,3); diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim index 9a660741e7..6a4c7913a7 100644 --- a/tests/script/tsim/stream/partitionby.sim +++ b/tests/script/tsim/stream/partitionby.sim @@ -14,7 +14,8 @@ sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test0.streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by ta,tb,tc interval(10s); -sleep 500 +run tsim/stream/checkTaskStatus.sim + sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); @@ -70,6 +71,7 @@ sql create table ts3 using st tags(1,4,5); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, count(a) c2 from st partition by ta,tb,tc interval(10s); +run tsim/stream/checkTaskStatus.sim sql insert into ts1 values(1648791211000,1,2,3); @@ -100,6 +102,9 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create stream stream_t2 trigger at_once watermark 20s IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ; + +run tsim/stream/checkTaskStatus.sim + sql insert into ts1 values(1648791211000,1,2,3,1); sql insert into ts1 values(1648791222001,2,2,3,2); sql insert into ts2 values(1648791211000,1,2,3,3); diff --git a/tests/script/tsim/stream/partitionby1.sim b/tests/script/tsim/stream/partitionby1.sim index 24c588d410..306cb20e1f 100644 --- a/tests/script/tsim/stream/partitionby1.sim +++ b/tests/script/tsim/stream/partitionby1.sim @@ -13,7 +13,7 @@ sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by tbname interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); @@ -47,6 +47,7 @@ sql create table ts3 using st tags(1,4,5); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, count(a) c2 from st partition by tbname interval(10s); +run tsim/stream/checkTaskStatus.sim sql insert into ts1 values(1648791211000,1,2,3); @@ -77,6 +78,9 @@ sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create stream stream_t2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by tbname interval(10s) ; + +run tsim/stream/checkTaskStatus.sim + sql insert into ts1 values(1648791211000,1,2,3,1); sql insert into ts1 values(1648791222001,2,2,3,2); sql insert into ts2 values(1648791211000,1,2,3,3); diff --git a/tests/script/tsim/stream/partitionbyColumnOther.sim b/tests/script/tsim/stream/partitionbyColumnOther.sim index 8e6c0c1f23..fdb6be9fc2 100644 --- a/tests/script/tsim/stream/partitionbyColumnOther.sim +++ b/tests/script/tsim/stream/partitionbyColumnOther.sim @@ -14,7 +14,7 @@ sql use test0; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 watermark 100s into streamt0 subtable("aaa-a") as select _wstart, count(*) from t1 partition by a count_window(10); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,0,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0); @@ -72,7 +72,7 @@ sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 subtable("aaa-a") as select _wstart, count(*) from t1 partition by a event_window start with b = 2 end with b = 2; -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,0,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0); diff --git a/tests/script/tsim/stream/partitionbyColumnSession.sim b/tests/script/tsim/stream/partitionbyColumnSession.sim index 1daa033399..0cf9013434 100644 --- a/tests/script/tsim/stream/partitionbyColumnSession.sim +++ b/tests/script/tsim/stream/partitionbyColumnSession.sim @@ -18,6 +18,8 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a session(ts, 5s); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -198,6 +200,8 @@ sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b session(ts, 5s); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,1,2,1,1.0); @@ -284,7 +288,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); @@ -481,7 +486,8 @@ sql create table t3 using st tags(2,2,2); sql create table t4 using st tags(2,2,2); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into test.streamt4 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s); -sleep 2000 +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,2,2,3,1.0); sql insert into t2 values(1648791213000,2,2,3,1.0); sql insert into t3 values(1648791213000,2,2,3,1.0); @@ -568,7 +574,7 @@ sql use test5; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams6 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt6 subtable("aaa-a") as select _wstart, count(*) from t1 partition by a session(ts, 10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,0,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0); diff --git a/tests/script/tsim/stream/partitionbyColumnState.sim b/tests/script/tsim/stream/partitionbyColumnState.sim index d741426786..85fb8b2ff3 100644 --- a/tests/script/tsim/stream/partitionbyColumnState.sim +++ b/tests/script/tsim/stream/partitionbyColumnState.sim @@ -13,6 +13,8 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a state_window(b); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); @@ -188,6 +190,8 @@ sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d int); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart c1, count(*) c2, max(d) c3, _group_key(a+b) c4 from t1 partition by a+b state_window(c); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); sql insert into t1 values(1648791213000,1,2,1,1); @@ -273,7 +277,7 @@ sql use test2; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams6 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt6 subtable("aaa-a") as select _wstart, count(*) from t1 partition by a session(ts, 10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,0,2,3,1.0); sql insert into t1 values(1648791213001,1,2,3,1.0); diff --git a/tests/script/tsim/stream/pauseAndResume.sim b/tests/script/tsim/stream/pauseAndResume.sim index b9d6e141be..adb85fa8c4 100644 --- a/tests/script/tsim/stream/pauseAndResume.sim +++ b/tests/script/tsim/stream/pauseAndResume.sim @@ -17,7 +17,8 @@ sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select _wstart, count(*) c1, sum(a) c3 from st interval(10s); -sleep 2000 + +run tsim/stream/checkTaskStatus.sim sql_error create stream stream1_same_dst into streamt1 as select _wstart, count(*) c1, sum(a) c3 from st interval(10s); @@ -126,6 +127,7 @@ sql_error create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE sql create stream if not exists streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt2 as select _wstart, count(*) c1, sum(a) c3 from t1 interval(10s); sleep 1000 +run tsim/stream/checkTaskStatus.sim sql pause stream streams2; @@ -264,6 +266,8 @@ sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 wate sql create stream streams5 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt5 as select _wstart, count(*) c1, sum(a) c3 from ts1 interval(10s); sleep 1000 +run tsim/stream/checkTaskStatus.sim + sql pause stream streams3; sql insert into ts1 values(1648791213001,1,12,3,1.0); @@ -337,6 +341,7 @@ sql create table ts4 using st tags(4,2,2); sql create stream streams6 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt6 as select _wstart, count(*) c1 from st interval(10s); sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); diff --git a/tests/script/tsim/stream/scalar.sim b/tests/script/tsim/stream/scalar.sim index ec55c3da66..45d734b76f 100644 --- a/tests/script/tsim/stream/scalar.sim +++ b/tests/script/tsim/stream/scalar.sim @@ -17,6 +17,8 @@ sql create stream streams0 into streamt0 as select ts c1, a, abs(b) c4 from t1 sql create stream streams1 into streamt1 as select ts c1, a, abs(b) c4 from t1; sql create stream streams2 into streamt2 as select ts c1, a, abs(b) c4 from st partition by tbname; +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,1,1,1); sql insert into t1 values(1648791213001,1,1,1,1); sql insert into t1 values(1648791213002,1,1,1,1); diff --git a/tests/script/tsim/stream/schedSnode.sim b/tests/script/tsim/stream/schedSnode.sim index e481fff83d..0fe53b0687 100644 --- a/tests/script/tsim/stream/schedSnode.sim +++ b/tests/script/tsim/stream/schedSnode.sim @@ -22,7 +22,7 @@ sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into target.streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); diff --git a/tests/script/tsim/stream/streamPrimaryKey0.sim b/tests/script/tsim/stream/streamPrimaryKey0.sim index 8905078a3e..94f62c4719 100644 --- a/tests/script/tsim/stream/streamPrimaryKey0.sim +++ b/tests/script/tsim/stream/streamPrimaryKey0.sim @@ -19,7 +19,7 @@ sql create table streamt2(ts timestamp, a int primary key, b bigint ) tags(ta va sql create stream streams0 trigger at_once ignore expired 0 ignore update 0 into streamt0 as select _wstart, count(*) c1, max(b) from t1 interval(1s); sql create stream streams2 trigger at_once ignore expired 0 ignore update 0 into streamt2 tags(ta) as select _wstart, count(*) c1, max(b) from st partition by tbname ta interval(1s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,2,3,1.0); @@ -109,7 +109,7 @@ sql create table streamt5(ts timestamp, a int primary key, b bigint ) tags(ta va sql create stream streams3 trigger at_once ignore expired 0 ignore update 0 into streamt3 as select _wstart, count(*) c1, max(b) from t1 session(ts,1s); sql create stream streams5 trigger at_once ignore expired 0 ignore update 0 into streamt5 tags(ta) as select _wstart, count(*) c1, max(b) from st partition by tbname ta session(ts,1s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,2,3,1.0); @@ -196,7 +196,7 @@ sql create table t1 using st tags("aa", 1, 2); sql create table streamt6(ts timestamp, a int primary key, b bigint ) tags(ta varchar(100),tb int,tc int); sql create stream streams6 trigger at_once ignore expired 0 ignore update 0 into streamt6 tags(ta) as select _wstart, count(*) c1, max(b) from st partition by tbname ta state_window(a); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,2,3,1.0); diff --git a/tests/script/tsim/stream/streamPrimaryKey1.sim b/tests/script/tsim/stream/streamPrimaryKey1.sim index 71971166d6..751a1ba7c7 100644 --- a/tests/script/tsim/stream/streamPrimaryKey1.sim +++ b/tests/script/tsim/stream/streamPrimaryKey1.sim @@ -18,7 +18,7 @@ sql create table streamt1(ts timestamp, a int primary key, b bigint ) tags(ta va sql create stream streams1 trigger at_once ignore expired 0 ignore update 0 into streamt1 as select _wstart, count(*) c1, max(b) from st interval(1s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,2,3,1.0); @@ -74,7 +74,7 @@ sql create table t1 using st tags("aa", 1, 2); sql create table streamt3(ts timestamp, a int primary key, b bigint ) tags(ta varchar(100),tb int,tc int); sql create stream streams3 trigger at_once ignore expired 0 ignore update 0 into streamt3 as select _wstart, count(*) c1, max(b) from st session(ts,1s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,2,3,1.0); diff --git a/tests/script/tsim/stream/streamPrimaryKey2.sim b/tests/script/tsim/stream/streamPrimaryKey2.sim index c47dc2bb4b..d6687c038d 100644 --- a/tests/script/tsim/stream/streamPrimaryKey2.sim +++ b/tests/script/tsim/stream/streamPrimaryKey2.sim @@ -18,7 +18,7 @@ sql create table streamt1(ts timestamp, a int primary key, b bigint ) tags(ta va sql create stream streams1 trigger at_once ignore expired 0 ignore update 0 into streamt1 tags(ta) as select _wstart, count(*) c1, max(b) from st partition by tbname ta EVENT_WINDOW start with a = 1 end with a = 3; -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,2,3,1.0); sql insert into t1 values(1648791210010,3,2,3,1.0); @@ -83,7 +83,7 @@ sql create table t1 using st tags("aa", 1, 2); sql create table streamt3(ts timestamp, a int primary key, b bigint ) tags(ta varchar(100),tb int,tc int); sql create stream streams3 trigger at_once ignore expired 1 ignore update 0 WATERMARK 1000s into streamt3 tags(ta) as select _wstart, count(*) c1, max(b) from st partition by tbname ta COUNT_WINDOW(2); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,2,3,1.0); diff --git a/tests/script/tsim/stream/streamPrimaryKey3.sim b/tests/script/tsim/stream/streamPrimaryKey3.sim index 73038b6732..e6041b107b 100644 --- a/tests/script/tsim/stream/streamPrimaryKey3.sim +++ b/tests/script/tsim/stream/streamPrimaryKey3.sim @@ -17,7 +17,7 @@ sql create table t1 using st tags("aa", 1, 2); sql create stream streams1 trigger at_once ignore expired 0 ignore update 0 into streamt1(ts, a primary key, b) as select ts, a, b from t1 partition by b; -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,2,3,1.0); sql insert into t1 values(1648791210000,2,4,3,1.0); @@ -79,7 +79,7 @@ sql create table t1 using st tags("aa", 1, 2); sql create stream streams2 trigger at_once ignore expired 0 ignore update 0 into streamt2(ts, a primary key, b) as select _wstart, max(b), count(*) from t1 partition by b interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,2,1,1.0); sql insert into t1 values(1648791210000,2,4,2,1.0); @@ -226,7 +226,7 @@ sql create table t1 using st tags("aa", 1, 2); sql create stream streams3 trigger at_once ignore expired 0 ignore update 0 into streamt3(ts, a primary key, b) as select _wstart, max(b), count(*) from t1 partition by b session(ts, 10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791210000,1,2,1,1.0); sql insert into t1 values(1648791210000,2,4,2,1.0); From b594decc14f94b9b85e72f92f6f2a03b70a0f60f Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jul 2024 18:54:27 +0800 Subject: [PATCH 21/37] adj ci --- tests/script/tsim/stream/session0.sim | 9 +++++++++ tests/script/tsim/stream/session1.sim | 8 ++++++++ tests/script/tsim/stream/state0.sim | 10 +++++----- tests/script/tsim/stream/state1.sim | 6 +++++- tests/script/tsim/stream/triggerInterval0.sim | 2 ++ tests/script/tsim/stream/triggerSession0.sim | 2 ++ tests/script/tsim/stream/udTableAndTag0.sim | 15 ++++++++++----- tests/script/tsim/stream/udTableAndTag1.sim | 14 +++++++++----- tests/script/tsim/stream/udTableAndTag2.sim | 14 +++++++++----- tests/script/tsim/stream/windowClose.sim | 14 +++++++++----- 10 files changed, 68 insertions(+), 26 deletions(-) diff --git a/tests/script/tsim/stream/session0.sim b/tests/script/tsim/stream/session0.sim index 3221a38de1..38adae4261 100644 --- a/tests/script/tsim/stream/session0.sim +++ b/tests/script/tsim/stream/session0.sim @@ -18,6 +18,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double,id int); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a), max(a), min(d), stddev(a), last(a), first(d), max(id) s from t1 session(ts,10s); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL,1); sql insert into t1 values(1648791223001,10,2,3,1.1,2); sql insert into t1 values(1648791233002,3,2,3,2.1,3); @@ -180,6 +183,9 @@ sql create database test2 vgroups 1; sql use test2; sql create table t2(ts timestamp, a int, b int , c int, d double, id int); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt2 as select _wstart,apercentile(a,30) c1, apercentile(a,70), apercentile(a,20,"t-digest") c2, apercentile(a,60,"t-digest") c3, max(id) c4 from t2 session(ts,10s); + +run tsim/stream/checkTaskStatus.sim + sql insert into t2 values(1648791213001,1,1,3,1.0,1); sql insert into t2 values(1648791213002,2,2,6,3.4,2); sql insert into t2 values(1648791213003,4,9,3,4.8,3); @@ -236,6 +242,9 @@ sql create stream streams4 trigger at_once watermark 1d IGNORE EXPIRED 0 IGNORE # sql create stream streams7 trigger at_once watermark 1d IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt7 as select _wstart, spread(a), elapsed(ts), hyperloglog(a) from t1 session(ts,10s); sql create stream streams7 trigger at_once watermark 1d IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt7 as select _wstart, spread(a), hyperloglog(a) from t1 session(ts,10s); # sql create stream streams8 trigger at_once watermark 1d IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt8 as select _wstart, histogram(a,"user_input", "[1,3,5,7]", 1), histogram(a,"user_input", "[1,3,5,7]", 0) from t1 session(ts,10s); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213001,1,1,1,1.0); sql insert into t1 values(1648791213002,2,3,2,3.4); sql insert into t1 values(1648791213003,4,9,3,4.8); diff --git a/tests/script/tsim/stream/session1.sim b/tests/script/tsim/stream/session1.sim index aae17053b2..04fd61d1d2 100644 --- a/tests/script/tsim/stream/session1.sim +++ b/tests/script/tsim/stream/session1.sim @@ -18,6 +18,9 @@ sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double,id int); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a), min(b), max(id) s from t1 session(ts,10s); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791210000,1,1,1,1.1,1); sql insert into t1 values(1648791220000,2,2,2,2.1,2); sql insert into t1 values(1648791230000,3,3,3,3.1,3); @@ -237,6 +240,9 @@ sql create database test1 vgroups 1; sql use test1; sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1 from t1 where a > 5 session(ts, 5s); + +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0); $loop_count = 0 @@ -334,6 +340,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams4 trigger at_once ignore update 0 ignore expired 0 into streamt4 as select _wstart, count(*) c1, count(a) c2 from st session(ts, 2s) ; +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791255100,1,2,3); sql insert into t1 values(1648791255300,1,2,3); diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 37081ad1ae..71b3efab5e 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -21,7 +21,7 @@ print create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,3,1.0,2); @@ -459,7 +459,7 @@ print create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791212000,2,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,3,1.0,1); @@ -509,7 +509,7 @@ print create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1, sum(b) c3 from t1 state_window(a); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791212000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,2,2,3,1.0,1); @@ -564,7 +564,7 @@ print create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 sql create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 (ts, c1) values (1668073288209, 11); sql insert into t1 (ts, c1) values (1668073288210, 11); @@ -755,7 +755,7 @@ print create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 i sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into b values(1648791213000,NULL,NULL,NULL,NULL); sql select * from streamt order by c1, c2, c3; diff --git a/tests/script/tsim/stream/state1.sim b/tests/script/tsim/stream/state1.sim index e6f7a81851..8a48212181 100644 --- a/tests/script/tsim/stream/state1.sim +++ b/tests/script/tsim/stream/state1.sim @@ -22,6 +22,8 @@ print create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1 from t1 state_window(a); +run tsim/stream/checkTaskStatus.sim + sql insert into t1(ts) values(1648791213000); $loop_count = 0 @@ -105,6 +107,8 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); print create stream streams2 trigger at_once watermark 1000s IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(*) c1, count(d) c2 from t1 partition by b state_window(a) sql create stream streams2 trigger at_once watermark 1000s IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(*) c1, count(d) c2 from t1 partition by b state_window(a); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213010,1,2,3,1.1); @@ -164,7 +168,7 @@ sql insert into t1 values(1648791213001,1,2,3,1.0); print create stream streams3 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into streamt3 as select _wstart, max(a), count(*) c1 from t1 state_window(a); sql create stream streams3 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into streamt3 as select _wstart, max(a), count(*) c1 from t1 state_window(a); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791203000,2,2,3,1.0); diff --git a/tests/script/tsim/stream/triggerInterval0.sim b/tests/script/tsim/stream/triggerInterval0.sim index 14c2093ba1..0077c70625 100644 --- a/tests/script/tsim/stream/triggerInterval0.sim +++ b/tests/script/tsim/stream/triggerInterval0.sim @@ -17,6 +17,8 @@ sql use test sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream streams1 trigger window_close IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213001,1,2,3,1.0); sleep 1000 sql select * from streamt; diff --git a/tests/script/tsim/stream/triggerSession0.sim b/tests/script/tsim/stream/triggerSession0.sim index c74169d133..283d993e4c 100644 --- a/tests/script/tsim/stream/triggerSession0.sim +++ b/tests/script/tsim/stream/triggerSession0.sim @@ -17,6 +17,8 @@ sql use test; sql create table t2(ts timestamp, a int, b int , c int, d double); sql create stream streams2 trigger window_close IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t2 session(ts, 10s); +run tsim/stream/checkTaskStatus.sim + sql insert into t2 values(1648791213000,1,2,3,1.0); sql insert into t2 values(1648791222999,1,2,3,1.0); sql insert into t2 values(1648791223000,1,2,3,1.0); diff --git a/tests/script/tsim/stream/udTableAndTag0.sim b/tests/script/tsim/stream/udTableAndTag0.sim index e8b51a3846..4a8cb371db 100644 --- a/tests/script/tsim/stream/udTableAndTag0.sim +++ b/tests/script/tsim/stream/udTableAndTag0.sim @@ -22,7 +22,8 @@ sql create table t2 using st tags(2,2,2); #sql_error create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result.streamt SUBTABLE(concat("aaa-", tbname)) as select _wstart, count(*) c1 from st partition by tbname interval(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,1,2,3); @@ -81,7 +82,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as cc interval(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,1,2,3); @@ -167,7 +169,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,1,2,3); @@ -269,7 +272,8 @@ sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(3,3,3); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); @@ -374,7 +378,8 @@ sql create table t2 using st tags("2",2,2); sql create table t3 using st tags("3",3,3); sql create stream streams6 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result6.streamt6 TAGS(dd int) as select _wstart, count(*) c1 from st partition by concat(ta, "0") as dd, tbname interval(10s); -sleep 1000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); diff --git a/tests/script/tsim/stream/udTableAndTag1.sim b/tests/script/tsim/stream/udTableAndTag1.sim index b109a4dbc9..3c788d99db 100644 --- a/tests/script/tsim/stream/udTableAndTag1.sim +++ b/tests/script/tsim/stream/udTableAndTag1.sim @@ -22,7 +22,8 @@ sql create table t2 using st tags(2,2,2); #sql_error create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result.streamt SUBTABLE( concat("aaa-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by a interval(10s); -sleep 2000 + +run tsim/stream/checkTaskStatus.sim print ===== insert into 1 sql insert into t1 values(1648791213000,1,2,3); @@ -80,7 +81,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as cc interval(10s); -sleep 2000 + +run tsim/stream/checkTaskStatus.sim print ===== insert into 2 sql insert into t1 values(1648791213000,1,2,3); @@ -165,7 +167,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as dd, a interval(10s); -sleep 2000 + +run tsim/stream/checkTaskStatus.sim print ===== insert into 3 sql insert into t1 values(1648791213000,1,2,3); @@ -269,7 +272,8 @@ sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(3,3,3); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", dd)) as select _wstart, count(*) c1 from st partition by concat("t", cast(a as varchar(10) ) ) as dd interval(10s); -sleep 2000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); @@ -347,7 +351,7 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stable streamt5(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create stream streams5 trigger at_once ignore expired 0 ignore update 0 into streamt5(ts,b,a) as select _wstart, count(*), 1000 c1 from t1 interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223001,2,2,3,1.1); diff --git a/tests/script/tsim/stream/udTableAndTag2.sim b/tests/script/tsim/stream/udTableAndTag2.sim index 8977dc219b..571a87d450 100644 --- a/tests/script/tsim/stream/udTableAndTag2.sim +++ b/tests/script/tsim/stream/udTableAndTag2.sim @@ -21,7 +21,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); -sleep 2000 + +run tsim/stream/checkTaskStatus.sim print ===== insert into 1 sql insert into t1 values(1648791213000,1,2,3); @@ -96,7 +97,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st interval(10s); -sleep 2000 + +run tsim/stream/checkTaskStatus.sim print ===== insert into 2 sql insert into t1 values(1648791213000,1,2,3); @@ -197,7 +199,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", "1") ) as select _wstart, count(*) c1 from st interval(10s); -sleep 2000 + +run tsim/stream/checkTaskStatus.sim print ===== insert into 3 sql insert into t1 values(1648791213000,1,2,3); @@ -308,7 +311,8 @@ sql create table t2 using st tags(2,2,2); sql create table t3 using st tags(3,3,3); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", "1")) as select _wstart, count(*) c1 from st interval(10s); -sleep 2000 + +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); @@ -382,7 +386,7 @@ sql create stream streams51 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 in sql create stream streams52 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result5.streamt52 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st interval(10s); sql create stream streams53 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into result5.streamt53 TAGS(dd varchar(100)) SUBTABLE(concat("aaa-", "1") ) as select _wstart, count(*) c1 from st interval(10s); -sleep 2000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3); sql insert into t2 values(1648791213000,2,2,3); diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index ce5c57572e..a9c55bd9de 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -20,6 +20,8 @@ sql create table tu2 using st tags(2); sql create stream stream1 trigger window_close into streamt as select _wstart, sum(a) from st interval(10s); +run tsim/stream/checkTaskStatus.sim + sql insert into tu1 values(now, 1); $loop_count = 0 @@ -56,6 +58,8 @@ sql create stream stream9 trigger max_delay 1s into streamt9 as select _wstart, sql create stream stream10 trigger window_close into streamt10 as select _wstart, sum(a) from t1 state_window(b); sql create stream stream11 trigger max_delay 1s into streamt11 as select _wstart, sum(a) from t1 state_window(b); +run tsim/stream/checkTaskStatus.sim + sql insert into t1 values(1648791213000,1,1); sql insert into t1 values(1648791213001,2,1); sql insert into t1 values(1648791213002,3,1); @@ -141,7 +145,7 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream stream13 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223001,2,2,3,1.1); @@ -195,7 +199,7 @@ sql create table t2 using st tags(2,2,2); sql create stream stream14 trigger max_delay 2s into streamt14 as select _wstart, sum(a), now from st partition by tbname interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223000,2,2,3,1.1); @@ -297,7 +301,7 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream stream15 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 session(ts, 10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791233001,2,2,3,1.1); @@ -365,7 +369,7 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream stream16 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 state_window(a); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791233001,2,2,3,1.1); @@ -433,7 +437,7 @@ sql create table t1(ts timestamp, a int, b int , c int, d double); sql create stream stream17 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 event_window start with a = 1 end with a = 9; -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791213001,9,2,3,1.0); From 2ca67269229593d0d1e51ee2d70496d9ca65cba4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 18 Jul 2024 22:52:32 +0800 Subject: [PATCH 22/37] refactor: check return value for each function. --- include/common/tdatablock.h | 2 +- include/libs/stream/tstream.h | 10 +- source/common/src/tdatablock.c | 12 +- source/libs/executor/src/exchangeoperator.c | 2 +- source/libs/executor/src/streamfilloperator.c | 9 +- source/libs/stream/inc/streamInt.h | 8 +- source/libs/stream/src/streamBackendRocksdb.c | 44 ++--- source/libs/stream/src/streamCheckStatus.c | 134 +++++++------ source/libs/stream/src/streamCheckpoint.c | 132 +++++++------ source/libs/stream/src/streamData.c | 61 +++--- source/libs/stream/src/streamDispatch.c | 184 ++++++++++-------- source/libs/stream/src/streamExec.c | 134 ++++++++----- source/libs/stream/src/streamHb.c | 12 +- source/libs/stream/src/streamMeta.c | 47 +++-- source/libs/stream/src/streamQueue.c | 61 ++++-- source/libs/stream/src/streamSched.c | 52 ++--- source/libs/stream/src/streamStartHistory.c | 18 +- source/libs/stream/src/streamTask.c | 46 ++--- source/libs/stream/src/streamTaskSm.c | 46 ++--- source/libs/stream/src/streamTimer.c | 2 +- source/libs/stream/src/streamUtil.c | 26 +++ 21 files changed, 580 insertions(+), 462 deletions(-) create mode 100644 source/libs/stream/src/streamUtil.c diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index ec998e9365..c271f118ae 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -253,7 +253,7 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc); SSDataBlock* createDataBlock(); -void* blockDataDestroy(SSDataBlock* pBlock); +void blockDataDestroy(SSDataBlock* pBlock); void blockDataFreeRes(SSDataBlock* pBlock); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); SSDataBlock* createSpecialDataBlock(EStreamType type); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 5327428f5b..dff212b15c 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -629,7 +629,7 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask); int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); -int32_t streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo); +void streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo); #if 0 SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId); #endif @@ -659,10 +659,10 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask); int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt); int32_t streamExecTask(SStreamTask* pTask); -int32_t streamResumeTask(SStreamTask* pTask); +void streamResumeTask(SStreamTask* pTask); int32_t streamTrySchedExec(SStreamTask* pTask); int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType); -int32_t streamTaskResumeInFuture(SStreamTask* pTask); +void streamTaskResumeInFuture(SStreamTask* pTask); void streamTaskClearSchedIdleInfo(SStreamTask* pTask); void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime); @@ -675,8 +675,8 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp); // check downstream status -int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask); -int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id); +void streamTaskStartMonitorCheckRsp(SStreamTask* pTask); +void streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id); void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo); // fill-history task diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 965246326a..43bac56f93 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1523,9 +1523,9 @@ void blockDataFreeRes(SSDataBlock* pBlock) { memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); } -void* blockDataDestroy(SSDataBlock* pBlock) { +void blockDataDestroy(SSDataBlock* pBlock) { if (pBlock == NULL) { - return NULL; + return; } if (IS_VAR_DATA_TYPE(pBlock->info.pks[0].type)) { @@ -1535,7 +1535,6 @@ void* blockDataDestroy(SSDataBlock* pBlock) { blockDataFreeRes(pBlock); taosMemoryFreeClear(pBlock); - return NULL; } // todo remove it @@ -2468,19 +2467,18 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, char* cname) { if (stbFullName[0] == 0) { - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; + return TSDB_CODE_INVALID_PARA; } SArray* tags = taosArrayInit(0, sizeof(SSmlKv)); if (tags == NULL) { - return TSDB_CODE_FAILED; + return TSDB_CODE_OUT_OF_MEMORY; } if (cname == NULL) { terrno = TSDB_CODE_INVALID_PARA; taosArrayDestroy(tags); - return TSDB_CODE_FAILED; + return terrno; } int8_t type = TSDB_DATA_TYPE_UBIGINT; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 059e1f2663..a6a273cce8 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -607,7 +607,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo blockDataAppendColInfo(pBlock, &idata); } - blockDecode(pBlock, pStart); + (void) blockDecode(pBlock, pStart); blockDataEnsureCapacity(pRes, pBlock->info.rows); // data from mnode diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 5f188c2c8a..384c763063 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -126,9 +126,12 @@ static void destroyStreamFillOperatorInfo(void* param) { SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); - pInfo->pRes = blockDataDestroy(pInfo->pRes); - pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + blockDataDestroy(pInfo->pRes); + pInfo->pRes = NULL; + blockDataDestroy(pInfo->pSrcBlock); + pInfo->pSrcBlock = NULL; + blockDataDestroy(pInfo->pDelRes); + pInfo->pDelRes = NULL; taosArrayDestroy(pInfo->matchInfo.pList); pInfo->matchInfo.pList = NULL; taosMemoryFree(pInfo); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index fd248861e3..2fe86817e3 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -208,7 +208,7 @@ int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ); void streamQueueClose(SStreamQueue* pQueue, int32_t taskId); void streamQueueProcessSuccess(SStreamQueue* queue); void streamQueueProcessFail(SStreamQueue* queue); -void* streamQueueNextItem(SStreamQueue* pQueue); +void streamQueueNextItem(SStreamQueue* pQueue, SStreamQueueItem** pItem); void streamFreeQitem(SStreamQueueItem* data); int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); @@ -226,7 +226,7 @@ int32_t streamTaskDownloadCheckpointData(const char* id, char* path); int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask); int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask); -int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, +void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId, SEpSet* pEpset, int64_t checkpointId); int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId, int64_t checkpointId, SRpcMsg* pMsg); @@ -236,6 +236,10 @@ typedef int32_t (*__stream_async_exec_fn_t)(void* param); int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code); void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock); +void streamMutexLock(TdThreadMutex *pMutex); +void streamMutexUnlock(TdThreadMutex *pMutex); +void streamMutexDestroy(TdThreadMutex *pMutex); + #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 8b87019ee0..86144a3099 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -887,8 +887,8 @@ _EXIT: rocksdb_options_destroy(opts); rocksdb_cache_destroy(cache); rocksdb_env_destroy(env); - taosThreadMutexDestroy(&pHandle->mutex); - taosThreadMutexDestroy(&pHandle->cfMutex); + streamMutexDestroy(&pHandle->mutex); + streamMutexDestroy(&pHandle->cfMutex); taosHashCleanup(pHandle->cfInst); tdListFree(pHandle->list); taosMemoryFree(pHandle); @@ -923,9 +923,9 @@ void streamBackendCleanup(void* arg) { } tdListFree(pHandle->list); - taosThreadMutexDestroy(&pHandle->mutex); + streamMutexDestroy(&pHandle->mutex); - taosThreadMutexDestroy(&pHandle->cfMutex); + streamMutexDestroy(&pHandle->cfMutex); stDebug("destroy stream backend :%p", pHandle); taosMemoryFree(pHandle); return; @@ -1393,7 +1393,7 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { // vnode task->db SStreamMeta* pMeta = arg; - taosThreadMutexLock(&pMeta->backendMutex); + streamMutexLock(&pMeta->backendMutex); void* pIter = taosHashIterate(pMeta->pTaskDbUnique, NULL); int32_t code = 0; @@ -1434,14 +1434,14 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { pIter = taosHashIterate(pMeta->pTaskDbUnique, pIter); } - taosThreadMutexUnlock(&pMeta->backendMutex); + streamMutexUnlock(&pMeta->backendMutex); return code; } int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) { if (pSnapInfo == NULL) return 0; SStreamMeta* pMeta = arg; int32_t code = 0; - taosThreadMutexLock(&pMeta->backendMutex); + streamMutexLock(&pMeta->backendMutex); char buf[128] = {0}; for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) { @@ -1457,7 +1457,7 @@ int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) { taskDbUnRefChkp(*pTaskDb, pSnap->chkpId); } - taosThreadMutexUnlock(&pMeta->backendMutex); + streamMutexUnlock(&pMeta->backendMutex); return 0; } #ifdef BUILD_NO_CALL @@ -1697,17 +1697,17 @@ int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId, int64_t processVer) SListNode* streamBackendAddCompare(void* backend, void* arg) { SBackendWrapper* pHandle = (SBackendWrapper*)backend; SListNode* node = NULL; - taosThreadMutexLock(&pHandle->mutex); + streamMutexLock(&pHandle->mutex); node = tdListAdd(pHandle->list, arg); - taosThreadMutexUnlock(&pHandle->mutex); + streamMutexUnlock(&pHandle->mutex); return node; } void streamBackendDelCompare(void* backend, void* arg) { SBackendWrapper* pHandle = (SBackendWrapper*)backend; SListNode* node = NULL; - taosThreadMutexLock(&pHandle->mutex); + streamMutexLock(&pHandle->mutex); node = tdListPopNode(pHandle->list, arg); - taosThreadMutexUnlock(&pHandle->mutex); + streamMutexUnlock(&pHandle->mutex); if (node) { streamStateDestroyCompar(node->data); taosMemoryFree(node); @@ -2461,9 +2461,9 @@ int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** sta void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { STaskDbWrapper* p = pTaskDb; - taosThreadMutexLock(&p->mutex); + streamMutexLock(&p->mutex); p->chkpId = chkpId; - taosThreadMutexUnlock(&p->mutex); + streamMutexUnlock(&p->mutex); } STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { @@ -2622,7 +2622,7 @@ void taskDbDestroy(void* pDb, bool flush) { taosMemoryFree(wrapper->pCfOpts); taosMemoryFree(wrapper->pCfParams); - taosThreadMutexDestroy(&wrapper->mutex); + streamMutexDestroy(&wrapper->mutex); taskDbDestroyChkpOpt(wrapper); @@ -2957,7 +2957,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { SBackendWrapper* handle = backend; SBackendCfWrapper* pBackendCfWrapper = taosMemoryCalloc(1, sizeof(SBackendCfWrapper)); - taosThreadMutexLock(&handle->cfMutex); + streamMutexLock(&handle->cfMutex); RocksdbCfInst** ppInst = taosHashGet(handle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); if (ppInst != NULL && *ppInst != NULL) { RocksdbCfInst* inst = *ppInst; @@ -2970,7 +2970,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { pBackendCfWrapper->param = inst->param; pBackendCfWrapper->pBackend = handle; pBackendCfWrapper->pComparNode = inst->pCompareNode; - taosThreadMutexUnlock(&handle->cfMutex); + streamMutexUnlock(&handle->cfMutex); pBackendCfWrapper->backendId = pState->streamBackendRid; memcpy(pBackendCfWrapper->idstr, pState->pTdbState->idstr, sizeof(pState->pTdbState->idstr)); @@ -2987,7 +2987,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { inst->rOpt = NULL; return 0; } - taosThreadMutexUnlock(&handle->cfMutex); + streamMutexUnlock(&handle->cfMutex); char* err = NULL; int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); @@ -3046,14 +3046,14 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { stInfo("start to close state on backend: %p", pHandle); - taosThreadMutexLock(&pHandle->cfMutex); + streamMutexLock(&pHandle->cfMutex); RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, wrapper->idstr, strlen(pState->pTdbState->idstr) + 1); if (ppInst != NULL && *ppInst != NULL) { RocksdbCfInst* inst = *ppInst; taosMemoryFree(inst); taosHashRemove(pHandle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); } - taosThreadMutexUnlock(&pHandle->cfMutex); + streamMutexUnlock(&pHandle->cfMutex); char* status[] = {"close", "drop"}; stInfo("start to %s state %p on backendWrapper %p %s", status[remove == false ? 0 : 1], pState, wrapper, @@ -3085,7 +3085,7 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { return -1; } - taosThreadMutexLock(&wrapper->mutex); + streamMutexLock(&wrapper->mutex); rocksdb_column_family_handle_t* cf = wrapper->pCf[idx]; if (cf == NULL) { @@ -3100,7 +3100,7 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { wrapper->pCf[idx] = cf; } } - taosThreadMutexUnlock(&wrapper->mutex); + streamMutexUnlock(&wrapper->mutex); } return idx; diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index e588be0784..5e67f1766f 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -23,10 +23,10 @@ static void processDownstreamReadyRsp(SStreamTask* pTask); static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId); static void rspMonitorFn(void* param, void* tmrId); -static int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs); +static void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs); static int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id); -static int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id); -static int32_t streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id); +static void streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id); +static void streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id); static void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p); static void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList); static void handleNotReadyDownstreamTask(SStreamTask* pTask, SArray* pNotReadyList); @@ -41,9 +41,9 @@ static SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int64_t* oldStage) { SStreamUpstreamEpInfo* pInfo = NULL; - int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; + streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); + if (pInfo == NULL) { + return TSDB_CODE_STREAM_TASK_NOT_EXIST; } *oldStage = pInfo->stage; @@ -65,21 +65,21 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ ", prev:%" PRId64, id, upstreamTaskId, vgId, stage, pInfo->stage); // record the checkpoint failure id and sent to mnode - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); ETaskStatus status = streamTaskGetStatus(pTask).state; if (status == TASK_STATUS__CK) { streamTaskSetFailedCheckpointId(pTask); } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); } if (pInfo->stage != stage) { - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); ETaskStatus status = streamTaskGetStatus(pTask).state; if (status == TASK_STATUS__CK) { streamTaskSetFailedCheckpointId(pTask); } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return TASK_UPSTREAM_NEW_STAGE; } else if (pTask->status.downstreamReady != 1) { @@ -120,7 +120,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, pRange->range.minVer, pRange->range.maxVer, pWindow->skey, pWindow->ekey, req.reqId); - streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet); + (void) streamSendCheckMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet); } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamTaskStartMonitorCheckRsp(pTask); @@ -140,7 +140,7 @@ void streamTaskSendCheckMsg(SStreamTask* pTask) { stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " check downstream task:0x%x (vgId:%d) (shuffle), idx:%d, reqId:0x%" PRIx64, idstr, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, req.reqId); - streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + (void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } else { // for sink task, set it ready directly. stDebug("s-task:%s (vgId:%d) set downstream ready, since no downstream", idstr, pTask->info.nodeId); @@ -265,7 +265,7 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&encoder, (uint8_t*)abuf, len); - tEncodeStreamTaskCheckRsp(&encoder, pRsp); + (void) tEncodeStreamTaskCheckRsp(&encoder, pRsp); tEncoderClear(&encoder); SRpcMsg rspMsg = {.code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = *pRpcInfo}; @@ -274,14 +274,16 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa return 0; } -int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { +void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { + int32_t vgId = pTask->pMeta->vgId; STaskCheckInfo* pInfo = &pTask->taskCheckInfo; - taosThreadMutexLock(&pInfo->checkInfoLock); + + streamMutexLock(&pInfo->checkInfoLock); int32_t code = streamTaskStartCheckDownstream(pInfo, pTask->id.idStr); if (code != TSDB_CODE_SUCCESS) { - taosThreadMutexUnlock(&pInfo->checkInfoLock); - return TSDB_CODE_FAILED; + streamMutexUnlock(&pInfo->checkInfoLock); + return; } /*SStreamTask* p = */ streamMetaAcquireOneTask(pTask); // add task ref here @@ -293,20 +295,18 @@ int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { if (pInfo->checkRspTmr == NULL) { pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer); } else { - taosTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr); + streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor"); } - taosThreadMutexUnlock(&pInfo->checkInfoLock); - return 0; + streamMutexUnlock(&pInfo->checkInfoLock); } -int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) { - taosThreadMutexLock(&pInfo->checkInfoLock); +void streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id) { + streamMutexLock(&pInfo->checkInfoLock); pInfo->stopCheckProcess = 1; - taosThreadMutexUnlock(&pInfo->checkInfoLock); + streamMutexUnlock(&pInfo->checkInfoLock); stDebug("s-task:%s set stop check-rsp monitor flag", id); - return TSDB_CODE_SUCCESS; } void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { @@ -316,21 +316,21 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) { pInfo->pList = NULL; if (pInfo->checkRspTmr != NULL) { - /*bool ret = */ taosTmrStop(pInfo->checkRspTmr); + (void) taosTmrStop(pInfo->checkRspTmr); pInfo->checkRspTmr = NULL; } - taosThreadMutexDestroy(&pInfo->checkInfoLock); + streamMutexDestroy(&pInfo->checkInfoLock); } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void processDownstreamReadyRsp(SStreamTask* pTask) { EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST; - streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); + (void) streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL); int64_t checkTs = pTask->execInfo.checkTs; int64_t readyTs = pTask->execInfo.readyTs; - streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true); + (void) streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, checkTs, readyTs, true); if (pTask->status.taskStatus == TASK_STATUS__HALT) { ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask) && (pTask->info.fillHistory == 0)); @@ -338,21 +338,25 @@ void processDownstreamReadyRsp(SStreamTask* pTask) { // halt it self for count window stream task until the related fill history task completed. stDebug("s-task:%s level:%d initial status is %s from mnode, set it to be halt", pTask->id.idStr, pTask->info.taskLevel, streamTaskGetStatusStr(pTask->status.taskStatus)); - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); + int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_HALT); + if (code != 0) { + // todo: handle error + } } // start the related fill-history task, when current task is ready // not invoke in success callback due to the deadlock. + // todo: let's retry if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr); - streamLaunchFillHistoryTask(pTask); + (void) streamLaunchFillHistoryTask(pTask); } } void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { int32_t vgId = pTask->pMeta->vgId; - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); bool existed = false; for (int i = 0; i < num; ++i) { @@ -365,16 +369,18 @@ void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { if (!existed) { SDownstreamTaskEpset t = {.nodeId = nodeId}; - taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); - + void* p = taosArrayPush(pTask->outputInfo.pNodeEpsetUpdateList, &t); + if (p == NULL) { + // todo let's retry + } stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId, t.nodeId, (num + 1)); } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); } -int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs) { +void streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs) { taosArrayClear(pInfo->pList); if (pOutputInfo->type == TASK_OUTPUT__FIXED_DISPATCH) { @@ -387,7 +393,6 @@ int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOut pInfo->startTs = startTs; pInfo->timeoutStartTs = startTs; pInfo->stopCheckProcess = 0; - return TSDB_CODE_SUCCESS; } SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId) { @@ -403,7 +408,7 @@ SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t taskId) int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, int64_t reqId, int32_t* pNotReady, const char* id) { - taosThreadMutexLock(&pInfo->checkInfoLock); + streamMutexLock(&pInfo->checkInfoLock); SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); if (p != NULL) { @@ -411,7 +416,7 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 " expired check-rsp recv from downstream task:0x%x, discarded", id, reqId, p->reqId, taskId); - taosThreadMutexUnlock(&pInfo->checkInfoLock); + streamMutexUnlock(&pInfo->checkInfoLock); return TSDB_CODE_FAILED; } @@ -425,11 +430,11 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t p->status = status; p->rspTs = rspTs; - taosThreadMutexUnlock(&pInfo->checkInfoLock); + streamMutexUnlock(&pInfo->checkInfoLock); return TSDB_CODE_SUCCESS; } - taosThreadMutexUnlock(&pInfo->checkInfoLock); + streamMutexUnlock(&pInfo->checkInfoLock); stError("s-task:%s unexpected check rsp msg, invalid downstream task:0x%x, reqId:%" PRIx64 " discarded", id, taskId, reqId); return TSDB_CODE_FAILED; @@ -450,9 +455,9 @@ int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id) { return TSDB_CODE_SUCCESS; } -int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id) { +void streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* id) { if (lock) { - taosThreadMutexLock(&pInfo->checkInfoLock); + streamMutexLock(&pInfo->checkInfoLock); } if (pInfo->inCheckProcess) { @@ -474,27 +479,28 @@ int32_t streamTaskCompleteCheckRsp(STaskCheckInfo* pInfo, bool lock, const char* } if (lock) { - taosThreadMutexUnlock(&pInfo->checkInfoLock); + streamMutexUnlock(&pInfo->checkInfoLock); } - - return 0; } -int32_t streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id) { +// todo: retry until success +void streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, int32_t vgId, const char* id) { SDownstreamStatusInfo info = {.taskId = taskId, .status = -1, .vgId = vgId, .reqId = reqId, .rspTs = 0}; - taosThreadMutexLock(&pInfo->checkInfoLock); + streamMutexLock(&pInfo->checkInfoLock); SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); if (p != NULL) { stDebug("s-task:%s check info to task:0x%x already sent", id, taskId); - taosThreadMutexUnlock(&pInfo->checkInfoLock); - return TSDB_CODE_SUCCESS; + streamMutexUnlock(&pInfo->checkInfoLock); + return; } - taosArrayPush(pInfo->pList, &info); + void* px = taosArrayPush(pInfo->pList, &info); + if (px == NULL) { + // todo: retry + } - taosThreadMutexUnlock(&pInfo->checkInfoLock); - return TSDB_CODE_SUCCESS; + streamMutexUnlock(&pInfo->checkInfoLock); } void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { @@ -519,7 +525,7 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId); - streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); + (void) streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); } else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pOutputInfo->shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgs = taosArrayGetSize(vgInfo); @@ -533,7 +539,7 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) (shuffle), idx:%d reqId:0x%" PRIx64, id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, i, p->reqId); - streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + (void) streamSendCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); break; } } @@ -556,12 +562,12 @@ void getCheckRspStatus(STaskCheckInfo* pInfo, int64_t el, int32_t* numOfReady, i if (p->rspTs == 0) { // not response yet ASSERT(p->status == -1); if (el >= CHECK_NOT_RSP_DURATION) { // not receive info for 10 sec. - taosArrayPush(pTimeoutList, &p->taskId); + (void) taosArrayPush(pTimeoutList, &p->taskId); } else { // el < CHECK_NOT_RSP_DURATION (*numOfNotRsp) += 1; // do nothing and continue waiting for their rsp } } else { - taosArrayPush(pNotReadyList, &p->taskId); + (void) taosArrayPush(pNotReadyList, &p->taskId); } } } @@ -676,7 +682,7 @@ void rspMonitorFn(void* param, void* tmrId) { // not record the failed of the current task if try to close current vnode // otherwise, the put of message operation may incur invalid read of message queue. if (!pMeta->closeFlag) { - addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); + (void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); } streamMetaReleaseTask(pMeta, pTask); @@ -692,14 +698,14 @@ void rspMonitorFn(void* param, void* tmrId) { return; } - taosThreadMutexLock(&pInfo->checkInfoLock); + streamMutexLock(&pInfo->checkInfoLock); if (pInfo->notReadyTasks == 0) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat.name, vgId, ref); streamTaskCompleteCheckRsp(pInfo, false, id); - taosThreadMutexUnlock(&pInfo->checkInfoLock); + streamMutexUnlock(&pInfo->checkInfoLock); streamMetaReleaseTask(pMeta, pTask); return; } @@ -723,7 +729,7 @@ void rspMonitorFn(void* param, void* tmrId) { id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); streamTaskCompleteCheckRsp(pInfo, false, id); - taosThreadMutexUnlock(&pInfo->checkInfoLock); + streamMutexUnlock(&pInfo->checkInfoLock); streamMetaReleaseTask(pMeta, pTask); taosArrayDestroy(pNotReadyList); @@ -743,9 +749,9 @@ void rspMonitorFn(void* param, void* tmrId) { id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); streamTaskCompleteCheckRsp(pInfo, false, id); - taosThreadMutexUnlock(&pInfo->checkInfoLock); + streamMutexUnlock(&pInfo->checkInfoLock); - addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); + (void) addDownstreamFailedStatusResultAsync(pTask->pMsgCb, vgId, pTask->id.streamId, pTask->id.taskId); streamMetaReleaseTask(pMeta, pTask); taosArrayDestroy(pNotReadyList); @@ -761,8 +767,8 @@ void rspMonitorFn(void* param, void* tmrId) { handleTimeoutDownstreamTasks(pTask, pTimeoutList); } - taosTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr); - taosThreadMutexUnlock(&pInfo->checkInfoLock); + streamTmrReset(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer, &pInfo->checkRspTmr, vgId, "check-status-monitor"); + streamMutexUnlock(&pInfo->checkInfoLock); stDebug( "s-task:%s vgId:%d continue checking rsp in 300ms, total:%d, notRsp:%d, notReady:%d, fault:%d, timeout:%d, " diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 8de923e900..392f64f524 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -77,6 +77,7 @@ int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int6 } // this message must be put into inputq successfully, continue retrying until it succeeds +// todo must be success int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId, int32_t srcTaskId) { SStreamDataBlock* pCheckpoint = NULL; @@ -119,8 +120,8 @@ int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTri return TSDB_CODE_SUCCESS; } - appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pRsp->checkpointId, pRsp->transId, - pRsp->upstreamTaskId); + (void)appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pRsp->checkpointId, pRsp->transId, + pRsp->upstreamTaskId); return TSDB_CODE_SUCCESS; } @@ -164,7 +165,7 @@ int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStream int32_t code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock); if (code == 0) { ASSERT(pTask->chkInfo.pActiveInfo->dispatchTrigger == false); - streamDispatchStreamBlock(pTask); + code = streamDispatchStreamBlock(pTask); } else { stError("s-task:%s failed to put checkpoint into outputQ, code:%s", pTask->id.idStr, tstrerror(code)); streamFreeQitem((SStreamQueueItem*)pBlock); @@ -184,12 +185,12 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); if (pTask->chkInfo.checkpointId > checkpointId) { stError("s-task:%s vgId:%d current checkpointId:%" PRId64 " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); - code = taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamFreeQitem((SStreamQueueItem*)pBlock); return code; @@ -199,13 +200,15 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock { // send checkpoint-ready msg to upstream SRpcMsg msg = {0}; SStreamUpstreamEpInfo* pInfo = NULL; - code = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; + streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId, &pInfo); + if (pInfo == NULL) { + return TSDB_CODE_STREAM_TASK_NOT_EXIST; } - initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg); - tmsgSendReq(&pInfo->epSet, &msg); + code = initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg); + if (code == TSDB_CODE_SUCCESS) { + (void)tmsgSendReq(&pInfo->epSet, &msg); + } } stWarn( @@ -214,7 +217,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock id, vgId, pBlock->srcTaskId); streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); - code = taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamFreeQitem((SStreamQueueItem*)pBlock); return code; @@ -225,7 +228,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64 " discard", id, vgId, pActiveInfo->activeId, checkpointId); - code = taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamFreeQitem((SStreamQueueItem*)pBlock); return code; @@ -235,7 +238,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, " "checkpointId:%" PRId64 " transId:%d", id, vgId, checkpointId, transId); - code = taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamFreeQitem((SStreamQueueItem*)pBlock); return code; } @@ -250,7 +253,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock ", prev recvTs:%" PRId64 " discard", pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); - code = taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamFreeQitem((SStreamQueueItem*)pBlock); return code; } @@ -259,7 +262,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock } } - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64 ", transId:%d current active checkpointId:%" PRId64, @@ -284,7 +287,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock if (pActiveInfo->pChkptTriggerTmr == NULL) { pActiveInfo->pChkptTriggerTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer); } else { - taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); + streamTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr, vgId, "trigger-recv-monitor"); } } @@ -300,9 +303,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); - continueDispatchCheckpointTriggerBlock(pBlock, pTask); + (void)continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure } else { // only one task exists, no need to dispatch downstream info - appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId, -1); + (void)appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId, + -1); streamFreeQitem((SStreamQueueItem*)pBlock); } } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { @@ -312,8 +316,9 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock pTask->execInfo.checkpoint += 1; } + // todo: handle this // update the child Id for downstream tasks - streamAddCheckpointReadyMsg(pTask, pBlock->srcTaskId, pTask->info.selfChildId, checkpointId); + (void) streamAddCheckpointReadyMsg(pTask, pBlock->srcTaskId, pTask->info.selfChildId, checkpointId); // there are still some upstream tasks not send checkpoint request, do nothing and wait for then if (pActiveInfo->allUpstreamTriggerRecv != 1) { @@ -325,7 +330,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock if (taskLevel == TASK_LEVEL__SINK) { stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, send ready msg to upstream", id, num); streamFreeQitem((SStreamQueueItem*)pBlock); - streamTaskBuildCheckpoint(pTask); + (void)streamTaskBuildCheckpoint(pTask); // todo: not handle error yet } else { // source & agg tasks need to forward the checkpoint msg downwards stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num); @@ -369,7 +374,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId return -1; } - (void) taosThreadMutexLock(&pInfo->lock); + streamMutexLock(&pInfo->lock); // only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList); @@ -391,16 +396,16 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId .transId = pInfo->transId, .streamId = pTask->id.streamId, .downstreamNodeId = downstreamNodeId}; - taosArrayPush(pInfo->pCheckpointReadyRecvList, &info); + (void)taosArrayPush(pInfo->pCheckpointReadyRecvList, &info); } int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList); int32_t transId = pInfo->transId; - (void) taosThreadMutexUnlock(&pInfo->lock); + streamMutexUnlock(&pInfo->lock); if (notReady == 0) { stDebug("s-task:%s all downstream tasks have completed build checkpoint, do checkpoint for current task", id); - appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId, -1); + (void)appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId, -1); } return 0; @@ -411,7 +416,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream int64_t now = taosGetTimestampMs(); int32_t numOfConfirmed = 0; - (void) taosThreadMutexLock(&pInfo->lock); + streamMutexLock(&pInfo->lock); for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) { @@ -432,7 +437,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream stDebug("s-task:%s send checkpoint-ready msg to %d upstream confirmed, checkpointId:%" PRId64, pTask->id.idStr, numOfConfirmed, checkpointId); - (void) taosThreadMutexUnlock(&pInfo->lock); + streamMutexUnlock(&pTask->lock); return TSDB_CODE_SUCCESS; } @@ -440,12 +445,12 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { pTask->chkInfo.startTs = 0; // clear the recorded start time streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks - (void) taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock); + streamMutexLock(&pTask->chkInfo.pActiveInfo->lock); streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo); if (clearChkpReadyMsg) { streamClearChkptReadyMsg(pTask->chkInfo.pActiveInfo); } - (void) taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); + streamMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); } int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) { @@ -455,7 +460,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV const char* id = pTask->id.idStr; SCheckpointInfo* pInfo = &pTask->chkInfo; - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); if (pReq->checkpointId <= pInfo->checkpointId) { stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " checkpointVer:%" PRId64 @@ -463,7 +468,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV " transId:%d ignored", id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer, pReq->transId); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); { // destroy the related fill-history tasks // drop task should not in the meta-lock, and drop the related fill-history task now @@ -531,7 +536,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV return code; } - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamMetaWUnLock(pMeta); // drop task should not in the meta-lock, and drop the related fill-history task now @@ -730,9 +735,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask); } } else { // clear the checkpoint info if failed - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); streamTaskSetFailedCheckpointId(pTask); // set failed checkpoint id before clear the checkpoint info - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%" PRId64, id, ckId); @@ -763,20 +768,20 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { } if (++pActiveInfo->checkCounter < 100) { - taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); + streamTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr, vgId, "trigger-recv-monitor"); return; } pActiveInfo->checkCounter = 0; stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now); - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); SStreamTaskState pState = streamTaskGetStatus(pTask); if (pState.state != TASK_STATUS__CK) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pTask->pMeta, pTask); return; } @@ -787,14 +792,14 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { stDebug("s-task:%s vgId:%d all checkpoint-trigger recv, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pTask->pMeta, pTask); return; } - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); - (void) taosThreadMutexLock(&pActiveInfo->lock); + streamMutexLock(&pActiveInfo->lock); // send msg to retrieve checkpoint trigger msg SArray* pList = pTask->upstreamInfo.pList; @@ -820,19 +825,19 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message streamTaskOpenUpstreamInput(pTask, pInfo->taskId); - taosArrayPush(pNotSendList, pInfo); + (void)taosArrayPush(pNotSendList, pInfo); } } // do send retrieve checkpoint trigger msg to upstream int32_t size = taosArrayGetSize(pNotSendList); - doSendRetrieveTriggerMsg(pTask, pNotSendList); - (void) taosThreadMutexUnlock(&pActiveInfo->lock); + (void)doSendRetrieveTriggerMsg(pTask, pNotSendList); + streamMutexUnlock(&pActiveInfo->lock); // check every 100ms if (size > 0) { stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id); - taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); + streamTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr, vgId, "trigger-recv-monitor"); } else { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref); @@ -863,7 +868,7 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) { SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq)); if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; stError("vgId:%d failed to create msg to retrieve trigger msg for task:%s exec, code:out of memory", vgId, pId); continue; } @@ -880,11 +885,16 @@ int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) { initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, pReq, sizeof(SRetrieveChkptTriggerReq)); code = tmsgSendReq(&pUpstreamTask->epSet, &rpcMsg); - stDebug("s-task:%s vgId:%d send checkpoint-trigger retrieve msg to 0x%x(vgId:%d) checkpointId:%" PRId64, pId, vgId, - pUpstreamTask->taskId, pUpstreamTask->nodeId, checkpointId); + if (code == TSDB_CODE_SUCCESS) { + stDebug("s-task:%s vgId:%d send checkpoint-trigger retrieve msg to 0x%x(vgId:%d) checkpointId:%" PRId64, pId, + vgId, pUpstreamTask->taskId, pUpstreamTask->nodeId, checkpointId); + } else { + stError("s-task:%s vgId:%d failed to send checkpoint-trigger retrieve msg to 0x%x(vgId:%d) checkpointId:%" PRId64, + pId, vgId, pUpstreamTask->taskId, pUpstreamTask->nodeId, checkpointId); + } } - return TSDB_CODE_SUCCESS; + return code; } bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) { @@ -897,9 +907,9 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) return false; } - (void) taosThreadMutexLock(&pInfo->lock); + streamMutexLock(&pInfo->lock); if (!pInfo->dispatchTrigger) { - (void) taosThreadMutexUnlock(&pInfo->lock); + streamMutexUnlock(&pTask->lock); return false; } @@ -921,7 +931,7 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId); } - (void) taosThreadMutexUnlock(&pInfo->lock); + streamMutexUnlock(&pTask->lock); return true; } @@ -945,7 +955,7 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; int64_t now = taosGetTimestampMs(); - (void) taosThreadMutexLock(&pInfo->lock); + streamMutexLock(&pInfo->lock); // outputQ should be empty here ASSERT(streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) == 0); @@ -971,21 +981,21 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { } } - (void) taosThreadMutexUnlock(&pInfo->lock); + streamMutexUnlock(&pTask->lock); } int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) { SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; int32_t num = 0; - (void) taosThreadMutexLock(&pInfo->lock); + streamMutexLock(&pInfo->lock); for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); if (p->recved) { num++; } } - (void) taosThreadMutexUnlock(&pInfo->lock); + streamMutexUnlock(&pTask->lock); return num; } @@ -993,7 +1003,7 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) { SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; int32_t taskId = 0; - (void) taosThreadMutexLock(&pInfo->lock); + streamMutexLock(&pInfo->lock); for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); @@ -1007,7 +1017,7 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) { } } - (void) taosThreadMutexUnlock(&pInfo->lock); + streamMutexUnlock(&pTask->lock); int32_t numOfConfirmed = streamTaskGetNumOfConfirmed(pTask); int32_t total = streamTaskGetNumOfDownstream(pTask); @@ -1184,16 +1194,16 @@ int32_t deleteCheckpointFile(const char* id, const char* name) { int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) { const char* id = pTask->id.idStr; - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); if (pTask->status.sendConsensusChkptId == true) { stDebug("s-task:%s already start to consensus-checkpointId, not start again before it completed", id); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return TSDB_CODE_SUCCESS; } else { pTask->status.sendConsensusChkptId = true; } - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); ASSERT(pTask->pBackend == NULL); pTask->status.requireConsensusChkptId = true; @@ -1207,12 +1217,12 @@ int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask) { return code; } - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); SStreamTaskState p = streamTaskGetStatus(pTask); if (p.state == TASK_STATUS__CK) { code = streamTaskSendCheckpointSourceRsp(pTask); } - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return code; } \ No newline at end of file diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index af4946cf81..c0ee503f77 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -49,7 +49,7 @@ int32_t createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t pInput = p; } - blockDecode(pDataBlock, pInput); + (void) blockDecode(pDataBlock, pInput); if (pRetrieve->compressed && compLen < fullLen) { taosMemoryFree(pInput); @@ -116,11 +116,11 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock return terrno; } - taosArrayPush(pArray, &(SSDataBlock){0}); + (void) taosArrayPush(pArray, &(SSDataBlock){0}); SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); - blockDecode(pDataBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN); + (void) blockDecode(pDataBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); @@ -156,28 +156,31 @@ void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit) { taosFreeQitem(pDataSubmit); } -SStreamMergedSubmit* streamMergedSubmitNew() { - SStreamMergedSubmit* pMerged; +int32_t streamMergedSubmitNew(SStreamMergedSubmit** pSubmit) { + *pSubmit = NULL; - int32_t code = taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0, (void**)&pMerged); + int32_t code = taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0, (void**)pSubmit); if (code) { - terrno = code; - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } - pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); - if (pMerged->submits == NULL) { - taosArrayDestroy(pMerged->submits); - taosFreeQitem(pMerged); - return NULL; + (*pSubmit)->submits = taosArrayInit(0, sizeof(SPackedData)); + if ((*pSubmit)->submits == NULL) { + taosFreeQitem(*pSubmit); + *pSubmit = NULL; + return TSDB_CODE_OUT_OF_MEMORY; } - pMerged->type = STREAM_INPUT__MERGED_SUBMIT; - return pMerged; + (*pSubmit)->type = STREAM_INPUT__MERGED_SUBMIT; + return TSDB_CODE_SUCCESS; } int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) { - taosArrayPush(pMerged->submits, &pSubmit->submit); + void* p = taosArrayPush(pMerged->submits, &pSubmit->submit); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (pSubmit->ver > pMerged->ver) { pMerged->ver = pSubmit->ver; } @@ -187,11 +190,12 @@ int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubm // todo handle memory error int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem, SStreamQueueItem** pRes) { *pRes = NULL; + int32_t code = 0; if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem; - taosArrayAddAll(pBlock->blocks, pBlockSrc->blocks); + (void) taosArrayAddAll(pBlock->blocks, pBlockSrc->blocks); taosArrayDestroy(pBlockSrc->blocks); streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem)); @@ -201,33 +205,38 @@ int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst; SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)pElem; - streamMergeSubmit(pMerged, pBlockSrc); + + code = streamMergeSubmit(pMerged, pBlockSrc); streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem)); taosFreeQitem(pElem); *pRes = dst; *pRes = dst; - return TSDB_CODE_SUCCESS; + return code; } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); - if (pMerged == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + SStreamMergedSubmit* pMerged = NULL; + code = streamMergedSubmitNew(&pMerged); + if (code != 0) { + return code; } streamQueueItemIncSize((SStreamQueueItem*)pMerged, streamQueueItemGetSize(pElem)); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)pElem); + code = streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst); + if (code == 0) { + code = streamMergeSubmit(pMerged, (SStreamDataSubmit*)pElem); + } taosFreeQitem(dst); taosFreeQitem(pElem); *pRes = (SStreamQueueItem*)pMerged; - return TSDB_CODE_SUCCESS; + return code; } else { + code = TSDB_CODE_FAILED; stDebug("block type:%s not merged with existed blocks list, type:%d", streamQueueItemGetTypeStr(pElem->type), dst->type); - return TSDB_CODE_FAILED; + return code; } } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9e4b6bc09d..d59bb0dd91 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -121,7 +121,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); SEncoder encoder; tEncoderInit(&encoder, abuf, len); - tEncodeStreamRetrieveReq(&encoder, req); + (void) tEncodeStreamRetrieveReq(&encoder, req); tEncoderClear(&encoder); SRpcMsg rpcMsg = {0}; @@ -189,6 +189,7 @@ int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock) return code; } +// no need to do anything if failed int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) { void* buf = NULL; int32_t code = -1; @@ -202,7 +203,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, buf = rpcMallocCont(sizeof(SMsgHead) + tlen); if (buf == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } ((SMsgHead*)buf)->vgId = htonl(nodeId); @@ -220,8 +221,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, stDebug("s-task:%s (level:%d) send check msg to s-task:0x%" PRIx64 ":0x%x (vgId:%d)", pTask->id.idStr, pTask->info.taskLevel, pReq->streamId, pReq->downstreamTaskId, nodeId); - tmsgSendReq(pEpSet, &msg); - return 0; + return tmsgSendReq(pEpSet, &msg); } void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups) { @@ -239,7 +239,7 @@ void clearBufferedDispatchMsg(SStreamTask* pTask) { destroyDispatchMsg(pMsgInfo->pData, streamTaskGetNumOfDownstream(pTask)); } - taosThreadMutexLock(&pMsgInfo->lock); + streamMutexLock(&pMsgInfo->lock); pMsgInfo->checkpointId = -1; pMsgInfo->transId = -1; @@ -249,7 +249,7 @@ void clearBufferedDispatchMsg(SStreamTask* pTask) { clearDispatchInfo(pMsgInfo); taosArrayClear(pTask->msgInfo.pSendInfo); - taosThreadMutexUnlock(&pMsgInfo->lock); + streamMutexUnlock(&pMsgInfo->lock); } static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStreamDataBlock* pData) { @@ -424,9 +424,9 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch } static void setNotInDispatchMonitor(SDispatchMsgInfo* pMsgInfo) { - taosThreadMutexLock(&pMsgInfo->lock); + streamMutexLock(&pMsgInfo->lock); pMsgInfo->inMonitor = 0; - taosThreadMutexUnlock(&pMsgInfo->lock); + streamMutexUnlock(&pMsgInfo->lock); } static void setResendInfo(SDispatchEntry* pEntry, int64_t now) { @@ -440,13 +440,13 @@ static void addDispatchEntry(SDispatchMsgInfo* pMsgInfo, int32_t nodeId, int64_t SDispatchEntry entry = {.nodeId = nodeId, .rspTs = -1, .status = 0, .sendTs = now}; if (lock) { - taosThreadMutexLock(&pMsgInfo->lock); + streamMutexLock(&pMsgInfo->lock); } - taosArrayPush(pMsgInfo->pSendInfo, &entry); + (void) taosArrayPush(pMsgInfo->pSendInfo, &entry); if (lock) { - taosThreadMutexUnlock(&pMsgInfo->lock); + streamMutexUnlock(&pMsgInfo->lock); } } @@ -496,16 +496,16 @@ static void doMonitorDispatchData(void* param, void* tmrId) { return; } - taosThreadMutexLock(&pMsgInfo->lock); + streamMutexLock(&pMsgInfo->lock); if (pTask->outputq.status == TASK_OUTPUT_STATUS__NORMAL) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s not in dispatch procedure, abort from timer, ref:%d", pTask->id.idStr, ref); pTask->msgInfo.inMonitor = 0; - taosThreadMutexUnlock(&pMsgInfo->lock); + streamMutexUnlock(&pMsgInfo->lock); return; } - taosThreadMutexUnlock(&pMsgInfo->lock); + streamMutexUnlock(&pMsgInfo->lock); int32_t numOfFailed = getFailedDispatchInfo(pMsgInfo, now); if (numOfFailed == 0) { @@ -577,8 +577,9 @@ static void doMonitorDispatchData(void* param, void* tmrId) { } void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) { + int32_t vgId = pTask->pMeta->vgId; if (pTask->msgInfo.pRetryTmr != NULL) { - taosTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr); + streamTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr, vgId, "dispatch-monitor-tmr"); } else { pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer); } @@ -612,7 +613,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S } } } else { - buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); + (void) buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); } snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db, @@ -624,8 +625,10 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S SBlockName bln = {0}; bln.hashValue = hashValue; memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName)); + + // failed to put into name buffer, no need to do anything if (tSimpleHashGetSize(pTask->pNameMap) < MAX_BLOCK_NAME_NUM) { - tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); + (void) tSimpleHashPut(pTask->pNameMap, &groupId, sizeof(int64_t), &bln, sizeof(SBlockName)); } } @@ -633,14 +636,14 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S int32_t numOfVgroups = taosArrayGetSize(vgInfo); // TODO: optimize search - taosThreadMutexLock(&pTask->msgInfo.lock); + streamMutexLock(&pTask->msgInfo.lock); for (int32_t j = 0; j < numOfVgroups; j++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) { - taosThreadMutexUnlock(&pTask->msgInfo.lock); + streamMutexUnlock(&pTask->msgInfo.lock); return -1; } @@ -655,7 +658,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S } } - taosThreadMutexUnlock(&pTask->msgInfo.lock); + streamMutexUnlock(&pTask->msgInfo.lock); ASSERT(found); return 0; } @@ -690,7 +693,8 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { ASSERT(pTask->msgInfo.pData == NULL); stDebug("s-task:%s start to dispatch msg, set output status:%d", id, pTask->outputq.status); - SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputq.queue); + SStreamDataBlock* pBlock = NULL; + streamQueueNextItem(pTask->outputq.queue, (SStreamQueueItem**)&pBlock); if (pBlock == NULL) { atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); stDebug("s-task:%s not dispatch since no elems in outputQ, output status:%d", id, pTask->outputq.status); @@ -703,9 +707,9 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { pTask->execInfo.dispatch += 1; - taosThreadMutexLock(&pTask->msgInfo.lock); + streamMutexLock(&pTask->msgInfo.lock); initDispatchInfo(&pTask->msgInfo, pTask->execInfo.dispatch); - taosThreadMutexUnlock(&pTask->msgInfo.lock); + streamMutexUnlock(&pTask->msgInfo.lock); int32_t code = doBuildDispatchMsg(pTask, pBlock); if (code == 0) { @@ -719,7 +723,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { code = sendDispatchMsg(pTask, pTask->msgInfo.pData); - taosThreadMutexLock(&pTask->msgInfo.lock); + streamMutexLock(&pTask->msgInfo.lock); if (pTask->msgInfo.inMonitor == 0) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s start dispatch monitor tmr in %dms, ref:%d, dispatch code:%s", id, DISPATCH_RETRY_INTERVAL_MS, @@ -730,7 +734,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { stDebug("s-task:%s already in dispatch monitor tmr", id); } - taosThreadMutexUnlock(&pTask->msgInfo.lock); + streamMutexUnlock(&pTask->msgInfo.lock); // this block can not be deleted until it has been sent to downstream task successfully. return TSDB_CODE_SUCCESS; @@ -758,7 +762,7 @@ int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32 buf = rpcMallocCont(sizeof(SMsgHead) + tlen); if (buf == NULL) { - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId); @@ -791,33 +795,33 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; if (++pActiveInfo->sendReadyCheckCounter < 100) { - taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); + streamTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr, vgId, "chkpt-ready-monitor"); return; } pActiveInfo->sendReadyCheckCounter = 0; stDebug("s-task:%s in sending checkpoint-ready msg monitor timer", id); - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); SStreamTaskState pState = streamTaskGetStatus(pTask); if (pState.state != TASK_STATUS__CK) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready send, ref:%d", id, vgId, pState.name, ref); - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pTask->pMeta, pTask); return; } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); - taosThreadMutexLock(&pActiveInfo->lock); + streamMutexLock(&pActiveInfo->lock); SArray* pList = pActiveInfo->pReadyMsgList; int32_t num = taosArrayGetSize(pList); // active checkpoint info is cleared for now if ((pActiveInfo->activeId == 0) && (pActiveInfo->transId == 0) && (num == 0) && (pTask->chkInfo.startTs == 0)) { - taosThreadMutexUnlock(&pActiveInfo->lock); + streamMutexUnlock(&pActiveInfo->lock); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stWarn("s-task:%s vgId:%d active checkpoint may be cleared, quit from readyMsg send tmr, ref:%d", id, vgId, ref); @@ -835,7 +839,7 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { continue; } - taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); + (void) taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, pTask->info.taskLevel, pInfo->upstreamTaskId); } @@ -852,17 +856,25 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { if (taskId == pReadyInfo->upstreamTaskId) { // send msg again SRpcMsg msg = {0}; - initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId, + int32_t code = initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId, checkpointId, &msg); - tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg); - stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel, - pReadyInfo->upstreamTaskId); + if (code == TSDB_CODE_SUCCESS) { + code = tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg); + if (code == TSDB_CODE_SUCCESS) { + stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel, + pReadyInfo->upstreamTaskId); + } else { + stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id); + } + } else { + stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id); + } } } } - taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); - taosThreadMutexUnlock(&pActiveInfo->lock); + streamTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr, vgId, "chkpt-ready-monitor"); + streamMutexUnlock(&pActiveInfo->lock); } else { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug( @@ -871,7 +883,7 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { id, vgId, ref); streamClearChkptReadyMsg(pActiveInfo); - taosThreadMutexUnlock(&pActiveInfo->lock); + streamMutexUnlock(&pActiveInfo->lock); streamMetaReleaseTask(pTask->pMeta, pTask); } @@ -883,9 +895,10 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; const char* id = pTask->id.idStr; + int32_t vgId = pTask->pMeta->vgId; SArray* pList = pActiveInfo->pReadyMsgList; - taosThreadMutexLock(&pActiveInfo->lock); + streamMutexLock(&pActiveInfo->lock); int32_t num = taosArrayGetSize(pList); ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num); @@ -894,15 +907,22 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i); SRpcMsg msg = {0}; - initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId, + int32_t code = initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId, &msg); - tmsgSendReq(&pInfo->upstreamNodeEpset, &msg); - - stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x", id, pTask->info.taskLevel, - pInfo->upstreamTaskId); + if (code == TSDB_CODE_SUCCESS) { + code = tmsgSendReq(&pInfo->upstreamNodeEpset, &msg); + if (code == TSDB_CODE_SUCCESS) { + stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x", id, pTask->info.taskLevel, + pInfo->upstreamTaskId); + } else { + stError("s-task:%s failed to send checkpoint-ready msg, try nex time in 10s", id); + } + } else { + stError("s-task:%s failed to prepare the checkpoint-ready msg, try nex time in 10s", id); + } } - taosThreadMutexUnlock(&pActiveInfo->lock); + streamMutexUnlock(&pActiveInfo->lock); stDebug("s-task:%s level:%d checkpoint-ready msg sent to all %d upstreams", id, pTask->info.taskLevel, num); // start to check if checkpoint ready msg has successfully received by upstream tasks. @@ -914,7 +934,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { if (pActiveInfo->pSendReadyMsgTmr == NULL) { pActiveInfo->pSendReadyMsgTmr = taosTmrStart(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer); } else { - taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); + streamTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr, vgId, "chkpt-ready-monitor"); } } @@ -925,7 +945,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { SArray* pList = pTask->chkInfo.pActiveInfo->pReadyMsgList; - taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock); + streamMutexLock(&pTask->chkInfo.pActiveInfo->lock); ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); if (taosArrayGetSize(pList) == 1) { @@ -940,7 +960,7 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { pTask->info.taskLevel); } - taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); + streamMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); return TSDB_CODE_SUCCESS; } @@ -978,8 +998,8 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch payloadLen += sizeof(SRetrieveTableRsp); - taosArrayPush(pReq->dataLen, &payloadLen); - taosArrayPush(pReq->data, &buf); + (void) taosArrayPush(pReq->dataLen, &payloadLen); + (void) taosArrayPush(pReq->data, &buf); pReq->totalLen += dataStrLen; return 0; @@ -1056,7 +1076,7 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp void* abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); tEncoderInit(&encoder, (uint8_t*)abuf, len); - tEncodeStreamCheckpointSourceRsp(&encoder, &rsp); + (void) tEncodeStreamCheckpointSourceRsp(&encoder, &rsp); tEncoderClear(&encoder); initRpcMsg(pMsg, 0, pBuf, sizeof(SMsgHead) + len); @@ -1071,10 +1091,11 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa STaskCheckpointReadyInfo info = { .recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId}; - streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS); + // todo retry until it success + (void) streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS); SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - taosThreadMutexLock(&pActiveInfo->lock); + streamMutexLock(&pActiveInfo->lock); int32_t size = taosArrayGetSize(pActiveInfo->pReadyMsgList); if (size > 0) { @@ -1091,15 +1112,15 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa ASSERT(0); // failed to handle it } } else { - taosArrayPush(pActiveInfo->pReadyMsgList, &info); + (void) taosArrayPush(pActiveInfo->pReadyMsgList, &info); stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1); } - taosThreadMutexUnlock(&pActiveInfo->lock); + streamMutexUnlock(&pActiveInfo->lock); return TSDB_CODE_SUCCESS; } -int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, +void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId, SEpSet* pEpset, int64_t checkpointId) { ASSERT(upstreamTaskId != 0); @@ -1109,8 +1130,6 @@ int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t up pReadyInfo->recvTs = taosGetTimestampMs(); pReadyInfo->checkpointId = checkpointId; pReadyInfo->childId = childId; - - return TSDB_CODE_SUCCESS; } int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) { @@ -1119,9 +1138,9 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, } SStreamUpstreamEpInfo* pInfo = NULL; - int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; + streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo); + if (pInfo == NULL) { + return TSDB_CODE_STREAM_TASK_NOT_EXIST; } STaskCheckpointReadyInfo info = {0}; @@ -1132,8 +1151,8 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - taosThreadMutexLock(&pActiveInfo->lock); - taosArrayPush(pActiveInfo->pReadyMsgList, &info); + streamMutexLock(&pActiveInfo->lock); + (void) taosArrayPush(pActiveInfo->pReadyMsgList, &info); int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList); int32_t total = streamTaskGetNumOfUpstream(pTask); @@ -1145,7 +1164,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, stDebug("s-task:%s %d/%d checkpoint-trigger recv", pTask->id.idStr, numOfRecv, total); } - taosThreadMutexUnlock(&pActiveInfo->lock); + streamMutexUnlock(&pActiveInfo->lock); return 0; } @@ -1187,10 +1206,8 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId if (delayDispatch) { return 0; } else { - streamDispatchStreamBlock(pTask); + return streamDispatchStreamBlock(pTask); } - - return 0; } static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, int32_t* pNotRsp, const char* id) { @@ -1200,7 +1217,7 @@ static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t bool allRsp = false; *pNotRsp = 0; - taosThreadMutexLock(&pMsgInfo->lock); + streamMutexLock(&pMsgInfo->lock); int32_t numOfDispatchBranch = taosArrayGetSize(pMsgInfo->pSendInfo); for(int32_t i = 0; i < numOfDispatchBranch; ++i) { @@ -1228,7 +1245,7 @@ static bool setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t *pNotRsp = numOfDispatchBranch - numOfRsp; allRsp = (numOfRsp == numOfDispatchBranch); - taosThreadMutexUnlock(&pMsgInfo->lock); + streamMutexUnlock(&pMsgInfo->lock); ASSERT(updated); return allRsp; @@ -1240,7 +1257,7 @@ bool isDispatchRspTimeout(SDispatchEntry* pEntry, int64_t now) { int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now) { int32_t numOfFailed = 0; - taosThreadMutexLock(&pMsgInfo->lock); + streamMutexLock(&pMsgInfo->lock); for (int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j); @@ -1248,7 +1265,7 @@ int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now) { numOfFailed += 1; } } - taosThreadMutexUnlock(&pMsgInfo->lock); + streamMutexUnlock(&pMsgInfo->lock); return numOfFailed; } @@ -1260,9 +1277,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i bool allRsp = false; int32_t notRsp = 0; - taosThreadMutexLock(&pMsgInfo->lock); + streamMutexLock(&pMsgInfo->lock); int32_t msgId = pMsgInfo->msgId; - taosThreadMutexUnlock(&pMsgInfo->lock); + streamMutexUnlock(&pMsgInfo->lock); // follower not handle the dispatch rsp if ((pTask->pMeta->role == NODE_ROLE_FOLLOWER) || (pTask->status.downstreamReady != 1)) { @@ -1315,7 +1332,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i { bool delayDispatch = (pMsgInfo->dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); if (delayDispatch) { - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); // we only set the dispatch msg info for current checkpoint trans if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK && pTask->chkInfo.pActiveInfo->activeId == pMsgInfo->checkpointId) { @@ -1329,7 +1346,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i " transId:%d discard, since expired", pTask->id.idStr, pMsgInfo->checkpointId, pMsgInfo->transId); } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); } } } @@ -1374,12 +1391,12 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i // now ready for next data output atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); } else { - handleDispatchSuccessRsp(pTask, pRsp->downstreamTaskId, pRsp->downstreamNodeId); + code = handleDispatchSuccessRsp(pTask, pRsp->downstreamTaskId, pRsp->downstreamNodeId); } } } - return 0; + return code; } static int32_t buildDispatchRsp(const SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t status, void** pBuf) { @@ -1438,9 +1455,9 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId); SStreamUpstreamEpInfo* pInfo = NULL; - int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; + streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId, &pInfo); + if (pInfo == NULL) { + return TSDB_CODE_STREAM_TASK_NOT_EXIST; } if (pMeta->role == NODE_ROLE_FOLLOWER) { @@ -1484,6 +1501,5 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S tmsgSendRsp(pRsp); } - streamTrySchedExec(pTask); - return code; + return streamTrySchedExec(pTask); } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 238db27d60..d222004fb7 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -24,7 +24,7 @@ #define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec static int32_t streamTransferStateDoPrepare(SStreamTask* pTask); -static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); +static void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); bool streamTaskShouldStop(const SStreamTask* pTask) { SStreamTaskState pState = streamTaskGetStatus(pTask); @@ -52,8 +52,9 @@ static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBl return code; } + // not handle error, if dispatch failed, try next time. // checkpoint trigger will be checked - streamDispatchStreamBlock(pTask); + (void) streamDispatchStreamBlock(pTask); } return code; @@ -90,7 +91,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* return code; } -int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { +void streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { int32_t code = TSDB_CODE_SUCCESS; void* pExecutor = pTask->exec.pExecutor; @@ -108,7 +109,7 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* if (streamTaskShouldStop(pTask)) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - return 0; + return; } SSDataBlock* output = NULL; @@ -128,11 +129,16 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); - assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); + (void) assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); block.info.type = STREAM_PULL_OVER; block.info.childId = pTask->info.selfChildId; - taosArrayPush(pRes, &block); - numOfBlocks += 1; + + void* p = taosArrayPush(pRes, &block); + if (p != NULL) { + numOfBlocks += 1; + } else { + stError("s-task:%s failed to add retrieve block", pTask->id.idStr); + } stDebug("s-task:%s(child %d) retrieve process completed, reqId:0x%" PRIx64 " dump results", pTask->id.idStr, pTask->info.selfChildId, pRetrieveBlock->reqId); @@ -151,13 +157,21 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* } SSDataBlock block = {0}; - assignOneDataBlock(&block, output); + code = assignOneDataBlock(&block, output); + if (code) { + stError("s-task:%s failed to build result block due to out of memory", pTask->id.idStr); + continue; + } + block.info.childId = pTask->info.selfChildId; size += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); numOfBlocks += 1; - taosArrayPush(pRes, &block); + void* p = taosArrayPush(pRes, &block); + if (p == NULL) { + stError("s-task:%s failed to add computing results, the final res may be incorrect", pTask->id.idStr); + } stDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr, pTask->info.selfChildId, numOfBlocks, SIZE_IN_MiB(size)); @@ -166,8 +180,9 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) { ASSERT(numOfBlocks == taosArrayGetSize(pRes)); code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); + // todo: here we need continue retry to put it into output buffer if (code != TSDB_CODE_SUCCESS) { - return code; + return; } pRes = NULL; @@ -182,16 +197,16 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* } else { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); } - - return code; } -static int32_t handleSanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) { +// todo contiuous try to create result blocks +static int32_t handleScanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) { int32_t code = TSDB_CODE_SUCCESS; if (taosArrayGetSize(pRes) > 0) { SStreamDataBlock* pStreamBlocks = NULL; code = createStreamBlockFromResults(NULL, pTask, size, pRes, &pStreamBlocks); if (code) { + stError("s-task:%s failed to build history result blocks", pTask->id.idStr); return code; } @@ -236,9 +251,13 @@ static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t* } SSDataBlock block = {0}; - assignOneDataBlock(&block, output); + (void) assignOneDataBlock(&block, output); block.info.childId = pTask->info.selfChildId; - taosArrayPush(pRes, &block); + + void* p = taosArrayPush(pRes, &block); + if (p == NULL) { + stError("s-task:%s failed to add computing results, the final res may be incorrect", pTask->id.idStr); + } (*pSize) += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); numOfBlocks += 1; @@ -264,7 +283,7 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { const char* id = pTask->id.idStr; if (!pTask->hTaskInfo.operatorOpen) { - qSetStreamOpOpen(exec); + (void) qSetStreamOpOpen(exec); pTask->hTaskInfo.operatorOpen = true; } @@ -301,8 +320,8 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0); } - // dispatch the generated results - /*int32_t code = */handleSanhistoryResultBlocks(pTask, pRes, size); + // dispatch the generated results, todo fix error + (void) handleScanhistoryResultBlocks(pTask, pRes, size); if (finished) { return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0); @@ -323,14 +342,15 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { SStreamTask* pStreamTask = NULL; int32_t code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask); - if (pStreamTask == NULL) { + if (pStreamTask == NULL || code != TSDB_CODE_SUCCESS) { stError( "s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related " "fill-history task", id, (int32_t) pTask->streamTaskId.taskId); // 1. free it and remove fill-history task from disk meta-store - streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); + // todo: this function should never be failed. + (void) streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); // 2. save to disk streamMetaWLock(pMeta); @@ -388,14 +408,14 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN, pTimeWindow->ekey, p, pStreamTask->status.schedStatus); - streamTaskResetTimewindowFilter(pStreamTask); + (void) streamTaskResetTimewindowFilter(pStreamTask); } else { stDebug("s-task:%s no need to update/reset filter time window for non-source tasks", pStreamTask->id.idStr); } // NOTE: transfer the ownership of executor state before handle the checkpoint block during stream exec // 2. send msg to mnode to launch a checkpoint to keep the state for current stream - streamTaskSendCheckpointReq(pStreamTask); + (void) streamTaskSendCheckpointReq(pStreamTask); // 3. assign the status to the value that will be kept in disk pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask).state; @@ -409,7 +429,7 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) { static int32_t haltCallback(SStreamTask* pTask, void* param) { streamTaskOpenAllUpstreamInput(pTask); - streamTaskSendCheckpointReq(pTask); + (void) streamTaskSendCheckpointReq(pTask); return TSDB_CODE_SUCCESS; } @@ -445,18 +465,19 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) { } // set input -static void doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_t* pVer, const char* id) { +static int32_t doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_t* pVer, const char* id) { void* pExecutor = pTask->exec.pExecutor; + int32_t code = 0; const SStreamQueueItem* pItem = pInput; if (pItem->type == STREAM_INPUT__GET_RES) { const SStreamTrigger* pTrigger = (const SStreamTrigger*)pInput; - qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); + code = qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; - qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); + code = qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); stDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit, pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); ASSERT((*pVer) <= pSubmit->submit.ver); @@ -468,7 +489,7 @@ static void doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_ SArray* pBlockList = pBlock->blocks; int32_t numOfBlocks = taosArrayGetSize(pBlockList); stDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, id, numOfBlocks, pBlock->sourceVer); - qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); + code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)pInput; @@ -477,24 +498,26 @@ static void doSetStreamInputBlock(SStreamTask* pTask, const void* pInput, int64_ int32_t numOfBlocks = taosArrayGetSize(pBlockList); stDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d, ver:%" PRId64, id, pTask, numOfBlocks, pMerged->ver); - qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); + code = qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); ASSERT((*pVer) <= pMerged->ver); (*pVer) = pMerged->ver; } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput; - qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); + code = qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__CHECKPOINT || pItem->type == STREAM_INPUT__CHECKPOINT_TRIGGER) { const SStreamDataBlock* pCheckpoint = (const SStreamDataBlock*)pInput; - qSetMultiStreamInput(pExecutor, pCheckpoint->blocks, 1, pItem->type); + code = qSetMultiStreamInput(pExecutor, pCheckpoint->blocks, 1, pItem->type); } else { ASSERT(0); } + + return code; } -int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { +void streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { const char* id = pTask->id.idStr; int32_t code = TSDB_CODE_SUCCESS; int32_t level = pTask->info.taskLevel; @@ -505,7 +528,7 @@ int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBloc if (remain > 0) { streamFreeQitem((SStreamQueueItem*)pBlock); stDebug("s-task:%s receive upstream trans-state msg, not sent remain:%d", id, remain); - return 0; + return; } } @@ -526,7 +549,7 @@ int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBloc pBlock->srcVgId = pTask->pMeta->vgId; code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock); if (code == 0) { - streamDispatchStreamBlock(pTask); + (void) streamDispatchStreamBlock(pTask); } else { // todo put into queue failed, retry streamFreeQitem((SStreamQueueItem*)pBlock); } @@ -540,11 +563,9 @@ int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBloc code = streamTransferStatePrepare(pTask); if (code != TSDB_CODE_SUCCESS) { - /*int8_t status = */ streamTaskSetSchedStatusInactive(pTask); + (void)streamTaskSetSchedStatusInactive(pTask); } } - - return code; } //static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } @@ -559,7 +580,11 @@ static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, i stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type)); - doSetStreamInputBlock(pTask, pBlock, &ver, id); + int32_t code = doSetStreamInputBlock(pTask, pBlock, &ver, id); + if(code) { + stError("s-task:%s failed to set input block, not exec for these blocks", id); + return; + } int64_t totalSize = 0; int32_t totalBlocks = 0; @@ -601,9 +626,9 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB STaskId* pHTaskId = &pTask->hTaskInfo.id; SStreamTask* pHTask = NULL; int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); - if (pHTask != NULL) { - streamTaskReleaseState(pHTask); - streamTaskReloadState(pTask); + if (code == TSDB_CODE_SUCCESS) { // ignore the error code. + (void) streamTaskReleaseState(pHTask); + (void) streamTaskReloadState(pTask); stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr, streamTaskGetStatus(pHTask).name); @@ -676,7 +701,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { // dispatch checkpoint msg to all downstream tasks int32_t type = pInput->type; if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { - streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); + (void) streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); continue; } @@ -694,7 +719,10 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { if (type == STREAM_INPUT__DATA_BLOCK) { pTask->execInfo.sink.dataSize += blockSize; stDebug("s-task:%s sink task start to sink %d blocks, size:%.2fKiB", id, numOfBlocks, SIZE_IN_KiB(blockSize)); - doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput); + int32_t code = doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput); + if (code != TSDB_CODE_SUCCESS) { + // todo handle error. + } double el = (taosGetTimestampMs() - st) / 1000.0; if (fabs(el - 0.0) <= DBL_EPSILON) { @@ -712,11 +740,11 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { streamFreeQitem(pInput); } else { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); SStreamTaskState pState = streamTaskGetStatus(pTask); if (pState.state == TASK_STATUS__CK) { stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name); - streamTaskBuildCheckpoint(pTask); + (void) streamTaskBuildCheckpoint(pTask); // ignore this error msg, and continue } else { // todo refactor int32_t code = 0; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { @@ -732,7 +760,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamFreeQitem(pInput); return 0; } @@ -765,21 +793,21 @@ bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) { } } -int32_t streamResumeTask(SStreamTask* pTask) { +void streamResumeTask(SStreamTask* pTask) { ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__ACTIVE); const char* id = pTask->id.idStr; while (1) { - /*int32_t code = */ doStreamExecTask(pTask); + (void) doStreamExecTask(pTask); // check if continue - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue); if ((numOfItems == 0) || streamTaskShouldStop(pTask) || streamTaskShouldPause(pTask)) { atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); streamTaskClearSchedIdleInfo(pTask); - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); setLastExecTs(pTask, taosGetTimestampMs()); @@ -787,19 +815,19 @@ int32_t streamResumeTask(SStreamTask* pTask) { stDebug("s-task:%s exec completed, status:%s, sched-status:%d, lastExecTs:%" PRId64, id, p, pTask->status.schedStatus, pTask->status.lastExecTs); - return 0; + return; } else { // check if this task needs to be idle for a while if (pTask->status.schedIdleTime > 0) { streamTaskResumeInFuture(pTask); - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); setLastExecTs(pTask, taosGetTimestampMs()); - return 0; + return; } } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); } } diff --git a/source/libs/stream/src/streamHb.c b/source/libs/stream/src/streamHb.c index 4aaaed615c..9804943ec2 100644 --- a/source/libs/stream/src/streamHb.c +++ b/source/libs/stream/src/streamHb.c @@ -54,7 +54,7 @@ static bool existInHbMsg(SStreamHbMsg* pMsg, SDownstreamTaskEpset* pTaskEpset) { static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) { SStreamMeta* pMeta = pTask->pMeta; - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); for (int j = 0; j < num; ++j) { @@ -73,7 +73,7 @@ static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) { } taosArrayClear(pTask->outputInfo.pNodeEpsetUpdateList); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); } static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* pEpset) { @@ -170,9 +170,9 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { continue; } - (void) taosThreadMutexLock(&(*pTask)->lock); + streamMutexLock(&(*pTask)->lock); STaskStatusEntry entry = streamTaskGetStatusEntry(*pTask); - (void) taosThreadMutexUnlock(&(*pTask)->lock); + streamMutexUnlock(&(*pTask)->lock); entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) { @@ -190,9 +190,9 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) { stInfo("s-task:%s set kill checkpoint trans in hbMsg, transId:%d, clear the active checkpointInfo", (*pTask)->id.idStr, p->transId); - (void) taosThreadMutexLock(&(*pTask)->lock); + streamMutexLock(&(*pTask)->lock); streamTaskClearCheckInfo((*pTask), true); - (void) taosThreadMutexUnlock(&(*pTask)->lock); + streamMutexUnlock(&(*pTask)->lock); } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 0a627034b1..806eb0ce91 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -89,11 +89,11 @@ void metaRefMgtCleanup() { } taosHashCleanup(gMetaRefMgt.pTable); - taosThreadMutexDestroy(&gMetaRefMgt.mutex); + streamMutexDestroy(&gMetaRefMgt.mutex); } int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) { - taosThreadMutexLock(&gMetaRefMgt.mutex); + streamMutexLock(&gMetaRefMgt.mutex); void* p = taosHashGet(gMetaRefMgt.pTable, &vgId, sizeof(vgId)); if (p == NULL) { @@ -105,7 +105,7 @@ int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) { taosArrayPush(list, &rid); } - taosThreadMutexUnlock(&gMetaRefMgt.mutex); + streamMutexUnlock(&gMetaRefMgt.mutex); return 0; } @@ -238,7 +238,7 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) { int64_t chkpId = pTask->chkInfo.checkpointId; - taosThreadMutexLock(&pMeta->backendMutex); + streamMutexLock(&pMeta->backendMutex); void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key)); if ((ppBackend != NULL) && (*ppBackend != NULL)) { taskDbAddRef(*ppBackend); @@ -247,7 +247,7 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) pBackend->pMeta = pMeta; pTask->pBackend = pBackend; - taosThreadMutexUnlock(&pMeta->backendMutex); + streamMutexUnlock(&pMeta->backendMutex); stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); return 0; } @@ -260,11 +260,11 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) break; } - taosThreadMutexUnlock(&pMeta->backendMutex); + streamMutexUnlock(&pMeta->backendMutex); taosMsleep(1000); stDebug("backend held by other task, restart later, path:%s, key:%s", pMeta->path, key); - taosThreadMutexLock(&pMeta->backendMutex); + streamMutexLock(&pMeta->backendMutex); } int64_t tref = taosAddRef(taskDbWrapperId, pBackend); @@ -276,7 +276,7 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) if (processVer != -1) pTask->chkInfo.processedVer = processVer; taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*)); - taosThreadMutexUnlock(&pMeta->backendMutex); + streamMutexUnlock(&pMeta->backendMutex); stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); return 0; @@ -286,10 +286,10 @@ void streamMetaRemoveDB(void* arg, char* key) { if (arg == NULL || key == NULL) return; SStreamMeta* pMeta = arg; - taosThreadMutexLock(&pMeta->backendMutex); + streamMutexLock(&pMeta->backendMutex); taosHashRemove(pMeta->pTaskDbUnique, key, strlen(key)); - taosThreadMutexUnlock(&pMeta->backendMutex); + streamMutexUnlock(&pMeta->backendMutex); } int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, int32_t vgId, @@ -541,7 +541,7 @@ void streamMetaCloseImpl(void* arg) { pMeta->pHbInfo = NULL; taosMemoryFree(pMeta->path); - taosThreadMutexDestroy(&pMeta->backendMutex); + streamMutexDestroy(&pMeta->backendMutex); taosCleanUpScheduler(pMeta->qHandle); taosMemoryFree(pMeta->qHandle); @@ -1132,7 +1132,7 @@ int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { continue; } - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); SStreamTaskState pState = streamTaskGetStatus(pTask); if (pState.state == TASK_STATUS__CK) { @@ -1141,7 +1141,7 @@ int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState.name); } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); } @@ -1271,7 +1271,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { streamLaunchFillHistoryTask(pTask); } - streamMetaAddTaskLaunchResult(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->checkTs, pInfo->readyTs, true); + (void) streamMetaAddTaskLaunchResult(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->checkTs, pInfo->readyTs, true); streamMetaReleaseTask(pMeta, pTask); continue; } @@ -1388,16 +1388,16 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas ASSERT(pTask->status.downstreamReady == 0); // avoid initialization and destroy running concurrently. - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); if (pTask->pBackend == NULL) { code = pMeta->expandTaskFn(pTask); - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); if (code != TSDB_CODE_SUCCESS) { streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs); } } else { - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); } if (code == TSDB_CODE_SUCCESS) { @@ -1440,7 +1440,6 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3 STaskId id = {.streamId = streamId, .taskId = taskId}; streamMetaWLock(pMeta); - SStreamTask** p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (p == NULL) { // task does not exists in current vnode, not record the complete info stError("vgId:%d s-task:0x%x not exists discard the check downstream info", pMeta->vgId, taskId); @@ -1449,9 +1448,9 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3 } // clear the send consensus-checkpointId flag - taosThreadMutexLock(&(*p)->lock); + streamMutexLock(&(*p)->lock); (*p)->status.sendConsensusChkptId = false; - taosThreadMutexUnlock(&(*p)->lock); + streamMutexUnlock(&(*p)->lock); if (pStartInfo->startAllTasks != 1) { int64_t el = endTs - startTs; @@ -1537,9 +1536,9 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta streamMetaRUnLock(pMeta); // add the failed task info, along with the related fill-history task info into tasks list. - streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false); + (void) streamMetaAddTaskLaunchResult(pMeta, streamId, taskId, startTs, now, false); if (hasFillhistoryTask) { - streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false); + (void) streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false); } } else { streamMetaRUnLock(pMeta); @@ -1554,12 +1553,12 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs) { int32_t startTs = pTask->execInfo.checkTs; - streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false); + (void) streamMetaAddTaskLaunchResult(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, startTs, failedTs, false); // automatically set the related fill-history task to be failed. if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { STaskId* pId = &pTask->hTaskInfo.id; - streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false); + (void) streamMetaAddTaskLaunchResult(pTask->pMeta, pId->streamId, pId->taskId, startTs, failedTs, false); } } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index b8cdcd4cf5..537062b04e 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -27,20 +27,24 @@ typedef struct SQueueReader { int32_t waitDuration; // maximum wait time to format several block into a batch to process, unit: ms } SQueueReader; +#define streamQueueCurItem(_q) ((_q)->qItem) + static bool streamTaskExtractAvailableToken(STokenBucket* pBucket, const char* id); static void streamTaskPutbackToken(STokenBucket* pBucket); static void streamTaskConsumeQuota(STokenBucket* pBucket, int32_t bytes); static void streamQueueCleanup(SStreamQueue* pQueue) { - void* qItem = NULL; - while ((qItem = streamQueueNextItem(pQueue)) != NULL) { + SStreamQueueItem* qItem = NULL; + while (1) { + streamQueueNextItem(pQueue, &qItem); + if (qItem == NULL) { + break; + } streamFreeQitem(qItem); } pQueue->status = STREAM_QUEUE__SUCESS; } -static void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; } - int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ) { *pQ = NULL; int32_t code = 0; @@ -81,21 +85,22 @@ void streamQueueClose(SStreamQueue* pQueue, int32_t taskId) { taosMemoryFree(pQueue); } -void* streamQueueNextItem(SStreamQueue* pQueue) { +void streamQueueNextItem(SStreamQueue* pQueue, SStreamQueueItem** pItem) { + *pItem = NULL; int8_t flag = atomic_exchange_8(&pQueue->status, STREAM_QUEUE__PROCESSING); if (flag == STREAM_QUEUE__FAILED) { ASSERT(pQueue->qItem != NULL); - return streamQueueCurItem(pQueue); + *pItem = streamQueueCurItem(pQueue); } else { pQueue->qItem = NULL; - taosGetQitem(pQueue->qall, &pQueue->qItem); + (void) taosGetQitem(pQueue->qall, &pQueue->qItem); if (pQueue->qItem == NULL) { - taosReadAllQitems(pQueue->pQueue, pQueue->qall); - taosGetQitem(pQueue->qall, &pQueue->qItem); + (void) taosReadAllQitems(pQueue->pQueue, pQueue->qall); + (void) taosGetQitem(pQueue->qall, &pQueue->qItem); } - return streamQueueCurItem(pQueue); + *pItem = streamQueueCurItem(pQueue); } } @@ -181,7 +186,8 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte return EXEC_CONTINUE; } - SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputq.queue); + SStreamQueueItem* qItem = NULL; + streamQueueNextItem(pTask->inputq.queue, (SStreamQueueItem**)&qItem); if (qItem == NULL) { // restore the token to bucket if (*numOfBlocks > 0) { @@ -338,7 +344,8 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER && (pTask->info.delaySchedParam != 0)) { - atomic_val_compare_exchange_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); + (void)atomic_val_compare_exchange_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE, + TASK_TRIGGER_STATUS__ACTIVE); stDebug("s-task:%s new data arrived, active the sched-trigger, triggerStatus:%d", pTask->id.idStr, pTask->schedInfo.status); } @@ -347,18 +354,19 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) } int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) { - int32_t code; - SStreamDataBlock* pTranstate; + int32_t code = 0; + SStreamDataBlock* pTranstate = NULL; + SSDataBlock* pBlock = NULL; code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock), (void**)&pTranstate); if (code) { return code; } - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pBlock == NULL) { - taosFreeQitem(pTranstate); - return TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } pTranstate->type = STREAM_INPUT__TRANS_STATE; @@ -368,15 +376,30 @@ int32_t streamTaskPutTranstateIntoInputQ(SStreamTask* pTask) { pBlock->info.childId = pTask->info.selfChildId; pTranstate->blocks = taosArrayInit(4, sizeof(SSDataBlock)); // pBlock; - taosArrayPush(pTranstate->blocks, pBlock); + if (pTranstate->blocks == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + void* p = taosArrayPush(pTranstate->blocks, pBlock); + if (p == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } taosMemoryFree(pBlock); if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTranstate) < 0) { - return TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } pTask->status.appendTranstateBlock = true; return TSDB_CODE_SUCCESS; + +_err: + taosMemoryFree(pBlock); + taosFreeQitem(pTranstate); + return code; } // the result should be put into the outputQ in any cases, the result may be lost otherwise. diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index f11b135168..74f370d199 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -36,7 +36,7 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask) { int32_t streamTrySchedExec(SStreamTask* pTask) { if (streamTaskSetSchedStatusWait(pTask)) { - streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId, 0); + return streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId, 0); } else { stTrace("s-task:%s not launch task since sched status:%d", pTask->id.idStr, pTask->status.schedStatus); } @@ -47,10 +47,9 @@ int32_t streamTrySchedExec(SStreamTask* pTask) { int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType) { SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); if (pRunReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; stError("vgId:%d failed to create msg to start stream task:0x%x exec, type:%d, code:%s", vgId, taskId, execType, terrstr()); - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } if (streamId != 0) { @@ -65,29 +64,27 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3 pRunReq->reqType = execType; SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; - tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); - return TSDB_CODE_SUCCESS; + return tmsgPutToQueue(pMsgCb, STREAM_QUEUE, &msg); } void streamTaskClearSchedIdleInfo(SStreamTask* pTask) { pTask->status.schedIdleTime = 0; } void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } -int32_t streamTaskResumeInFuture(SStreamTask* pTask) { +void streamTaskResumeInFuture(SStreamTask* pTask) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s task should idle, add into timer to retry in %dms, ref:%d", pTask->id.idStr, pTask->status.schedIdleTime, ref); // add one ref count for task - /*SStreamTask* pAddRefTask = */ streamMetaAcquireOneTask(pTask); + streamMetaAcquireOneTask(pTask); if (pTask->schedInfo.pIdleTimer == NULL) { pTask->schedInfo.pIdleTimer = taosTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer); } else { - taosTmrReset(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer, &pTask->schedInfo.pIdleTimer); + streamTmrReset(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer, + &pTask->schedInfo.pIdleTimer, pTask->pMeta->vgId, "resume-task-tmr"); } - - return TSDB_CODE_SUCCESS; } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -97,7 +94,7 @@ void streamTaskResumeHelper(void* param, void* tmrId) { SStreamTaskState p = streamTaskGetStatus(pTask); if (p.state == TASK_STATUS__DROPPING || p.state == TASK_STATUS__STOP) { - streamTaskSetSchedStatusInactive(pTask); + (void) streamTaskSetSchedStatusInactive(pTask); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s status:%s not resume task, ref:%d", pId->idStr, p.name, ref); @@ -106,21 +103,25 @@ void streamTaskResumeHelper(void* param, void* tmrId) { return; } - streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK); + int32_t code = streamTaskSchedTask(pTask->pMsgCb, pTask->info.nodeId, pId->streamId, pId->taskId, STREAM_EXEC_T_RESUME_TASK); + if (code) { + stError("s-task:%s sched task failed, code:%s", pTask->id.idStr, strerror(code)); + } else { + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("trigger to resume s-task:%s after being idled for %dms, ref:%d", pId->idStr, pTask->status.schedIdleTime, + ref); - int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("trigger to resume s-task:%s after being idled for %dms, ref:%d", pId->idStr, pTask->status.schedIdleTime, - ref); - - // release the task ref count - streamTaskClearSchedIdleInfo(pTask); - streamMetaReleaseTask(pTask->pMeta, pTask); + // release the task ref count + streamTaskClearSchedIdleInfo(pTask); + streamMetaReleaseTask(pTask->pMeta, pTask); + } } void streamTaskSchedHelper(void* param, void* tmrId) { SStreamTask* pTask = (void*)param; const char* id = pTask->id.idStr; int32_t nextTrigger = (int32_t)pTask->info.delaySchedParam; + int32_t vgId = pTask->pMeta->vgId; int8_t status = atomic_load_8(&pTask->schedInfo.status); stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger); @@ -140,7 +141,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) { if (code) { stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", nextTrigger); - taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); + streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr"); terrno = code; return; } @@ -152,7 +153,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) { stError("s-task:%s failed to prepare retrieve data trigger, code:%s, try again in %dms", id, "out of memory", nextTrigger); - taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); + streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr"); return; } @@ -161,13 +162,16 @@ void streamTaskSchedHelper(void* param, void* tmrId) { code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pTrigger); if (code != TSDB_CODE_SUCCESS) { - taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); + streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr"); return; } - streamTrySchedExec(pTask); + code = streamTrySchedExec(pTask); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s failed to sched to run, wait for next time", pTask->id.idStr); + } } } - taosTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer); + streamTmrReset(streamTaskSchedHelper, nextTrigger, pTask, streamTimer, &pTask->schedInfo.pDelayTimer, vgId, "sched-run-tmr"); } diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index 3c7ad2639a..1efb2af381 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -212,7 +212,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId, pStatus.name); - streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); + (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); return -1; // todo set the correct error code } @@ -228,11 +228,11 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { code = streamMetaAcquireTask(pMeta, hStreamId, hTaskId, &pHisTask); if (pHisTask == NULL) { stDebug("s-task:%s failed acquire and start fill-history task, it may have been dropped/stopped", idStr); - streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); + (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); } else { if (pHisTask->status.downstreamReady == 1) { // it's ready now, do nothing stDebug("s-task:%s fill-history task is ready, no need to check downstream", pHisTask->id.idStr); - streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, true); + (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, true); } else { // exist, but not ready, continue check downstream task status if (pHisTask->pBackend == NULL) { code = pMeta->expandTaskFn(pHisTask); @@ -289,7 +289,7 @@ void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, SHistoryTaskInfo* pHTaskInfo = &pTask->hTaskInfo; int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); + (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); stError("s-task:%s max retry:%d reached, quit from retrying launch related fill-history task:0x%x, ref:%d", pTask->id.idStr, MAX_RETRY_LAUNCH_HISTORY_TASK, (int32_t)pHTaskInfo->id.taskId, ref); @@ -307,7 +307,7 @@ void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, i stDebug("s-task:0x%" PRIx64 " stopped, not launch rel history task:0x%" PRIx64 ", ref:%d", pInfo->id.taskId, pInfo->hTaskId.taskId, ref); - streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); + (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); taosMemoryFree(pInfo); } else { char* p = streamTaskGetStatus(pTask).name; @@ -350,7 +350,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { streamMetaWUnLock(pMeta); // record the related fill-history task failed - streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); + (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); taosMemoryFree(pInfo); return; } @@ -407,7 +407,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { streamMetaReleaseTask(pMeta, pTask); } else { - streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); + (void) streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1); stError("s-task:0x%x rel fill-history task:0x%" PRIx64 " may have been destroyed, not launch, ref:%d", @@ -448,7 +448,7 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId); if (pInfo == NULL) { stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr); - streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); + (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); return terrno; } @@ -465,7 +465,7 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { stError("s-task:%s failed to start timer, related fill-history task not launched, ref:%d", idStr, ref); taosMemoryFree(pInfo); - streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); + (void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); return terrno; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 39e12a9da7..7c2d0b3556 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -197,13 +197,13 @@ void tFreeStreamTask(SStreamTask* pTask) { STaskExecStatisInfo* pStatis = &pTask->execInfo; ETaskStatus status1 = TASK_STATUS__UNINIT; - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); if (pTask->status.pSM != NULL) { SStreamTaskState pStatus = streamTaskGetStatus(pTask); p = pStatus.name; status1 = pStatus.state; } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); stDebug("start to free s-task:0x%x %p, state:%s", taskId, pTask, p); @@ -284,11 +284,11 @@ void tFreeStreamTask(SStreamTask* pTask) { streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo); taosMemoryFree(pTask->outputInfo.pTokenBucket); - taosThreadMutexDestroy(&pTask->lock); + streamMutexDestroy(&pTask->lock); taosArrayDestroy(pTask->msgInfo.pSendInfo); pTask->msgInfo.pSendInfo = NULL; - taosThreadMutexDestroy(&pTask->msgInfo.lock); + streamMutexDestroy(&pTask->msgInfo.lock); taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList); pTask->outputInfo.pNodeEpsetUpdateList = NULL; @@ -644,11 +644,7 @@ void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) { void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) { SStreamUpstreamEpInfo* pInfo = NULL; - - int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - return; - } + streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo); if ((pInfo != NULL) && pInfo->dataAllowed) { pInfo->dataAllowed = false; @@ -659,11 +655,7 @@ void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) { void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) { SStreamUpstreamEpInfo* pInfo = NULL; - - int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - return; - } + streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo); if (pInfo != NULL && (!pInfo->dataAllowed)) { int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); @@ -679,34 +671,34 @@ bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask) { bool streamTaskSetSchedStatusWait(SStreamTask* pTask) { bool ret = false; - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); if (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE) { pTask->status.schedStatus = TASK_SCHED_STATUS__WAITING; ret = true; } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return ret; } int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask) { - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); int8_t status = pTask->status.schedStatus; if (status == TASK_SCHED_STATUS__WAITING) { pTask->status.schedStatus = TASK_SCHED_STATUS__ACTIVE; } - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return status; } int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) { - taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); int8_t status = pTask->status.schedStatus; ASSERT(status == TASK_SCHED_STATUS__WAITING || status == TASK_SCHED_STATUS__ACTIVE || status == TASK_SCHED_STATUS__INACTIVE); pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; - taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return status; } @@ -723,7 +715,7 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) { stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr, (int32_t)sTaskId.taskId); - taosThreadMutexLock(&(*ppStreamTask)->lock); + streamMutexLock(&(*ppStreamTask)->lock); CLEAR_RELATED_FILLHISTORY_TASK((*ppStreamTask)); if (resetRelHalt) { @@ -734,7 +726,7 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) { } streamMetaSaveTask(pMeta, *ppStreamTask); - taosThreadMutexUnlock(&(*ppStreamTask)->lock); + streamMutexUnlock(&(*ppStreamTask)->lock); } return TSDB_CODE_SUCCESS; @@ -923,6 +915,7 @@ void streamTaskResume(SStreamTask* pTask) { bool streamTaskIsSinkTask(const SStreamTask* pTask) { return pTask->info.taskLevel == TASK_LEVEL__SINK; } +// this task must success int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) { int32_t code; int32_t tlen = 0; @@ -960,24 +953,23 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) { return 0; } -int32_t streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo) { +void streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo) { *pEpInfo = NULL; int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList); for (int32_t i = 0; i < num; ++i) { SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); if (pInfo == NULL) { - return TSDB_CODE_FAILED; + return; } if (pInfo->taskId == taskId) { *pEpInfo = pInfo; - return TSDB_CODE_SUCCESS; + return; } } stError("s-task:%s failed to find upstream task:0x%x", pTask->id.idStr, taskId); - return TSDB_CODE_FAILED; } SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) { @@ -1100,7 +1092,7 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) { return; } - taosThreadMutexDestroy(&pInfo->lock); + streamMutexDestroy(&pInfo->lock); taosArrayDestroy(pInfo->pDispatchTriggerList); pInfo->pDispatchTriggerList = NULL; taosArrayDestroy(pInfo->pReadyMsgList); diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 7e47857a39..c3e0df52d4 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -201,7 +201,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, pSM->pActiveTrans = pNextTrans; pSM->startTs = taosGetTimestampMs(); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); code = pNextTrans->pAction(pSM->pTask); if (pNextTrans->autoInvokeEndFn) { @@ -210,7 +210,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, return code; } } else { - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr, pSM->current.name, GET_EVT_NAME(pEvtInfo->event), StreamTaskStatusList[pEvtInfo->status].name); @@ -247,7 +247,7 @@ int32_t streamTaskRestoreStatus(SStreamTask* pTask) { SStreamTaskSM* pSM = pTask->status.pSM; int32_t code = 0; - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) { SStreamTaskState state = pSM->current; @@ -326,13 +326,13 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt return code; } - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); while (1) { // wait for the task to be here - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); ETaskStatus s = streamTaskGetStatus(pTask).state; - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event)); @@ -349,7 +349,7 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt } else { // override current active trans pSM->pActiveTrans = pTrans; pSM->startTs = taosGetTimestampMs(); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); code = pTrans->pAction(pTask); @@ -374,11 +374,11 @@ static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, ST info.callBackFn = callbackFn; code = attachWaitedEvent(pTask, &info); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); } else { // override current active trans pSM->pActiveTrans = pTrans; pSM->startTs = taosGetTimestampMs(); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); code = pTrans->pAction(pTask); // todo handle error code; @@ -400,11 +400,11 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) { STaskStateTrans* pTrans = NULL; while (1) { - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { EStreamTaskEvent evt = pSM->pActiveTrans->event; - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); @@ -414,7 +414,7 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) { pTrans = streamTaskFindTransform(pSM->current.state, event); if (pTrans == NULL) { stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, GET_EVT_NAME(event)); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } @@ -439,11 +439,11 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _ STaskStateTrans* pTrans = NULL; while (1) { - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { EStreamTaskEvent evt = pSM->pActiveTrans->event; - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); @@ -453,7 +453,7 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _ pTrans = streamTaskFindTransform(pSM->current.state, event); if (pTrans == NULL) { stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } @@ -485,7 +485,7 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even int32_t code = 0; // do update the task status - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); STaskStateTrans* pTrans = pSM->pActiveTrans; if (pTrans == NULL) { @@ -497,14 +497,14 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } if (pTrans->event != event) { stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pTrans->event)); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_INVALID_STATETRANS; } @@ -518,7 +518,7 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even // on success callback, add into lock if necessary, or maybe we should add an option for this? code = pTrans->pSuccAction(pTask); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); // todo: add parameter to control lock // after handling the callback function assigned by invoker, go on handling the waiting tasks @@ -532,13 +532,13 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event)); } - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); // tasks in waiting list if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { code = doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask); } else { - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); int64_t el = (taosGetTimestampMs() - pSM->startTs); stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id, @@ -563,14 +563,14 @@ const char* streamTaskGetStatusStr(ETaskStatus status) { void streamTaskResetStatus(SStreamTask* pTask) { SStreamTaskSM* pSM = pTask->status.pSM; - (void) taosThreadMutexLock(&pTask->lock); + streamMutexLock(&pTask->lock); stDebug("s-task:%s level:%d fill-history:%d vgId:%d set uninit, prev status:%s", pTask->id.idStr, pTask->info.taskLevel, pTask->info.fillHistory, pTask->pMeta->vgId, pSM->current.name); pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT]; pSM->pActiveTrans = NULL; taosArrayClear(pSM->pWaitingEventList); - (void) taosThreadMutexUnlock(&pTask->lock); + streamMutexUnlock(&pTask->lock); // clear the downstream ready status pTask->status.downstreamReady = 0; diff --git a/source/libs/stream/src/streamTimer.c b/source/libs/stream/src/streamTimer.c index c76ec92e33..4a6c9b5c2f 100644 --- a/source/libs/stream/src/streamTimer.c +++ b/source/libs/stream/src/streamTimer.c @@ -46,6 +46,6 @@ void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* h if (ret) { break; } - stError("vgId:%d failed to reset %s, try again", vgId, pMsg); + stError("vgId:%d failed to reset tmr: %s, try again", vgId, pMsg); } } diff --git a/source/libs/stream/src/streamUtil.c b/source/libs/stream/src/streamUtil.c new file mode 100644 index 0000000000..44c6adce5f --- /dev/null +++ b/source/libs/stream/src/streamUtil.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "streamInt.h" + +void streamMutexLock(TdThreadMutex *pMutex) { + (void) taosThreadMutexLock(pMutex); +} + +void streamMutexUnlock(TdThreadMutex *pMutex) { + (void) taosThreadMutexUnlock(pMutex); +} + +void streamMutexDestroy(TdThreadMutex *pMutex) { (void) taosThreadMutexDestroy(pMutex); } From 852dd833d05719db736f8ebf0c959e243306fccf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 Jul 2024 00:09:28 +0800 Subject: [PATCH 23/37] fix(stream): fix syntax error. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 3 ++- source/libs/executor/src/executorInt.c | 2 +- source/libs/executor/src/filloperator.c | 6 ++++-- source/libs/executor/src/hashjoinoperator.c | 3 ++- source/libs/executor/src/mergejoin.c | 14 +++++++++----- source/libs/executor/src/mergeoperator.c | 10 +++++++--- source/libs/executor/src/scanoperator.c | 12 ++++++++---- source/libs/executor/src/sortoperator.c | 6 ++++-- source/libs/executor/src/timesliceoperator.c | 3 ++- source/libs/executor/src/tsort.c | 3 ++- source/libs/stream/src/streamDispatch.c | 2 +- source/libs/stream/src/streamMeta.c | 7 +++++-- 12 files changed, 47 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 2474666e93..23e396b6a4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4648,7 +4648,8 @@ void tsdbReaderClose2(STsdbReader* pReader) { } if (pReader->resBlockInfo.freeBlock) { - pReader->resBlockInfo.pResBlock = blockDataDestroy(pReader->resBlockInfo.pResBlock); + blockDataDestroy(pReader->resBlockInfo.pResBlock); + pReader->resBlockInfo.pResBlock = NULL; } taosMemoryFree(pSupInfo->colId); diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index fad2b263b2..0476a7981c 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -970,7 +970,7 @@ void cleanupExprSupp(SExprSupp* pSupp) { taosMemoryFree(pSupp->rowEntryInfoOffset); } -void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } +void cleanupBasicInfo(SOptrBasicInfo* pInfo) { blockDataDestroy(pInfo->pRes); pInfo->pRes = NULL;} bool groupbyTbname(SNodeList* pGroupList) { bool bytbname = false; diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index e1aa75d828..ad1c6d67d4 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -310,8 +310,10 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { void destroyFillOperatorInfo(void* param) { SFillOperatorInfo* pInfo = (SFillOperatorInfo*)param; pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); - pInfo->pRes = blockDataDestroy(pInfo->pRes); - pInfo->pFinalRes = blockDataDestroy(pInfo->pFinalRes); + blockDataDestroy(pInfo->pRes); + pInfo->pRes = NULL; + blockDataDestroy(pInfo->pFinalRes); + pInfo->pFinalRes = NULL; cleanupExprSupp(&pInfo->noFillExprSupp); diff --git a/source/libs/executor/src/hashjoinoperator.c b/source/libs/executor/src/hashjoinoperator.c index 2fe2ccc56f..adc1055a6b 100755 --- a/source/libs/executor/src/hashjoinoperator.c +++ b/source/libs/executor/src/hashjoinoperator.c @@ -1065,7 +1065,8 @@ static void destroyHashJoinOperator(void* param) { hJoinFreeTableInfo(&pJoinOperator->tbs[0]); hJoinFreeTableInfo(&pJoinOperator->tbs[1]); - pJoinOperator->finBlk = blockDataDestroy(pJoinOperator->finBlk); + blockDataDestroy(pJoinOperator->finBlk); + pJoinOperator->finBlk = NULL; taosMemoryFreeClear(pJoinOperator->pResColMap); taosArrayDestroyEx(pJoinOperator->pRowBufs, hJoinFreeBufPage); diff --git a/source/libs/executor/src/mergejoin.c b/source/libs/executor/src/mergejoin.c index 5f0a2eadfb..50ce604a7c 100755 --- a/source/libs/executor/src/mergejoin.c +++ b/source/libs/executor/src/mergejoin.c @@ -3304,9 +3304,11 @@ void mJoinDestroyWindowCtx(SMJoinOperatorInfo* pJoin) { SMJoinWindowCtx* pCtx = &pJoin->ctx.windowCtx; mWinJoinResetWindowCache(pCtx, &pCtx->cache); - - pCtx->finBlk = blockDataDestroy(pCtx->finBlk); - pCtx->cache.outBlk = blockDataDestroy(pCtx->cache.outBlk); + + blockDataDestroy(pCtx->finBlk); + pCtx->finBlk = NULL; + blockDataDestroy(pCtx->cache.outBlk); + pCtx->cache.outBlk = NULL; taosArrayDestroy(pCtx->cache.grps); } @@ -3378,9 +3380,11 @@ int32_t mJoinInitWindowCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* p void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin) { SMJoinMergeCtx* pCtx = &pJoin->ctx.mergeCtx; + blockDataDestroy(pCtx->finBlk); + blockDataDestroy(pCtx->midBlk); - pCtx->finBlk = blockDataDestroy(pCtx->finBlk); - pCtx->midBlk = blockDataDestroy(pCtx->midBlk); + pCtx->finBlk = NULL; + pCtx->midBlk = NULL; } diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index 993e8c72fd..2816bae03c 100755 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -232,8 +232,11 @@ int32_t getSortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, u void destroySortMergeOperatorInfo(void* param) { SSortMergeInfo* pSortMergeInfo = param; - pSortMergeInfo->pInputBlock = blockDataDestroy(pSortMergeInfo->pInputBlock); - pSortMergeInfo->pIntermediateBlock = blockDataDestroy(pSortMergeInfo->pIntermediateBlock); + blockDataDestroy(pSortMergeInfo->pInputBlock); + pSortMergeInfo->pInputBlock = NULL; + + blockDataDestroy(pSortMergeInfo->pIntermediateBlock); + pSortMergeInfo->pIntermediateBlock = NULL; taosArrayDestroy(pSortMergeInfo->matchInfo.pList); @@ -429,7 +432,8 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { void destroyMultiwayMergeOperatorInfo(void* param) { SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param; - pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); + blockDataDestroy(pInfo->binfo.pRes); + pInfo->binfo.pRes = NULL; if (NULL != gMultiwayMergeFps[pInfo->type].closeFn) { (*gMultiwayMergeFps[pInfo->type].closeFn)(&pInfo->sortMergeInfo); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 678d8d569e..b6f24bf367 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3785,7 +3785,8 @@ static void destroyTagScanOperatorInfo(void* param) { taosArrayDestroy(pInfo->aFilterIdxs); taosArrayDestroyEx(pInfo->aUidTags, tagScanFreeUidTag); - pInfo->pRes = blockDataDestroy(pInfo->pRes); + blockDataDestroy(pInfo->pRes); + pInfo->pRes = NULL; taosArrayDestroy(pInfo->matchInfo.pList); pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo); taosMemoryFreeClear(param); @@ -4738,15 +4739,18 @@ void destroyTableMergeScanOperatorInfo(void* param) { pTableScanInfo->pSortHandle = NULL; taosHashCleanup(pTableScanInfo->mSkipTables); pTableScanInfo->mSkipTables = NULL; - pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); + blockDataDestroy(pTableScanInfo->pSortInputBlock); + pTableScanInfo->pSortInputBlock = NULL; // end one reader variable cleanupQueryTableDataCond(&pTableScanInfo->base.cond); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); - pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); + return;blockDataDestroy(pTableScanInfo->pResBlock); + pTableScanInfo->pResBlock = NULL; // remove it from the task->result list - pTableScanInfo->pReaderBlock = blockDataDestroy(pTableScanInfo->pReaderBlock); + blockDataDestroy(pTableScanInfo->pReaderBlock); + pTableScanInfo->pReaderBlock = NULL; taosArrayDestroy(pTableScanInfo->pSortInfo); stopSubTablesTableMergeScan(pTableScanInfo); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 507dbe7ee2..82eebf5310 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -368,7 +368,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { void destroySortOperatorInfo(void* param) { SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; - pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); + blockDataDestroy(pInfo->binfo.pRes); + pInfo->binfo.pRes = NULL; tsortDestroySortHandle(pInfo->pSortHandle); taosArrayDestroy(pInfo->pSortInfo); @@ -611,7 +612,8 @@ int32_t getGroupSortExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, u void destroyGroupSortOperatorInfo(void* param) { SGroupSortOperatorInfo* pInfo = (SGroupSortOperatorInfo*)param; - pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); + blockDataDestroy(pInfo->binfo.pRes); + pInfo->binfo.pRes = NULL; taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->matchInfo.pList); diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index cda22fa320..cdcc702629 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -1087,7 +1087,8 @@ _error: void destroyTimeSliceOperatorInfo(void* param) { STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param; - pInfo->pRes = blockDataDestroy(pInfo->pRes); + blockDataDestroy(pInfo->pRes); + pInfo->pRes = NULL; for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) { SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index d9bcc954a4..6d88eaef99 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -555,7 +555,8 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT (*numOfCompleted) += 1; pSource->src.rowIndex = -1; pSource->pageIndex = -1; - pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock); + blockDataDestroy(pSource->src.pBlock); + pSource->src.pBlock = NULL; } else { if (pSource->pageIndex % 512 == 0) { qDebug("begin source %p page %d", pSource, pSource->pageIndex); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index d59bb0dd91..4e128ace54 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1491,7 +1491,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S { // do send response with the input status - code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont); + int32_t code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont); if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s failed to build dispatch rsp, msgId:%d, code:%s", id, pReq->msgId, tstrerror(code)); return code; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 806eb0ce91..75dc327104 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -57,10 +57,13 @@ static void streamMetaEnvInit() { streamMetaId = taosOpenRef(64, streamMetaCloseImpl); metaRefMgtInit(); - streamTimerInit(); + int32_t code = streamTimerInit(); + if (code != 0) { + stError("failed to init stream meta env, start failed"); + } } -void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } +void streamMetaInit() { (void) taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } void streamMetaCleanup() { taosCloseRef(streamBackendId); From fbfffc4aec2e165ba539643386f15f22ca2db09a Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 19 Jul 2024 08:55:26 +0800 Subject: [PATCH 24/37] adj ci --- tests/script/tsim/stream/checkTaskStatus.sim | 4 ++-- tests/script/tsim/stream/udTableAndCol0.sim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/tsim/stream/checkTaskStatus.sim b/tests/script/tsim/stream/checkTaskStatus.sim index 13570c2a86..afaeedc298 100644 --- a/tests/script/tsim/stream/checkTaskStatus.sim +++ b/tests/script/tsim/stream/checkTaskStatus.sim @@ -7,8 +7,8 @@ loopCheck0: sleep 1000 $loop_count = $loop_count + 1 -if $loop_count == 30 then - return -1 +if $loop_count == 60 then + return 1 endi print 1 select * from information_schema.ins_stream_tasks; diff --git a/tests/script/tsim/stream/udTableAndCol0.sim b/tests/script/tsim/stream/udTableAndCol0.sim index a0aeb5bb30..29d428e276 100644 --- a/tests/script/tsim/stream/udTableAndCol0.sim +++ b/tests/script/tsim/stream/udTableAndCol0.sim @@ -24,7 +24,7 @@ sql_error create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3(a, b) as select count(*) c1, max(a) from st interval(10s); -sleep 1000 +run tsim/stream/checkTaskStatus.sim sql desc streamt3; From 140a128a1d583c298fa420a28f166a6a45e1e80b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 19 Jul 2024 09:23:46 +0800 Subject: [PATCH 25/37] adj ci --- tests/script/tsim/stream/checkTaskStatus.sim | 1 + tests/script/tsim/stream/pauseAndResume.sim | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tests/script/tsim/stream/checkTaskStatus.sim b/tests/script/tsim/stream/checkTaskStatus.sim index afaeedc298..8b30d27841 100644 --- a/tests/script/tsim/stream/checkTaskStatus.sim +++ b/tests/script/tsim/stream/checkTaskStatus.sim @@ -4,6 +4,7 @@ $loop_count = 0 loopCheck0: +print loop_count=$loop_count sleep 1000 $loop_count = $loop_count + 1 diff --git a/tests/script/tsim/stream/pauseAndResume.sim b/tests/script/tsim/stream/pauseAndResume.sim index adb85fa8c4..c2998dea30 100644 --- a/tests/script/tsim/stream/pauseAndResume.sim +++ b/tests/script/tsim/stream/pauseAndResume.sim @@ -114,6 +114,7 @@ print ===== idle 60 sec completed , continue print ===== step 1 over print ===== step2 +sql drop database if exists test; sql drop stream if exists streams2; sql drop database if exists test2; sql create database test2 vgroups 1; @@ -251,6 +252,7 @@ print ===== step 2 over print ===== step3 +sql drop database if exists test2; sql drop stream if exists streams3; sql drop database if exists test3; sql create database test3 vgroups 3; @@ -329,6 +331,7 @@ sql resume stream IF EXISTS streams66666666; print ===== step 4 over print ===== step5 +sql drop database if exists test3; sql drop stream if exists streams6; sql drop database if exists test6; sql create database test6 vgroups 10; From 144855ef83829847226ebaba0539ee19dbe9c3bd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 Jul 2024 09:39:29 +0800 Subject: [PATCH 26/37] fix(stream): fix deadlock --- source/libs/stream/src/streamCheckpoint.c | 12 +++++++----- source/libs/stream/src/streamTimer.c | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 392f64f524..60019977cc 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -202,6 +202,7 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock SStreamUpstreamEpInfo* pInfo = NULL; streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId, &pInfo); if (pInfo == NULL) { + streamMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } @@ -437,7 +438,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream stDebug("s-task:%s send checkpoint-ready msg to %d upstream confirmed, checkpointId:%" PRId64, pTask->id.idStr, numOfConfirmed, checkpointId); - streamMutexUnlock(&pTask->lock); + streamMutexUnlock(&pInfo->lock); return TSDB_CODE_SUCCESS; } @@ -530,13 +531,14 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV pTask->status.taskStatus = TASK_STATUS__READY; code = streamMetaSaveTask(pMeta, pTask); + streamMutexUnlock(&pTask->lock); + if (code != TSDB_CODE_SUCCESS) { stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id, vgId, pReq->checkpointId, terrstr()); return code; } - streamMutexUnlock(&pTask->lock); streamMetaWUnLock(pMeta); // drop task should not in the meta-lock, and drop the related fill-history task now @@ -981,7 +983,7 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { } } - streamMutexUnlock(&pTask->lock); + streamMutexUnlock(&pInfo->lock); } int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) { @@ -995,7 +997,7 @@ int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) { num++; } } - streamMutexUnlock(&pTask->lock); + streamMutexUnlock(&pInfo->lock); return num; } @@ -1017,7 +1019,7 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) { } } - streamMutexUnlock(&pTask->lock); + streamMutexUnlock(&pInfo->lock); int32_t numOfConfirmed = streamTaskGetNumOfConfirmed(pTask); int32_t total = streamTaskGetNumOfDownstream(pTask); diff --git a/source/libs/stream/src/streamTimer.c b/source/libs/stream/src/streamTimer.c index 4a6c9b5c2f..931de397cc 100644 --- a/source/libs/stream/src/streamTimer.c +++ b/source/libs/stream/src/streamTimer.c @@ -41,11 +41,11 @@ tmr_h streamTimerGetInstance() { void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, const char* pMsg) { - while (1) { +// while (1) { bool ret = taosTmrReset(fp, mseconds, param, handle, pTmrId); if (ret) { - break; +// break; } - stError("vgId:%d failed to reset tmr: %s, try again", vgId, pMsg); - } +// stError("vgId:%d failed to reset tmr: %s, try again", vgId, pMsg); +// } } From c36d6d44cdd29aab3655234edb76f0e7e851c354 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 Jul 2024 11:49:05 +0800 Subject: [PATCH 27/37] fix(stream): fix syntax error. --- source/libs/executor/src/scanoperator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b6f24bf367..4d4565ec56 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -4746,8 +4746,9 @@ void destroyTableMergeScanOperatorInfo(void* param) { cleanupQueryTableDataCond(&pTableScanInfo->base.cond); destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); - return;blockDataDestroy(pTableScanInfo->pResBlock); + blockDataDestroy(pTableScanInfo->pResBlock); pTableScanInfo->pResBlock = NULL; + // remove it from the task->result list blockDataDestroy(pTableScanInfo->pReaderBlock); pTableScanInfo->pReaderBlock = NULL; From acfced4b47755eba3c5ae682f1a197341968f8a7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jul 2024 13:02:34 +0800 Subject: [PATCH 28/37] fix issue --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 0d620827fb..0b329d23b9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -154,7 +154,9 @@ int32_t save_fs(const TFileSetArray *arr, const char *fname) { int32_t lino = 0; cJSON *json = cJSON_CreateObject(); - TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + if (json == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } // fmtv if (cJSON_AddNumberToObject(json, "fmtv", 1) == NULL) { From d475cd8449e8afa4ac0a822eee937588937dd980 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 Jul 2024 13:46:35 +0800 Subject: [PATCH 29/37] fix: function return code --- source/libs/catalog/test/catalogTests.cpp | 578 +++++++++++++--------- 1 file changed, 331 insertions(+), 247 deletions(-) diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 23fa8d000b..534070c540 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -106,7 +106,7 @@ int32_t ctgTestRspIdx = 0; void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { SCreateDbReq createReq = {0}; - strcpy(createReq.db, "1.db1"); + TAOS_STRCPY(createReq.db, "1.db1"); createReq.numOfVgroups = 2; createReq.buffer = -1; createReq.pageSize = -1; @@ -127,16 +127,19 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { createReq.ignoreExist = 1; int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + ASSERT(contLen > 0); void *pReq = rpcMallocCont(contLen); - tSerializeSCreateDbReq(pReq, contLen, &createReq); - + ASSERT(pReq != NULL); + contLen = tSerializeSCreateDbReq(pReq, contLen, &createReq); + ASSERT(contLen > 0); + SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; rpcMsg.contLen = contLen; rpcMsg.msgType = TDMT_MND_CREATE_DB; SRpcMsg rpcRsp = {0}; - rpcSendRecv(shandle, pEpSet, &rpcMsg, &rpcRsp); + ASSERT(0 == rpcSendRecv(shandle, pEpSet, &rpcMsg, &rpcRsp)); ASSERT_EQ(rpcRsp.code, 0); } @@ -152,15 +155,16 @@ void ctgTestInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; tmrDebugFlag = 159; - strcpy(tsLogDir, TD_LOG_DIR_PATH); + TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); - ctgdEnableDebug("api", true); - ctgdEnableDebug("meta", true); - ctgdEnableDebug("cache", true); - ctgdEnableDebug("lock", true); + (void)ctgdEnableDebug("api", true); + (void)ctgdEnableDebug("meta", true); + (void)ctgdEnableDebug("cache", true); + (void)ctgdEnableDebug("lock", true); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); + (void)printf("failed to open log file in directory:%s\n", tsLogDir); + ASSERT(0); } } @@ -170,21 +174,21 @@ int32_t ctgTestGetVgNumFromVgVersion(int32_t vgVersion) { void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { SName cn = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(cn.dbname, "db1"); - strcpy(cn.tname, ctgTestCTablename); + TAOS_STRCPY(cn.dbname, "db1"); + TAOS_STRCPY(cn.tname, ctgTestCTablename); SName sn = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(sn.dbname, "db1"); - strcpy(sn.tname, ctgTestSTablename); + TAOS_STRCPY(sn.dbname, "db1"); + TAOS_STRCPY(sn.tname, ctgTestSTablename); char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&cn, db); + (void)tNameGetFullDbName(&cn, db); - strcpy(output->dbFName, db); + TAOS_STRCPY(output->dbFName, db); SET_META_TYPE_BOTH_TABLE(output->metaType); - strcpy(output->ctbName, cn.tname); - strcpy(output->tbName, sn.tname); + TAOS_STRCPY(output->ctbName, cn.tname); + TAOS_STRCPY(output->tbName, sn.tname); output->ctbMeta.vgId = 9; output->ctbMeta.tableType = TSDB_CHILD_TABLE; @@ -193,6 +197,7 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { output->tbMeta = (STableMeta *)taosMemoryCalloc(1, sizeof(STableMeta) + sizeof(SSchema) * (ctgTestColNum + ctgTestColNum)); + ASSERT(NULL != output->tbMeta); output->tbMeta->vgId = 9; output->tbMeta->tableType = TSDB_SUPER_TABLE; output->tbMeta->uid = 2; @@ -209,19 +214,19 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { s->type = TSDB_DATA_TYPE_TIMESTAMP; s->colId = 1; s->bytes = 8; - strcpy(s->name, "ts"); + TAOS_STRCPY(s->name, "ts"); s = &output->tbMeta->schema[1]; s->type = TSDB_DATA_TYPE_INT; s->colId = 2; s->bytes = 4; - strcpy(s->name, "col1s"); + TAOS_STRCPY(s->name, "col1s"); s = &output->tbMeta->schema[2]; s->type = TSDB_DATA_TYPE_BINARY; s->colId = 3; s->bytes = 12; - strcpy(s->name, "tag1s"); + TAOS_STRCPY(s->name, "tag1s"); } void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { @@ -229,7 +234,8 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { int32_t vgNum = 0; SVgroupInfo vgInfo = {0}; SDBVgInfo *dbVgroup = (SDBVgInfo *)taosMemoryCalloc(1, sizeof(SDBVgInfo)); - + ASSERT(NULL != dbVgroup); + dbVgroup->vgVersion = vgVersion++; ctgTestCurrentVgVersion = dbVgroup->vgVersion; @@ -238,6 +244,7 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { dbVgroup->hashPrefix = 0; dbVgroup->hashSuffix = 0; dbVgroup->vgHash = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + ASSERT(NULL != dbVgroup->vgHash); vgNum = ctgTestGetVgNumFromVgVersion(dbVgroup->vgVersion); uint32_t hashUnit = UINT32_MAX / vgNum; @@ -250,20 +257,20 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { vgInfo.epSet.inUse = i % vgInfo.epSet.numOfEps; for (int32_t n = 0; n < vgInfo.epSet.numOfEps; ++n) { SEp *addr = &vgInfo.epSet.eps[n]; - strcpy(addr->fqdn, "a0"); + TAOS_STRCPY(addr->fqdn, "a0"); addr->port = n + 22; } - taosHashPut(dbVgroup->vgHash, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo)); + ASSERT(0 == taosHashPut(dbVgroup->vgHash, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo))); } *pdbVgroup = dbVgroup; } void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { - strcpy(rspMsg->dbFName, ctgTestDbname); - sprintf(rspMsg->tbName, "%s", ctgTestSTablename); - sprintf(rspMsg->stbName, "%s", ctgTestSTablename); + TAOS_STRCPY(rspMsg->dbFName, ctgTestDbname); + (void)sprintf(rspMsg->tbName, "%s", ctgTestSTablename); + (void)sprintf(rspMsg->stbName, "%s", ctgTestSTablename); rspMsg->numOfTags = ctgTestTagNum; rspMsg->numOfColumns = ctgTestColNum; rspMsg->precision = 1 + 1; @@ -275,41 +282,43 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { rspMsg->vgId = 1; rspMsg->pSchemas = (SSchema *)taosMemoryCalloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema)); - + ASSERT(NULL != rspMsg->pSchemas); + SSchema *s = NULL; s = &rspMsg->pSchemas[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; s->colId = 1; s->bytes = 8; - strcpy(s->name, "ts"); + TAOS_STRCPY(s->name, "ts"); s = &rspMsg->pSchemas[1]; s->type = TSDB_DATA_TYPE_INT; s->colId = 2; s->bytes = 4; - strcpy(s->name, "col1s"); + TAOS_STRCPY(s->name, "col1s"); s = &rspMsg->pSchemas[2]; s->type = TSDB_DATA_TYPE_BINARY; s->colId = 3; s->bytes = 12 + 1; - strcpy(s->name, "tag1s"); + TAOS_STRCPY(s->name, "tag1s"); return; } -void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +int32_t ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); SUseDbRsp usedbRsp = {0}; - strcpy(usedbRsp.db, ctgTestDbname); + TAOS_STRCPY(usedbRsp.db, ctgTestDbname); usedbRsp.vgVersion = ctgTestVgVersion; ctgTestCurrentVgVersion = ctgTestVgVersion; usedbRsp.vgNum = ctgTestVgNum; usedbRsp.hashMethod = 0; usedbRsp.uid = ctgTestDbId; usedbRsp.pVgroupInfos = taosArrayInit(usedbRsp.vgNum, sizeof(SVgroupInfo)); - + ASSERT(NULL != usedbRsp.pVgroupInfos); + uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; for (int32_t i = 0; i < ctgTestVgNum; ++i) { SVgroupInfo vg = {0}; @@ -324,31 +333,36 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * vg.epSet.inUse = i % vg.epSet.numOfEps; for (int32_t n = 0; n < vg.epSet.numOfEps; ++n) { SEp *addr = &vg.epSet.eps[n]; - strcpy(addr->fqdn, "a0"); + TAOS_STRCPY(addr->fqdn, "a0"); addr->port = n + 22; } vg.numOfTable = i % 2; - taosArrayPush(usedbRsp.pVgroupInfos, &vg); + ASSERT(NULL != taosArrayPush(usedbRsp.pVgroupInfos, &vg)); } int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); + ASSERT(contLen > 0); void *pReq = rpcMallocCont(contLen); - tSerializeSUseDbRsp(pReq, contLen, &usedbRsp); + ASSERT(pReq != NULL); + contLen = tSerializeSUseDbRsp(pReq, contLen, &usedbRsp); + ASSERT(contLen > 0); pRsp->code = 0; pRsp->contLen = contLen; pRsp->pCont = pReq; taosArrayDestroy(usedbRsp.pVgroupInfos); + + return 0; } -void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +int32_t ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); STableMetaRsp metaRsp = {0}; - strcpy(metaRsp.dbFName, ctgTestDbname); - strcpy(metaRsp.tbName, ctgTestTablename); + TAOS_STRCPY(metaRsp.dbFName, ctgTestDbname); + TAOS_STRCPY(metaRsp.tbName, ctgTestTablename); metaRsp.numOfTags = 0; metaRsp.numOfColumns = ctgTestColNum; metaRsp.precision = 1; @@ -359,29 +373,38 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * metaRsp.tuid = ctgTestNormalTblUid++; metaRsp.vgId = 8; metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + ASSERT(NULL != metaRsp.pSchemas); + metaRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchemaExt)); + ASSERT(NULL != metaRsp.pSchemaExt); + SSchema *s = NULL; s = &metaRsp.pSchemas[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; s->colId = 1; s->bytes = 8; - strcpy(s->name, "ts"); + TAOS_STRCPY(s->name, "ts"); s = &metaRsp.pSchemas[1]; s->type = TSDB_DATA_TYPE_INT; s->colId = 2; s->bytes = 4; - strcpy(s->name, "col1"); + TAOS_STRCPY(s->name, "col1"); int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); + ASSERT(contLen > 0); void *pReq = rpcMallocCont(contLen); - tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); - + ASSERT(pReq != NULL); + contLen = tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); + ASSERT(contLen > 0); + pRsp->code = 0; pRsp->contLen = contLen; pRsp->pCont = pReq; tFreeSTableMetaRsp(&metaRsp); + + return 0; } void ctgTestRspTableMetaNotExist(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { @@ -390,13 +413,13 @@ void ctgTestRspTableMetaNotExist(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, S pRsp->code = CTG_ERR_CODE_TABLE_NOT_EXIST; } -void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +int32_t ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); STableMetaRsp metaRsp = {0}; - strcpy(metaRsp.dbFName, ctgTestDbname); - strcpy(metaRsp.tbName, ctgTestCurrentCTableName ? ctgTestCurrentCTableName : ctgTestCTablename); - strcpy(metaRsp.stbName, ctgTestSTablename); + TAOS_STRCPY(metaRsp.dbFName, ctgTestDbname); + TAOS_STRCPY(metaRsp.tbName, ctgTestCurrentCTableName ? ctgTestCurrentCTableName : ctgTestCTablename); + TAOS_STRCPY(metaRsp.stbName, ctgTestSTablename); metaRsp.numOfTags = ctgTestTagNum; metaRsp.numOfColumns = ctgTestColNum; metaRsp.precision = 1; @@ -407,44 +430,54 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.tuid = 0x0000000000000003; metaRsp.vgId = 9; metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + ASSERT(NULL != metaRsp.pSchemas); + metaRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchemaExt)); + ASSERT(NULL != metaRsp.pSchemaExt); + SSchema *s = NULL; s = &metaRsp.pSchemas[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; s->colId = 1; s->bytes = 8; - strcpy(s->name, "ts"); + TAOS_STRCPY(s->name, "ts"); s = &metaRsp.pSchemas[1]; s->type = TSDB_DATA_TYPE_INT; s->colId = 2; s->bytes = 4; - strcpy(s->name, "col1s"); + TAOS_STRCPY(s->name, "col1s"); s = &metaRsp.pSchemas[2]; s->type = TSDB_DATA_TYPE_BINARY; s->colId = 3; s->bytes = 12; - strcpy(s->name, "tag1s"); + TAOS_STRCPY(s->name, "tag1s"); int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); + ASSERT(contLen > 0); void *pReq = rpcMallocCont(contLen); - tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); + ASSERT(pReq != NULL); + contLen = tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); + ASSERT(contLen > 0); + pRsp->code = 0; pRsp->contLen = contLen; pRsp->pCont = pReq; tFreeSTableMetaRsp(&metaRsp); + + return 0; } -void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +int32_t ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); STableMetaRsp metaRsp = {0}; - strcpy(metaRsp.dbFName, ctgTestDbname); - strcpy(metaRsp.tbName, ctgTestCurrentSTableName ? ctgTestCurrentSTableName : ctgTestSTablename); - strcpy(metaRsp.stbName, ctgTestSTablename); + TAOS_STRCPY(metaRsp.dbFName, ctgTestDbname); + TAOS_STRCPY(metaRsp.tbName, ctgTestCurrentSTableName ? ctgTestCurrentSTableName : ctgTestSTablename); + TAOS_STRCPY(metaRsp.stbName, ctgTestSTablename); metaRsp.numOfTags = ctgTestTagNum; metaRsp.numOfColumns = ctgTestColNum; metaRsp.precision = 1; @@ -455,46 +488,55 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.tuid = ctgTestSuid + 1; metaRsp.vgId = 0; metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + ASSERT(NULL != metaRsp.pSchemas); + + metaRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchemaExt)); + ASSERT(NULL != metaRsp.pSchemaExt); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; s->colId = 1; s->bytes = 8; - strcpy(s->name, "ts"); + TAOS_STRCPY(s->name, "ts"); s = &metaRsp.pSchemas[1]; s->type = TSDB_DATA_TYPE_INT; s->colId = 2; s->bytes = 4; - strcpy(s->name, "col1s"); + TAOS_STRCPY(s->name, "col1s"); s = &metaRsp.pSchemas[2]; s->type = TSDB_DATA_TYPE_BINARY; s->colId = 3; s->bytes = 12; - strcpy(s->name, "tag1s"); + TAOS_STRCPY(s->name, "tag1s"); int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); + ASSERT(contLen > 0); void *pReq = rpcMallocCont(contLen); - tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); + ASSERT(pReq != NULL); + contLen = tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); + ASSERT(contLen > 0); pRsp->code = 0; pRsp->contLen = contLen; pRsp->pCont = pReq; tFreeSTableMetaRsp(&metaRsp); + + return 0; } -void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +int32_t ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); static int32_t idx = 1; STableMetaRsp metaRsp = {0}; - strcpy(metaRsp.dbFName, ctgTestDbname); - sprintf(metaRsp.tbName, "%s_%d", ctgTestSTablename, idx); - sprintf(metaRsp.stbName, "%s_%d", ctgTestSTablename, idx); + TAOS_STRCPY(metaRsp.dbFName, ctgTestDbname); + (void)sprintf(metaRsp.tbName, "%s_%d", ctgTestSTablename, idx); + (void)sprintf(metaRsp.stbName, "%s_%d", ctgTestSTablename, idx); metaRsp.numOfTags = ctgTestTagNum; metaRsp.numOfColumns = ctgTestColNum; metaRsp.precision = 1; @@ -505,37 +547,46 @@ void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRp metaRsp.tuid = ctgTestSuid + idx; metaRsp.vgId = 0; metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); + ASSERT(NULL != metaRsp.pSchemas); + + metaRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchemaExt)); + ASSERT(NULL != metaRsp.pSchemaExt); SSchema *s = NULL; s = &metaRsp.pSchemas[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; s->colId = 1; s->bytes = 8; - strcpy(s->name, "ts"); + TAOS_STRCPY(s->name, "ts"); s = &metaRsp.pSchemas[1]; s->type = TSDB_DATA_TYPE_INT; s->colId = 2; s->bytes = 4; - strcpy(s->name, "col1s"); + TAOS_STRCPY(s->name, "col1s"); s = &metaRsp.pSchemas[2]; s->type = TSDB_DATA_TYPE_BINARY; s->colId = 3; s->bytes = 12; - strcpy(s->name, "tag1s"); + TAOS_STRCPY(s->name, "tag1s"); ++idx; int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); + ASSERT(contLen > 0); void *pReq = rpcMallocCont(contLen); - tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); + ASSERT(pReq != NULL); + contLen = tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); + ASSERT(contLen > 0); pRsp->code = 0; pRsp->contLen = contLen; pRsp->pCont = pReq; tFreeSTableMetaRsp(&metaRsp); + + return 0; } void ctgTestRspErrIndexInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { @@ -550,14 +601,17 @@ void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p rpcFreeCont(pMsg->pCont); SGetUserAuthRsp userRsp = {0}; - strcpy(userRsp.user, ctgTestUsername); + TAOS_STRCPY(userRsp.user, ctgTestUsername); userRsp.version = 1; userRsp.superAuth = 1; userRsp.enable = 1; int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &userRsp); + ASSERT(contLen > 0); void *pReq = rpcMallocCont(contLen); - tSerializeSGetUserAuthRsp(pReq, contLen, &userRsp); + ASSERT(pReq != NULL); + contLen = tSerializeSGetUserAuthRsp(pReq, contLen, &userRsp); + ASSERT(contLen > 0); pRsp->code = 0; pRsp->contLen = contLen; @@ -570,27 +624,34 @@ void ctgTestRspTableCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p static int32_t idx = 1; STableCfgRsp tblRsp = {0}; - strcpy(tblRsp.tbName, ctgTestTablename); + TAOS_STRCPY(tblRsp.tbName, ctgTestTablename); tblRsp.numOfColumns = ctgTestColNum; tblRsp.pSchemas = (SSchema *)taosMemoryMalloc((tblRsp.numOfTags + tblRsp.numOfColumns) * sizeof(SSchema)); + ASSERT(tblRsp.pSchemas != NULL); + + tblRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc((tblRsp.numOfTags + tblRsp.numOfColumns) * sizeof(SSchemaExt)); + ASSERT(NULL != tblRsp.pSchemaExt); SSchema *s = NULL; s = &tblRsp.pSchemas[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; s->colId = 1; s->bytes = 8; - strcpy(s->name, "ts"); + TAOS_STRCPY(s->name, "ts"); s = &tblRsp.pSchemas[1]; s->type = TSDB_DATA_TYPE_INT; s->colId = 2; s->bytes = 4; - strcpy(s->name, "col1"); + TAOS_STRCPY(s->name, "col1"); int32_t contLen = tSerializeSTableCfgRsp(NULL, 0, &tblRsp); + ASSERT(contLen > 0); void *pReq = rpcMallocCont(contLen); - tSerializeSTableCfgRsp(pReq, contLen, &tblRsp); + ASSERT(pReq != NULL); + contLen = tSerializeSTableCfgRsp(pReq, contLen, &tblRsp); + ASSERT(contLen > 0); pRsp->code = 0; pRsp->contLen = contLen; @@ -605,20 +666,24 @@ void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg static int32_t idx = 1; STableIndexRsp tblRsp = {0}; - strcpy(tblRsp.tbName, ctgTestSTablename); + TAOS_STRCPY(tblRsp.tbName, ctgTestSTablename); tblRsp.pIndex = taosArrayInit(ctgTestIndexNum, sizeof(STableIndexInfo)); - + ASSERT(NULL != tblRsp.pIndex); + STableIndexInfo info = {0}; for (int32_t i = 0; i < ctgTestIndexNum; ++i) { info.interval = 1 + i; info.expr = (char *)taosMemoryCalloc(1, 10); - taosArrayPush(tblRsp.pIndex, &info); + ASSERT(NULL != taosArrayPush(tblRsp.pIndex, &info)); } int32_t contLen = tSerializeSTableIndexRsp(NULL, 0, &tblRsp); + ASSERT(contLen > 0); void *pReq = rpcMallocCont(contLen); - tSerializeSTableIndexRsp(pReq, contLen, &tblRsp); + ASSERT(pReq != NULL); + contLen = tSerializeSTableIndexRsp(pReq, contLen, &tblRsp); + ASSERT(contLen > 0); pRsp->code = 0; pRsp->contLen = contLen; @@ -636,8 +701,11 @@ void ctgTestRspDBCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp dbRsp.numOfVgroups = ctgTestVgNum; int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &dbRsp); + ASSERT(contLen > 0); void *pReq = rpcMallocCont(contLen); - tSerializeSDbCfgRsp(pReq, contLen, &dbRsp); + ASSERT(pReq != NULL); + contLen = tSerializeSDbCfgRsp(pReq, contLen, &dbRsp); + ASSERT(contLen > 0); pRsp->code = 0; pRsp->contLen = contLen; @@ -653,15 +721,18 @@ void ctgTestRspQnodeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * SQueryNodeLoad nodeLoad = {0}; nodeLoad.addr.nodeId = i; - (void)taosArrayPush(qlistRsp.qnodeList, &nodeLoad); + ASSERT(NULL != taosArrayPush(qlistRsp.qnodeList, &nodeLoad)); } - int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); - void *pReq = rpcMallocCont(rspLen); - tSerializeSQnodeListRsp(pReq, rspLen, &qlistRsp); + int32_t contLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); + ASSERT(contLen > 0); + void *pReq = rpcMallocCont(contLen); + ASSERT(pReq != NULL); + contLen = tSerializeSQnodeListRsp(pReq, contLen, &qlistRsp); + ASSERT(contLen > 0); pRsp->code = 0; - pRsp->contLen = rspLen; + pRsp->contLen = contLen; pRsp->pCont = pReq; tFreeSQnodeListRsp(&qlistRsp); @@ -675,21 +746,24 @@ void ctgTestRspUdfInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pR funcRsp.pFuncInfos = taosArrayInit(1, sizeof(SFuncInfo)); funcRsp.pFuncExtraInfos = taosArrayInit(1, sizeof(SFuncExtraInfo)); SFuncInfo funcInfo = {0}; - strcpy(funcInfo.name, "func1"); + TAOS_STRCPY(funcInfo.name, "func1"); funcInfo.funcType = ctgTestFuncType; - (void)taosArrayPush(funcRsp.pFuncInfos, &funcInfo); + ASSERT(NULL != taosArrayPush(funcRsp.pFuncInfos, &funcInfo)); SFuncExtraInfo extraInfo = {0}; extraInfo.funcVersion = 0; extraInfo.funcCreatedTime = taosGetTimestampMs(); - (void)taosArrayPush(funcRsp.pFuncExtraInfos, &extraInfo); + ASSERT(NULL != taosArrayPush(funcRsp.pFuncExtraInfos, &extraInfo)); - int32_t rspLen = tSerializeSRetrieveFuncRsp(NULL, 0, &funcRsp); - void *pReq = rpcMallocCont(rspLen); - tSerializeSRetrieveFuncRsp(pReq, rspLen, &funcRsp); + int32_t contLen = tSerializeSRetrieveFuncRsp(NULL, 0, &funcRsp); + ASSERT(contLen > 0); + void *pReq = rpcMallocCont(contLen); + ASSERT(pReq != NULL); + contLen = tSerializeSRetrieveFuncRsp(pReq, contLen, &funcRsp); + ASSERT(contLen > 0); pRsp->code = 0; - pRsp->contLen = rspLen; + pRsp->contLen = contLen; pRsp->pCont = pReq; tFreeSRetrieveFuncRsp(&funcRsp); @@ -699,14 +773,17 @@ void ctgTestRspSvrVer(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRs rpcFreeCont(pMsg->pCont); SServerVerRsp verRsp = {0}; - strcpy(verRsp.ver, "1.0"); + TAOS_STRCPY(verRsp.ver, "1.0"); - int32_t rspLen = tSerializeSServerVerRsp(NULL, 0, &verRsp); - void *pReq = rpcMallocCont(rspLen); - tSerializeSServerVerRsp(pReq, rspLen, &verRsp); + int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &verRsp); + ASSERT(contLen > 0); + void *pReq = rpcMallocCont(contLen); + ASSERT(pReq != NULL); + contLen = tSerializeSServerVerRsp(pReq, contLen, &verRsp); + ASSERT(contLen > 0); pRsp->code = 0; - pRsp->contLen = rspLen; + pRsp->contLen = contLen; pRsp->pCont = pReq; } @@ -715,19 +792,23 @@ void ctgTestRspDndeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p SDnodeListRsp dRsp = {0}; dRsp.dnodeList = taosArrayInit(1, sizeof(SEpSet)); + ASSERT(dRsp.dnodeList != NULL); SEpSet epSet = {0}; epSet.numOfEps = 1; tstrncpy(epSet.eps[0].fqdn, "localhost", TSDB_FQDN_LEN); epSet.eps[0].port = 6030; - (void)taosArrayPush(dRsp.dnodeList, &epSet); + ASSERT(NULL != taosArrayPush(dRsp.dnodeList, &epSet)); - int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &dRsp); - void *pReq = rpcMallocCont(rspLen); - tSerializeSDnodeListRsp(pReq, rspLen, &dRsp); + int32_t contLen = tSerializeSDnodeListRsp(NULL, 0, &dRsp); + ASSERT(contLen > 0); + void *pReq = rpcMallocCont(contLen); + ASSERT(pReq != NULL); + contLen = tSerializeSDnodeListRsp(pReq, contLen, &dRsp); + ASSERT(contLen > 0); pRsp->code = 0; - pRsp->contLen = rspLen; + pRsp->contLen = contLen; pRsp->pCont = pReq; tFreeSDnodeListRsp(&dRsp); @@ -767,22 +848,22 @@ void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) return; } -void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +int32_t ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { switch (ctgTestRspFunc[ctgTestRspIdx]) { case CTGT_RSP_VGINFO: - ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); + ASSERT(0 == ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp)); break; case CTGT_RSP_TBMETA: - ctgTestRspTableMeta(shandle, pEpSet, pMsg, pRsp); + ASSERT(0 == ctgTestRspTableMeta(shandle, pEpSet, pMsg, pRsp)); break; case CTGT_RSP_CTBMETA: - ctgTestRspCTableMeta(shandle, pEpSet, pMsg, pRsp); + ASSERT(0 == ctgTestRspCTableMeta(shandle, pEpSet, pMsg, pRsp)); break; case CTGT_RSP_STBMETA: - ctgTestRspSTableMeta(shandle, pEpSet, pMsg, pRsp); + ASSERT(0 == ctgTestRspSTableMeta(shandle, pEpSet, pMsg, pRsp)); break; case CTGT_RSP_MSTBMETA: - ctgTestRspMultiSTableMeta(shandle, pEpSet, pMsg, pRsp); + ASSERT(0 == ctgTestRspMultiSTableMeta(shandle, pEpSet, pMsg, pRsp)); break; case CTGT_RSP_INDEXINFO_E: ctgTestRspErrIndexInfo(shandle, pEpSet, pMsg, pRsp); @@ -821,39 +902,39 @@ void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp ctgTestRspIdx++; - return; + return 0; } -void ctgTestRspDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); +int32_t ctgTestRspDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ASSERT(0 == ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp)); ctgTestSetRspTableMeta(); - return; + return 0; } -void ctgTestRspDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); +int32_t ctgTestRspDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ASSERT(0 == ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp)); ctgTestSetRspCTableMeta(); - return; + return 0; } -void ctgTestRspDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); +int32_t ctgTestRspDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ASSERT(0 == ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp)); ctgTestSetRspSTableMeta(); - return; + return 0; } -void ctgTestRspDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); +int32_t ctgTestRspDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { + ASSERT(0 == ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp)); ctgTestSetRspMultiSTableMeta(); - return; + return 0; } void ctgTestSetRspDbVgroups() { @@ -1069,7 +1150,7 @@ void *ctgTestGetDbVgroupThread(void *param) { while (!ctgTestStop) { code = catalogGetDBVgList(pCtg, mockPointer, ctgTestDbname, &vgList); if (code) { - printf("code:%x\n", code); + (void)printf("code:%x\n", code); assert(0); } @@ -1081,7 +1162,7 @@ void *ctgTestGetDbVgroupThread(void *param) { taosUsleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { - printf("Get:%d\n", n); + (void)printf("Get:%d\n", n); } } @@ -1105,7 +1186,7 @@ void *ctgTestSetSameDbVgroupThread(void *param) { taosUsleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { - printf("Set:%d\n", n); + (void)printf("Set:%d\n", n); } } @@ -1129,7 +1210,7 @@ void *ctgTestSetDiffDbVgroupThread(void *param) { taosUsleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { - printf("Set:%d\n", n); + (void)printf("Set:%d\n", n); } } @@ -1144,8 +1225,8 @@ void *ctgTestGetCtableMetaThread(void *param) { bool inCache = false; SName cn = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(cn.dbname, "db1"); - strcpy(cn.tname, ctgTestCTablename); + TAOS_STRCPY(cn.dbname, "db1"); + TAOS_STRCPY(cn.tname, ctgTestCTablename); SCtgTbMetaCtx ctx = {0}; ctx.pName = &cn; @@ -1164,7 +1245,7 @@ void *ctgTestGetCtableMetaThread(void *param) { } if (++n % ctgTestPrintNum == 0) { - printf("Get:%d\n", n); + (void)printf("Get:%d\n", n); } } @@ -1184,9 +1265,11 @@ void *ctgTestSetCtableMetaThread(void *param) { while (!ctgTestStop) { output = (STableMetaOutput *)taosMemoryMalloc(sizeof(STableMetaOutput)); + ASSERT(NULL != output); ctgTestBuildCTableMetaOutput(output); SCtgUpdateTbMetaMsg *msg = (SCtgUpdateTbMetaMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); + ASSERT(NULL != msg); msg->pCtg = pCtg; msg->pMeta = output; operation.data = msg; @@ -1200,7 +1283,7 @@ void *ctgTestSetCtableMetaThread(void *param) { taosUsleep(taosRand() % 5); } if (++n % ctgTestPrintNum == 0) { - printf("Set:%d\n", n); + (void)printf("Set:%d\n", n); } } @@ -1216,14 +1299,15 @@ void ctgTestFetchRows(TAOS_RES *result, int32_t *rows) { // fetch the records row by row while ((row = taos_fetch_row(result))) { (*rows)++; - memset(temp, 0, sizeof(temp)); - taos_print_row(temp, row, fields, num_fields); - printf("\t[%s]\n", temp); + TAOS_MEMSET(temp, 0, sizeof(temp)); + (void)taos_print_row(temp, row, fields, num_fields); + (void)printf("\t[%s]\n", temp); } } void ctgTestExecQuery(TAOS *taos, char *sql, bool fetch, int32_t *rows) { TAOS_RES *result = taos_query(taos, sql); + ASSERT(NULL != result); int code = taos_errno(result); ASSERT_EQ(code, 0); @@ -1255,8 +1339,8 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); code = catalogGetTableHashVgroup(pCtg, mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); @@ -1265,7 +1349,7 @@ TEST(tableMeta, normalTable) { while (true) { uint64_t n = 0; - ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); if (n != 1) { taosMsleep(50); } else { @@ -1273,7 +1357,7 @@ TEST(tableMeta, normalTable) { } } - memset(&vgInfo, 0, sizeof(vgInfo)); + TAOS_MEMSET(&vgInfo, 0, sizeof(vgInfo)); bool exists = false; code = catalogGetCachedTableHashVgroup(pCtg, &n, &vgInfo, &exists); ASSERT_EQ(code, 0); @@ -1322,7 +1406,7 @@ TEST(tableMeta, normalTable) { taosMemoryFree(tableMeta); tableMeta = NULL; - catalogGetCachedTableMeta(pCtg, &n, &tableMeta); + code = catalogGetCachedTableMeta(pCtg, &n, &tableMeta); ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 8); ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE); @@ -1345,19 +1429,19 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(code, 0); if (dbNum) { - printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); + (void)printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); taosMemoryFree(dbs); dbs = NULL; } else { - printf("no expired db\n"); + (void)printf("no expired db\n"); } if (stbNum) { - printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); + (void)printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); taosMemoryFree(stb); stb = NULL; } else { - printf("no expired stb\n"); + (void)printf("no expired stb\n"); } allDbNum += dbNum; @@ -1391,8 +1475,8 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestCTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestCTablename); STableMeta *tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, &n, &tableMeta); @@ -1431,7 +1515,7 @@ TEST(tableMeta, childTableCase) { taosMemoryFreeClear(tableMeta); - strcpy(n.tname, ctgTestSTablename); + TAOS_STRCPY(n.tname, ctgTestSTablename); code = catalogGetTableMeta(pCtg, mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 0); @@ -1457,19 +1541,19 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(code, 0); if (dbNum) { - printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); + (void)printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); taosMemoryFree(dbs); dbs = NULL; } else { - printf("no expired db\n"); + (void)printf("no expired db\n"); } if (stbNum) { - printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); + (void)printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); taosMemoryFree(stb); stb = NULL; } else { - printf("no expired stb\n"); + (void)printf("no expired stb\n"); } allDbNum += dbNum; @@ -1501,8 +1585,8 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestSTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestSTablename); STableMeta *tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, &n, &tableMeta); @@ -1548,8 +1632,8 @@ TEST(tableMeta, superTableCase) { tableMeta = NULL; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestCTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestCTablename); code = catalogGetTableMeta(pCtg, mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 9); @@ -1598,20 +1682,20 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(code, 0); if (dbNum) { - printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); + (void)printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); taosMemoryFree(dbs); dbs = NULL; } else { - printf("no expired db\n"); + (void)printf("no expired db\n"); } if (stbNum) { - printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); + (void)printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); taosMemoryFree(stb); stb = NULL; } else { - printf("no expired stb\n"); + (void)printf("no expired stb\n"); } allDbNum += dbNum; @@ -1645,8 +1729,8 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestSTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestSTablename); STableMeta *tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, &n, &tableMeta); @@ -1715,8 +1799,8 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestSTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestSTablename); STableMeta *tableMeta = NULL; code = catalogGetTableMeta(pCtg, mockPointer, &n, &tableMeta); @@ -1754,7 +1838,7 @@ TEST(tableMeta, updateStbMeta) { while (true) { uint64_t n = 0; - ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); if (n != 3) { taosMsleep(50); } else { @@ -1795,7 +1879,7 @@ TEST(getIndexInfo, notExists) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_INDEXINFO_E; @@ -1825,7 +1909,7 @@ TEST(refreshGetMeta, normal2normal) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; ctgTestRspFunc[1] = CTGT_RSP_TBMETA; @@ -1844,8 +1928,8 @@ TEST(refreshGetMeta, normal2normal) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); code = catalogGetTableHashVgroup(pCtg, mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); @@ -1854,7 +1938,7 @@ TEST(refreshGetMeta, normal2normal) { while (true) { uint64_t n = 0; - ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); if (n > 0) { break; } @@ -1904,7 +1988,7 @@ TEST(refreshGetMeta, normal2notexist) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; ctgTestRspFunc[1] = CTGT_RSP_TBMETA; @@ -1923,8 +2007,8 @@ TEST(refreshGetMeta, normal2notexist) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); code = catalogGetTableHashVgroup(pCtg, mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); @@ -1933,7 +2017,7 @@ TEST(refreshGetMeta, normal2notexist) { while (true) { uint64_t n = 0; - ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); if (n > 0) { break; } @@ -1974,7 +2058,7 @@ TEST(refreshGetMeta, normal2child) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; ctgTestRspFunc[1] = CTGT_RSP_TBMETA; @@ -1994,8 +2078,8 @@ TEST(refreshGetMeta, normal2child) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); ctgTestCurrentCTableName = ctgTestTablename; ctgTestCurrentSTableName = ctgTestSTablename; @@ -2006,7 +2090,7 @@ TEST(refreshGetMeta, normal2child) { while (true) { uint64_t n = 0; - ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); if (n > 0) { break; } @@ -2057,7 +2141,7 @@ TEST(refreshGetMeta, stable2child) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; ctgTestRspFunc[1] = CTGT_RSP_STBMETA; @@ -2078,8 +2162,8 @@ TEST(refreshGetMeta, stable2child) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); ctgTestCurrentSTableName = ctgTestTablename; ctgTestCurrentCTableName = ctgTestTablename; @@ -2090,7 +2174,7 @@ TEST(refreshGetMeta, stable2child) { while (true) { uint64_t n = 0; - ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); if (n > 0) { break; } @@ -2143,7 +2227,7 @@ TEST(refreshGetMeta, stable2stable) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; ctgTestRspFunc[1] = CTGT_RSP_STBMETA; @@ -2164,8 +2248,8 @@ TEST(refreshGetMeta, stable2stable) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); ctgTestCurrentSTableName = ctgTestTablename; code = catalogGetTableHashVgroup(pCtg, mockPointer, &n, &vgInfo); @@ -2175,7 +2259,7 @@ TEST(refreshGetMeta, stable2stable) { while (true) { uint64_t n = 0; - ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); if (n > 0) { break; } @@ -2229,7 +2313,7 @@ TEST(refreshGetMeta, child2stable) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; ctgTestRspFunc[1] = CTGT_RSP_CTBMETA; @@ -2250,8 +2334,8 @@ TEST(refreshGetMeta, child2stable) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); ctgTestCurrentCTableName = ctgTestTablename; ctgTestCurrentSTableName = ctgTestSTablename; @@ -2262,7 +2346,7 @@ TEST(refreshGetMeta, child2stable) { while (true) { uint64_t n = 0; - ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); if (n > 0) { break; } @@ -2315,7 +2399,7 @@ TEST(tableDistVgroup, normalTable) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; ctgTestRspFunc[1] = CTGT_RSP_TBMETA; @@ -2334,8 +2418,8 @@ TEST(tableDistVgroup, normalTable) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); code = catalogGetTableDistVgInfo(pCtg, mockPointer, &n, &vgList); ASSERT_TRUE(code != 0); @@ -2352,7 +2436,7 @@ TEST(tableDistVgroup, childTableCase) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; ctgTestRspFunc[1] = CTGT_RSP_CTBMETA; @@ -2372,8 +2456,8 @@ TEST(tableDistVgroup, childTableCase) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestCTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestCTablename); code = catalogGetTableDistVgInfo(pCtg, mockPointer, &n, &vgList); ASSERT_TRUE(code != 0); @@ -2390,7 +2474,7 @@ TEST(tableDistVgroup, superTableCase) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; ctgTestRspFunc[1] = CTGT_RSP_STBMETA; @@ -2409,8 +2493,8 @@ TEST(tableDistVgroup, superTableCase) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestSTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestSTablename); code = catalogGetTableDistVgInfo(pCtg, mockPointer, &n, &vgList); ASSERT_EQ(code, 0); @@ -2441,7 +2525,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; ctgTestRspFunc[1] = CTGT_RSP_TBMETA; @@ -2459,8 +2543,8 @@ TEST(dbVgroup, getSetDbVgroupCase) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); code = catalogGetDBVgList(pCtg, mockPointer, ctgTestDbname, &vgList); ASSERT_EQ(code, 0); @@ -2470,7 +2554,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { while (true) { uint64_t n = 0; - ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); if (n > 0) { break; } @@ -2501,7 +2585,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { while (true) { uint64_t n = 0; - ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); if (n != 3) { taosMsleep(50); } else { @@ -2545,17 +2629,17 @@ TEST(multiThread, getSetRmSameDbVgroup) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + (void)taosThreadAttrInit(&thattr); TdThread thread1, thread2; - taosThreadCreate(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg); + (void)taosThreadCreate(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg); taosSsleep(1); - taosThreadCreate(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); + (void)taosThreadCreate(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); while (true) { if (ctgTestDeadLoop) { @@ -2597,17 +2681,17 @@ TEST(multiThread, getSetRmDiffDbVgroup) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + (void)taosThreadAttrInit(&thattr); TdThread thread1, thread2; - taosThreadCreate(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg); + (void)taosThreadCreate(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg); taosSsleep(1); - taosThreadCreate(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); + (void)taosThreadCreate(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg); while (true) { if (ctgTestDeadLoop) { @@ -2649,16 +2733,16 @@ TEST(multiThread, ctableMeta) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + (void)taosThreadAttrInit(&thattr); TdThread thread1, thread2; - taosThreadCreate(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg); + (void)taosThreadCreate(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg); taosSsleep(1); - taosThreadCreate(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); + (void)taosThreadCreate(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); while (true) { if (ctgTestDeadLoop) { @@ -2701,10 +2785,10 @@ TEST(rentTest, allRent) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); + TAOS_STRCPY(n.dbname, "db1"); for (int32_t i = 1; i <= 10; ++i) { - sprintf(n.tname, "%s_%d", ctgTestSTablename, i); + (void)sprintf(n.tname, "%s_%d", ctgTestSTablename, i); STableMeta *tableMeta = NULL; code = catalogGetSTableMeta(pCtg, mockPointer, &n, &tableMeta); @@ -2728,25 +2812,25 @@ TEST(rentTest, allRent) { code = catalogGetExpiredDBs(pCtg, &dbs, &num); ASSERT_EQ(code, 0); - printf("%d - expired dbNum:%d\n", i, num); + (void)printf("%d - expired dbNum:%d\n", i, num); if (dbs) { - printf("%d - expired dbId:%" PRId64 ", vgVersion:%d\n", i, dbs->dbId, dbs->vgVersion); + (void)printf("%d - expired dbId:%" PRId64 ", vgVersion:%d\n", i, dbs->dbId, dbs->vgVersion); taosMemoryFree(dbs); dbs = NULL; } code = catalogGetExpiredSTables(pCtg, &stable, &num); ASSERT_EQ(code, 0); - printf("%d - expired stableNum:%d\n", i, num); + (void)printf("%d - expired stableNum:%d\n", i, num); if (stable) { for (int32_t n = 0; n < num; ++n) { - printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, + (void)printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); } taosMemoryFree(stable); stable = NULL; } - printf("*************************************************\n"); + (void)printf("*************************************************\n"); taosSsleep(2); } @@ -2761,7 +2845,7 @@ TEST(apiTest, catalogRefreshDBVgInfo_test) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; @@ -2788,7 +2872,7 @@ TEST(apiTest, catalogChkAuth_test) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_USERAUTH; @@ -2804,8 +2888,8 @@ TEST(apiTest, catalogChkAuth_test) { SUserAuthInfo authInfo = {0}; SUserAuthRes authRes = {0}; - strcpy(authInfo.user, ctgTestUsername); - toName(1, ctgTestDbname, ctgTestSTablename, &authInfo.tbName); + TAOS_STRCPY(authInfo.user, ctgTestUsername); + (void)toName(1, ctgTestDbname, ctgTestSTablename, &authInfo.tbName); authInfo.type = AUTH_TYPE_READ; bool exists = false; code = catalogChkAuthFromCache(pCtg, &authInfo, &authRes, &exists); @@ -2818,7 +2902,7 @@ TEST(apiTest, catalogChkAuth_test) { while (true) { uint64_t n = 0; - ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n)); if (n != 1) { taosMsleep(50); } else { @@ -2841,7 +2925,7 @@ TEST(apiTest, catalogRefreshGetTableCfg_test) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_VGINFO; ctgTestRspFunc[1] = CTGT_RSP_TBMETA; @@ -2857,8 +2941,8 @@ TEST(apiTest, catalogRefreshGetTableCfg_test) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); STableCfg *pCfg = NULL; code = catalogRefreshGetTableCfg(pCtg, mockPointer, &n, &pCfg); @@ -2879,7 +2963,7 @@ TEST(apiTest, catalogGetTableIndex_test) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_TBLINDEX; @@ -2894,8 +2978,8 @@ TEST(apiTest, catalogGetTableIndex_test) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); SArray *pRes = NULL; code = catalogGetTableIndex(pCtg, mockPointer, &n, &pRes); @@ -2915,7 +2999,7 @@ TEST(apiTest, catalogGetDBCfg_test) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_DBCFG; @@ -2930,8 +3014,8 @@ TEST(apiTest, catalogGetDBCfg_test) { ASSERT_EQ(code, 0); SName n = {TSDB_TABLE_NAME_T, 1, {0}, {0}}; - strcpy(n.dbname, "db1"); - strcpy(n.tname, ctgTestTablename); + TAOS_STRCPY(n.dbname, "db1"); + TAOS_STRCPY(n.tname, ctgTestTablename); SDbCfgInfo cfgInfo = {0}; code = catalogGetDBCfg(pCtg, mockPointer, ctgTestDbname, &cfgInfo); @@ -2948,7 +3032,7 @@ TEST(apiTest, catalogGetQnodeList_test) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_QNODELIST; @@ -2982,7 +3066,7 @@ TEST(apiTest, catalogGetUdfInfo_test) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_UDF; @@ -3011,7 +3095,7 @@ TEST(apiTest, catalogGetServerVersion_test) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_SVRVER; @@ -3040,7 +3124,7 @@ TEST(apiTest, catalogUpdateTableIndex_test) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_SVRVER; @@ -3055,8 +3139,8 @@ TEST(apiTest, catalogUpdateTableIndex_test) { ASSERT_EQ(code, 0); STableIndexRsp rsp = {0}; - strcpy(rsp.dbFName, ctgTestDbname); - strcpy(rsp.tbName, ctgTestSTablename); + TAOS_STRCPY(rsp.dbFName, ctgTestDbname); + TAOS_STRCPY(rsp.tbName, ctgTestSTablename); rsp.suid = ctgTestSuid; rsp.version = 1; code = catalogUpdateTableIndex(pCtg, &rsp); @@ -3072,7 +3156,7 @@ TEST(apiTest, catalogGetDnodeList_test) { ctgTestInitLogFile(); - memset(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); + TAOS_MEMSET(ctgTestRspFunc, 0, sizeof(ctgTestRspFunc)); ctgTestRspIdx = 0; ctgTestRspFunc[0] = CTGT_RSP_DNODElIST; @@ -3103,17 +3187,17 @@ TEST(intTest, autoCreateTableTest) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_TRUE(NULL != taos); - ctgdEnableDebug("api", true); - ctgdEnableDebug("meta", true); - ctgdEnableDebug("cache", true); - ctgdEnableDebug("lock", true); + (void)ctgdEnableDebug("api", true); + (void)ctgdEnableDebug("meta", true); + (void)ctgdEnableDebug("cache", true); + (void)ctgdEnableDebug("lock", true); ctgTestExecQuery(taos, "drop database if exists db1", false, NULL); ctgTestExecQuery(taos, "create database db1", false, NULL); ctgTestExecQuery(taos, "create stable db1.st1 (ts timestamp, f1 int) tags(tg1 int)", false, NULL); ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 1)", false, NULL); - ctgdGetOneHandle(&pCtg); + (void)ctgdGetOneHandle(&pCtg); while (true) { uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); From 6a71994570ae8d088b95b07bacd549535ee3c6fd Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 19 Jul 2024 14:50:31 +0800 Subject: [PATCH 30/37] adj stream op result --- .../executor/src/streamcountwindowoperator.c | 4 ++-- .../executor/src/streameventwindowoperator.c | 4 ++-- source/libs/executor/src/streamfilloperator.c | 4 ++-- .../executor/src/streamtimewindowoperator.c | 22 +++++++++---------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 68ff0057a5..04310276a9 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -55,13 +55,13 @@ void destroyStreamCountAggOperatorInfo(void* param) { pInfo->pUpdated = NULL; colDataDestroy(&pInfo->twAggSup.timeWindowData); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + blockDataDestroy(pInfo->pDelRes); tSimpleHashCleanup(pInfo->pStUpdated); tSimpleHashCleanup(pInfo->pStDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); + blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 2e4ea29437..2d6f9b1fc5 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -60,14 +60,14 @@ void destroyStreamEventOperatorInfo(void* param) { taosArrayDestroy(pInfo->pChildren); } colDataDestroy(&pInfo->twAggSup.timeWindowData); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + blockDataDestroy(pInfo->pDelRes); tSimpleHashCleanup(pInfo->pSeUpdated); tSimpleHashCleanup(pInfo->pAllUpdated); tSimpleHashCleanup(pInfo->pSeDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - pInfo->pDelRes = blockDataDestroy(pInfo->pCheckpointRes); + blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index 9468738d43..314a226b5d 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -123,8 +123,8 @@ void destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { static void destroyStreamFillOperatorInfo(void* param) { SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param; - pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo); - pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); + destroyStreamFillInfo(pInfo->pFillInfo); + destroyStreamFillSupporter(pInfo->pFillSup); blockDataDestroy(pInfo->pRes); pInfo->pRes = NULL; blockDataDestroy(pInfo->pSrcBlock); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index da68c0e2b5..50b20344d6 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -465,11 +465,11 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { taosHashCleanup(pInfo->pPullDataMap); taosHashCleanup(pInfo->pFinalPullDataMap); taosArrayDestroy(pInfo->pPullWins); - pInfo->pPullDataRes = blockDataDestroy(pInfo->pPullDataRes); + blockDataDestroy(pInfo->pPullDataRes); taosArrayDestroy(pInfo->pDelWins); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); - pInfo->pMidRetriveRes = blockDataDestroy(pInfo->pMidRetriveRes); - pInfo->pMidPulloverRes = blockDataDestroy(pInfo->pMidPulloverRes); + blockDataDestroy(pInfo->pDelRes); + blockDataDestroy(pInfo->pMidRetriveRes); + blockDataDestroy(pInfo->pMidPulloverRes); pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState); taosArrayDestroy(pInfo->pMidPullDatas); @@ -486,7 +486,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { pInfo->pUpdatedMap = NULL; tSimpleHashCleanup(pInfo->pDeletedMap); - pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); + blockDataDestroy(pInfo->pCheckpointRes); taosMemoryFreeClear(param); } @@ -1943,7 +1943,7 @@ _error: void destroyStreamAggSupporter(SStreamAggSupporter* pSup) { tSimpleHashCleanup(pSup->pResultRows); destroyDiskbasedBuf(pSup->pResultBuf); - pSup->pScanBlock = blockDataDestroy(pSup->pScanBlock); + blockDataDestroy(pSup->pScanBlock); pSup->stateStore.streamFileStateDestroy(pSup->pState->pFileState); taosMemoryFreeClear(pSup->pState); taosMemoryFreeClear(pSup->pDummyCtx); @@ -1968,14 +1968,14 @@ void destroyStreamSessionAggOperatorInfo(void* param) { } colDataDestroy(&pInfo->twAggSup.timeWindowData); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); - pInfo->pWinBlock = blockDataDestroy(pInfo->pWinBlock); + blockDataDestroy(pInfo->pDelRes); + blockDataDestroy(pInfo->pWinBlock); tSimpleHashCleanup(pInfo->pStUpdated); tSimpleHashCleanup(pInfo->pStDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); + blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); taosMemoryFreeClear(param); @@ -3946,13 +3946,13 @@ void destroyStreamStateOperatorInfo(void* param) { taosArrayDestroy(pInfo->pChildren); } colDataDestroy(&pInfo->twAggSup.timeWindowData); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + blockDataDestroy(pInfo->pDelRes); tSimpleHashCleanup(pInfo->pSeUpdated); tSimpleHashCleanup(pInfo->pSeDeleted); cleanupGroupResInfo(&pInfo->groupResInfo); taosArrayDestroy(pInfo->historyWins); - pInfo->pCheckpointRes = blockDataDestroy(pInfo->pCheckpointRes); + blockDataDestroy(pInfo->pCheckpointRes); tSimpleHashCleanup(pInfo->pPkDeleted); taosMemoryFreeClear(param); From 0792b025eabde7cc4240b7d7f93e8ebad2d18c25 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 19 Jul 2024 15:08:34 +0800 Subject: [PATCH 31/37] fix: insert clause mixed with bound and unspecified columns --- include/libs/qcom/query.h | 1 + source/libs/parser/inc/parInsertUtil.h | 1 + source/libs/parser/src/parInsertSql.c | 3 +++ source/libs/parser/src/parInsertUtil.c | 9 +++++++++ tests/system-test/1-insert/insert_column_value.py | 13 +++++++++++-- 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 2078455f1d..347fb203ef 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -192,6 +192,7 @@ typedef struct SBoundColInfo { int16_t* pColIndex; // bound index => schema index int32_t numOfCols; int32_t numOfBound; + bool hasBoundCols; } SBoundColInfo; typedef struct STableColsData { diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 4f2877fcf6..899661a18c 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -45,6 +45,7 @@ int16_t insFindCol(struct SToken *pColname, int16_t start, int16_t end, SSchema void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname, SArray *tagName, uint8_t tagNum, int32_t ttl); int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); +void insResetBoundColsInfo(SBoundColInfo *pInfo); void insInitColValues(STableMeta *pTableMeta, SArray *aColValues); void insCheckTableDataOrder(STableDataCxt *pTableCxt, SRowKey *rowKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 313d9449d2..0d0fe75ddb 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -183,6 +183,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E } pBoundInfo->numOfBound = 0; + pBoundInfo->hasBoundCols = true; bool hasPK = pTableMeta->tableInfo.numOfPKs; int16_t numOfBoundPKs = 0; @@ -1379,6 +1380,8 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOp if (NULL != pStmt->pBoundCols) { return parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo); + } else if (pTableCxt->boundColsInfo.hasBoundCols) { + insResetBoundColsInfo(&pTableCxt->boundColsInfo); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 17b5733ff7..7e1893ae00 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -180,6 +180,7 @@ void insInitColValues(STableMeta* pTableMeta, SArray* aColValues) { initColValue int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { pInfo->numOfCols = numOfBound; pInfo->numOfBound = numOfBound; + pInfo->hasBoundCols = false; pInfo->pColIndex = taosMemoryCalloc(numOfBound, sizeof(int16_t)); if (NULL == pInfo->pColIndex) { return TSDB_CODE_OUT_OF_MEMORY; @@ -190,6 +191,14 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { return TSDB_CODE_SUCCESS; } +void insResetBoundColsInfo(SBoundColInfo* pInfo) { + pInfo->numOfBound = pInfo->numOfCols; + pInfo->hasBoundCols = false; + for (int32_t i = 0; i < pInfo->numOfCols; ++i) { + pInfo->pColIndex[i] = i; + } +} + void insCheckTableDataOrder(STableDataCxt* pTableCxt, SRowKey* rowKey) { // once the data block is disordered, we do NOT keep last timestamp any more if (!pTableCxt->ordered) { diff --git a/tests/system-test/1-insert/insert_column_value.py b/tests/system-test/1-insert/insert_column_value.py index fcb83e2f97..956fa0d229 100644 --- a/tests/system-test/1-insert/insert_column_value.py +++ b/tests/system-test/1-insert/insert_column_value.py @@ -45,7 +45,7 @@ class TDTestCase: self.TIMESTAMP_BASE = 1706716800 tdSql.init(conn.cursor()) tdSql.execute(f'drop database if exists db') - tdSql.execute(f'create database if not exists db vgroups 1') + tdSql.execute(f'create database if not exists db vgroups 1 keep 10512000m') tdLog.printNoPrefix("create table") self.__create_tb() @@ -516,10 +516,19 @@ class TDTestCase: # check json self.__insert_query_json("db", "stb_js", "ctb_js", OK_JS, KO_JS, "\'{\"k1\":\"v1\",\"k2\":\"v2\"}\'") + def __insert_query_ts5184(self, dbname="db", stbname="stb_ts5184", ctbname="ctb_ts5184", ntbname="ntb_ts5184"): + TB_LIST = [ ctbname, ntbname] + tdSql.execute(f'create table {dbname}.{stbname} (ts timestamp, w_ts timestamp, opc nchar(100),quality int) tags(t0 int);') + tdSql.execute(f'create table {dbname}.{ntbname} (ts timestamp, w_ts timestamp, opc nchar(100),quality int);') + tdSql.execute(f'create table {dbname}.{ctbname} using {dbname}.{stbname} tags(1);') + for _tb in TB_LIST: + tdSql.execute(f'insert into {dbname}.{_tb} values(1721265436000,now(),"0",192) {dbname}.{_tb}(quality,w_ts,ts) values(192,now(),1721265326000) {dbname}.{_tb}(quality,w_ts,ts) values(190,now()+1s,1721265326000) {dbname}.{_tb} values(1721265436000,now()+2s,"1",191) {dbname}.{_tb}(quality,w_ts,ts) values(192,now()+3s,1721265326002) {dbname}.{_tb}(ts,w_ts,opc,quality) values(1721265436003,now()+4s,"3",193);'); + tdSql.query(f'select * from {dbname}.{_tb}', show=True) + tdSql.checkRows(4) def run(self): self.__insert_query_exec() - + self.__insert_query_ts5184() def stop(self): tdSql.close() From 41d175a6a11de11061475fc01a6da79a82e08659 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 19 Jul 2024 15:12:32 +0800 Subject: [PATCH 32/37] fix: insert clause mixed with bound and unspecified columns --- tests/system-test/1-insert/insert_column_value.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/1-insert/insert_column_value.py b/tests/system-test/1-insert/insert_column_value.py index 956fa0d229..c488a13a88 100644 --- a/tests/system-test/1-insert/insert_column_value.py +++ b/tests/system-test/1-insert/insert_column_value.py @@ -530,6 +530,7 @@ class TDTestCase: self.__insert_query_exec() self.__insert_query_ts5184() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") From 48ef6b348c311756db9129f8104a280a21f3b54c Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 19 Jul 2024 15:56:51 +0800 Subject: [PATCH 33/37] test: add ci test machines --- Jenkinsfile2 | 4 ++-- tests/parallel_test/cases.task | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 83a4b0ffcb..083d96e975 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -401,7 +401,7 @@ pipeline { } } stage('linux test') { - agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} + agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } when { changeRequest() @@ -454,7 +454,7 @@ pipeline { cd ${WKC}/tests/parallel_test export DEFAULT_RETRY_TIME=2 date - ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 900 ''' + extra_param + ''' + ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 1200 ''' + extra_param + ''' ''' } } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 848917ceb5..bb67aa5a1f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -27,7 +27,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f insert/insert_basic.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f cluster/splitVgroupByLearner.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f authorith/authBasic.py -N 3 -,,n,army,python3 ./test.py -f cmdline/fullopt.py +# ,,n,army,python3 ./test.py -f cmdline/fullopt.py ,,n,army,python3 ./test.py -f query/show.py -N 3 ,,n,army,python3 ./test.py -f alter/alterConfig.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/subquery/subqueryBugs.py -N 3 @@ -278,8 +278,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 +#,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 +#,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 From c7a0d4c72770d7181b9d64108561fb69444dfb77 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jul 2024 16:12:38 +0800 Subject: [PATCH 34/37] enh: refact morecode --- include/util/taoserror.h | 1 + source/dnode/vnode/src/vnd/vnodeAsync.c | 29 ++++++++------ source/dnode/vnode/src/vnd/vnodeBufPool.c | 22 ++++------- source/dnode/vnode/src/vnd/vnodeCfg.c | 3 +- source/dnode/vnode/src/vnd/vnodeInitApi.c | 8 ++-- source/dnode/vnode/src/vnd/vnodeModule.c | 20 ++-------- source/util/src/terror.c | 1 + source/util/src/tjson.c | 48 +++++++++++++++-------- 8 files changed, 67 insertions(+), 65 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 9c760a465a..41c9184d27 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -556,6 +556,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C) #define TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE TAOS_DEF_ERROR_CODE(0, 0x061D) +#define TSDB_CODE_TDB_INCONSISTENT_DB_ID TAOS_DEF_ERROR_CODE(0, 0x061E) // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index 6d6533463b..126c394f54 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -190,10 +190,10 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async, SArray *cancelArray) { task->prev->next = task->next; task->next->prev = task->prev; if (task->cancel) { - taosArrayPush(cancelArray, &(SVATaskCancelInfo){ - .cancel = task->cancel, - .arg = task->arg, - }); + TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = task->cancel, + .arg = task->arg, + })); } vnodeAsyncTaskDone(async, task); } @@ -206,6 +206,9 @@ static void *vnodeAsyncLoop(void *arg) { SVWorker *worker = (SVWorker *)arg; SVAsync *async = worker->async; SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo)); + if (cancelArray == NULL) { + return NULL; + } setThreadName(async->label); @@ -466,7 +469,7 @@ int32_t vnodeAsyncOpen(int32_t numOfThreads) { vnodeAsyncSetWorkers(2, numOfThreads); _exit: - return 0; + return code; } int32_t vnodeAsyncClose() { @@ -748,10 +751,10 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) { task->prev->next = task->next; task->next->prev = task->prev; if (task->cancel) { - taosArrayPush(cancelArray, &(SVATaskCancelInfo){ - .cancel = task->cancel, - .arg = task->arg, - }); + TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = task->cancel, + .arg = task->arg, + })); } vnodeAsyncTaskDone(async, task); } @@ -763,10 +766,10 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) { channel->scheduled->prev->next = channel->scheduled->next; channel->scheduled->next->prev = channel->scheduled->prev; if (channel->scheduled->cancel) { - taosArrayPush(cancelArray, &(SVATaskCancelInfo){ - .cancel = channel->scheduled->cancel, - .arg = channel->scheduled->arg, - }); + TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = channel->scheduled->cancel, + .arg = channel->scheduled->arg, + })); } vnodeAsyncTaskDone(async, channel->scheduled); } diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 40112c5579..ad183839d7 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -16,13 +16,12 @@ #include "vnd.h" /* ------------------------ STRUCTURES ------------------------ */ -static int vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPool **ppPool) { +static int32_t vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPool **ppPool) { SVBufPool *pPool; pPool = taosMemoryMalloc(sizeof(SVBufPool) + size); if (pPool == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } memset(pPool, 0, sizeof(SVBufPool)); @@ -44,14 +43,12 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPoo pPool->lock = taosMemoryMalloc(sizeof(TdThreadSpinlock)); if (!pPool->lock) { taosMemoryFree(pPool); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } if (taosThreadSpinInit(pPool->lock, 0) != 0) { taosMemoryFree((void *)pPool->lock); taosMemoryFree(pPool); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + return terrno = TAOS_SYSTEM_ERROR(errno); } } else { pPool->lock = NULL; @@ -77,10 +74,11 @@ int vnodeOpenBufPool(SVnode *pVnode) { for (int i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) { // create pool - if (vnodeBufPoolCreate(pVnode, i, size, &pVnode->aBufPool[i])) { + int32_t code; + if ((code = vnodeBufPoolCreate(pVnode, i, size, &pVnode->aBufPool[i]))) { vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); vnodeCloseBufPool(pVnode); - return -1; + return code; } // add to free list @@ -274,8 +272,6 @@ _exit: } int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) { - int32_t code = 0; - taosThreadMutexLock(&pPool->mutex); pQNode->pNext = pPool->qList.pNext; @@ -285,9 +281,7 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) { pPool->nQuery++; taosThreadMutexUnlock(&pPool->mutex); - -_exit: - return code; + return 0; } void vnodeBufPoolDeregisterQuery(SVBufPool *pPool, SQueryNode *pQNode, bool proactive) { diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index c7b54d36b6..85981ac9aa 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -394,8 +394,7 @@ int vnodeValidateTableHash(SVnode *pVnode, char *tableFName) { } if (hashValue < pVnode->config.hashBegin || hashValue > pVnode->config.hashEnd) { - terrno = TSDB_CODE_VND_HASH_MISMATCH; - return -1; + return terrno = TSDB_CODE_VND_HASH_MISMATCH; } return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 48852dd159..9be84b99f4 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -129,17 +129,17 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable; pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; - pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it -// pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it + pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it + // pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it pTq->tqGetResultBlock = tqGetResultBlock; -// pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; + // pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; pTq->tqGetResultBlockTime = tqGetResultBlockTime; pTq->tqGetStreamExecProgress = tqGetStreamExecInfo; - } +} void initStateStoreAPI(SStateStore* pStore) { pStore->streamFileStateInit = streamFileStateInit; diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 3a454c53ef..228cc9e0b2 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -19,31 +19,19 @@ static volatile int32_t VINIT = 0; int vnodeInit(int nthreads) { - int32_t init; - - init = atomic_val_compare_exchange_32(&VINIT, 0, 1); - if (init) { + if (atomic_val_compare_exchange_32(&VINIT, 0, 1)) { return 0; } - if (vnodeAsyncOpen(nthreads) != 0) { - return -1; - } - - if (walInit() < 0) { - return -1; - } + TAOS_CHECK_RETURN(vnodeAsyncOpen(nthreads)); + TAOS_CHECK_RETURN(walInit()); return 0; } void vnodeCleanup() { - int32_t init = atomic_val_compare_exchange_32(&VINIT, 1, 0); - if (init == 0) return; - - // set stop + if (atomic_val_compare_exchange_32(&VINIT, 1, 0) == 0) return; vnodeAsyncClose(); - walCleanUp(); smaCleanUp(); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 990c84371f..4db1475fa9 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -435,6 +435,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, "Table schema is old") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR, "TDB env open error") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE, "Table already exists in other stables") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INCONSISTENT_DB_ID, "Inconsistent database id") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 82993e8449..1d2b3f003c 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -49,8 +49,7 @@ int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number) { if (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } return TSDB_CODE_SUCCESS; @@ -58,8 +57,7 @@ int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double num int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean) { if (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } return TSDB_CODE_SUCCESS; @@ -67,8 +65,7 @@ int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) { if (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } return TSDB_CODE_SUCCESS; @@ -87,8 +84,7 @@ int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem) { return TSDB_CODE_SUCCESS; } - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) { @@ -96,8 +92,7 @@ int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem) { return TSDB_CODE_SUCCESS; } - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + return terrno = TSDB_CODE_OUT_OF_MEMORY; } int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj) { @@ -106,18 +101,27 @@ int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void } SJson* pJobj = tjsonCreateObject(); - if (NULL == pJobj || TSDB_CODE_SUCCESS != func(pObj, pJobj)) { + if (NULL == pJobj) { + return terrno; + } + int32_t rc = func(pObj, pJobj); + if (rc != TSDB_CODE_SUCCESS) { tjsonDelete(pJobj); - return TSDB_CODE_FAILED; + return rc; } return tjsonAddItemToObject(pJson, pName, pJobj); } int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) { SJson* pJobj = tjsonCreateObject(); - if (NULL == pJobj || TSDB_CODE_SUCCESS != func(pObj, pJobj)) { + if (pJobj == NULL) { + return terrno; + } + + int32_t rc = func(pObj, pJobj); + if (rc != TSDB_CODE_SUCCESS) { tjsonDelete(pJobj); - return TSDB_CODE_FAILED; + return rc; } return tjsonAddItemToArray(pJson, pJobj); } @@ -156,9 +160,21 @@ int32_t tjsonAddTArray(SJson* pJson, const char* pName, FToJson func, const SArr return TSDB_CODE_SUCCESS; } -char* tjsonToString(const SJson* pJson) { return cJSON_Print((cJSON*)pJson); } +char* tjsonToString(const SJson* pJson) { + char* p = cJSON_Print((cJSON*)pJson); + if (!p) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return p; +} -char* tjsonToUnformattedString(const SJson* pJson) { return cJSON_PrintUnformatted((cJSON*)pJson); } +char* tjsonToUnformattedString(const SJson* pJson) { + char* p = cJSON_PrintUnformatted((cJSON*)pJson); + if (!p) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + return p; +} SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName) { return cJSON_GetObjectItem(pJson, pName); } From b809761756e9420fa5d11de5de37821d64beb7ee Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Sat, 20 Jul 2024 15:16:47 +0800 Subject: [PATCH 35/37] test: increase timeout in all ci cases --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 083d96e975..904c8b1651 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -416,7 +416,7 @@ pipeline { echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}" } catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - timeout(time: 150, unit: 'MINUTES'){ + timeout(time: 200, unit: 'MINUTES'){ pre_test() script { sh ''' From 70191f6a5d85862a7e0620dd66349749963eb326 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Jul 2024 02:06:09 +0800 Subject: [PATCH 36/37] fix(stream): fix race condition when starting check downstream status and dropping task are executed concurrently. --- source/libs/stream/src/streamCheckStatus.c | 7 +++++++ source/libs/stream/src/streamMeta.c | 3 +++ 2 files changed, 10 insertions(+) diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 5e67f1766f..55209fc427 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -280,6 +280,13 @@ void streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { streamMutexLock(&pInfo->checkInfoLock); + // drop procedure already started, not start check downstream now + ETaskStatus s = streamTaskGetStatus(pTask).state; + if (s == TASK_STATUS__DROPPING) { + streamMutexUnlock(&pInfo->checkInfoLock); + return; + } + int32_t code = streamTaskStartCheckDownstream(pInfo, pTask->id.idStr); if (code != TSDB_CODE_SUCCESS) { streamMutexUnlock(&pInfo->checkInfoLock); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 42d2f86dac..6f3b7d8b32 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -742,7 +742,10 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t streamMetaRLock(pMeta); ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask) { + // to make sure check status will not start the check downstream status when we start to check timerActive count. + streamMutexLock(&pTask->taskCheckInfo.checkInfoLock); timerActive = (*ppTask)->status.timerActive; + streamMutexUnlock(&pTask->taskCheckInfo.checkInfoLock); } streamMetaRUnLock(pMeta); From 6cc009653437f8e6d296f76ac660b4f664e799cd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 21 Jul 2024 03:10:21 +0800 Subject: [PATCH 37/37] fix(stream): fix memory leak. --- source/dnode/vnode/src/tq/tqStreamTask.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index b4829d1dd8..0e5b1b6fb7 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -365,6 +365,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { code = streamTrySchedExec(pTask); if (code != TSDB_CODE_SUCCESS) { streamMetaReleaseTask(pStreamMeta, pTask); + taosArrayDestroy(pTaskList); return -1; } }