From d4b72613fd7d2a4dd5ee3597bacbb474e85aca34 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Wed, 1 Jun 2022 17:57:30 +0800 Subject: [PATCH 01/65] test: add debug cases --- tests/pytest/insert/insert_drop.py | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/pytest/insert/insert_drop.py diff --git a/tests/pytest/insert/insert_drop.py b/tests/pytest/insert/insert_drop.py new file mode 100644 index 0000000000..5ccaa5540b --- /dev/null +++ b/tests/pytest/insert/insert_drop.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +import threading + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def genMultiThreadSeq(self, sql_list): + tlist = list() + for insert_sql in sql_list: + t = threading.Thread(target=tdSql.execute, args=(insert_sql,)) + tlist.append(t) + return tlist + + def multiThreadRun(self, tlist): + for t in tlist: + t.start() + for t in tlist: + t.join() + + def run(self): + tdSql.prepare() + tdSql.execute('create database if not exists test;') + tdSql.execute('create table test.stb (ts timestamp, c11 int, c12 float ) TAGS(t11 int, t12 int );') + tdSql.execute('create table test.tb using test.stb TAGS (1, 1);') + sql_list = list() + for i in range(5): + sql = f'insert into test.tb values (now-{i}m, {i}, {i});' + sql_list.append(sql) + sql_list.append(f'drop database test;') + tlist = self.genMultiThreadSeq(sql_list) + self.multiThreadRun(tlist) + tdSql.query(f'show databases') + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From c45f6c4ba95f8c537d26e34c6e21e650f99c86a4 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Fri, 15 Jul 2022 19:51:53 +0800 Subject: [PATCH 02/65] test:test case for update_iotdata --- tests/system-test/1-insert/update_data.py | 193 ++++++++++++++++++++++ tests/system-test/fulltest.sh | 2 +- 2 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/1-insert/update_data.py diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py new file mode 100644 index 0000000000..8204df24af --- /dev/null +++ b/tests/system-test/1-insert/update_data.py @@ -0,0 +1,193 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db_test' + self.ntbname = 'ntb' + self.stbname = 'stb' + self.ts = 1537146000000 + self.binary_length = 20 + self.nchar_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.binary_length})', + 'col13': f'nchar({self.nchar_length})', + 'col_ts' : 'timestamp' + } + self.tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.bool = random.randint(0,100)%2 + self.float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.binary = tdCom.getLongName(self.binary_length) + self.tnchar = tdCom.getLongName(self.nchar_length) + def insert_base_data(self,col_type,tbname,value=None): + if value == None: + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.tinyint})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.smallint})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.int})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.bigint})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.untinyint})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.unsmallint})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.unint})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.unbigint})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.bool})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.float})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.double})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'insert into {tbname} values({self.ts},"{self.binary}")') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'insert into {tbname} values({self.ts},"{self.nchar}")') + + def update_and_check_data(self,tbname,col_name,col_type,value): + if 'binary' in col_type.lower() or 'nchar' in col_type.lower(): + tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{value})') + tdSql.query(f'select {col_name} from {tbname}') + if col_type.lower() == 'float' or col_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - value) / value <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{col_name} data check failure') + else: + tdSql.checkEqual(tdSql.queryResult[0][0],value) + + def update_check_ntb(self): + up_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + up_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + up_int = random.randint(constant.INT_MIN,constant.INT_MAX) + up_bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + up_untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + up_unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + up_unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + up_unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + up_bool = random.randint(0,100)%2 + up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + binary_length = random.randint(0,self.binary_length) + nchar_length = random.randint(0,self.binary_length) + up_binary = tdCom.getLongName(binary_length) + up_nchar = tdCom.getLongName(nchar_length) + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(f'use {self.dbname}') + + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_name,self.ntbname) + if col_type.lower() == 'tinyint': + self.update_and_check_data(self.ntbname,col_name,col_type,up_tinyint) + elif col_type.lower() == 'smallint': + self.update_and_check_data(self.ntbname,col_name,col_type,up_smallint) + elif col_type.lower() == 'int': + self.update_and_check_data(self.ntbname,col_name,col_type,up_int) + elif col_type.lower() == 'bigint': + self.update_and_check_data(self.ntbname,col_name,col_type,up_bigint) + elif col_type.lower() == 'tinyint unsigned': + self.update_and_check_data(self.ntbname,col_name,col_type,up_untinyint) + elif col_type.lower() == 'smallint unsigned': + self.update_and_check_data(self.ntbname,col_name,col_type,up_unsmallint) + elif col_type.lower() == 'int unsigned': + self.update_and_check_data(self.ntbname,col_name,col_type,up_unint) + elif col_type.lower() == 'bigint unsigned': + self.update_and_check_data(self.ntbname,col_name,col_type,up_unbigint) + elif col_type.lower() == 'bool': + self.update_and_check_data(self.ntbname,col_name,col_type,up_bool) + elif col_type.lower() == 'float': + self.update_and_check_data(self.ntbname,col_name,col_type,up_float) + elif col_type.lower() == 'double': + self.update_and_check_data(self.ntbname,col_name,col_type,up_double) + elif 'binary' in col_type.lower(): + self.update_and_check_data(self.ntbname,col_name,col_type,up_binary) + elif 'nchar' in col_type.lower(): + self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar) + tdSql.execute(f'drop table {self.ntbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_name,self.ntbname) + tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)') + tdSql.query(f'select {col_name} from {self.ntbname}') + tdSql.checkEqual(tdSql.queryResult[0][0],None) + tdSql.execute(f'drop table {self.ntbname}') + + + def update_check_error_ntb(self): + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(f'use {self.dbname}') + binary_length = self.binary_length+1 + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_name,self.ntbname) + if col_type.lower() in ['tinyint','smallint','int','bigint','tinyint unsigned','smallint unsigned','int unsigned','bigint unsigned']: + for error_value in [random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.binary_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() in ['float','double']: + for error_value in [tdCom.getLongName(self.binary_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): + for error_value in [tdCom.getLongName(binary_length)]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bool': + for error_value in [tdCom.getLongName(self.binary_length)]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + tdSql.execute(f'drop table {self.ntbname}') + + def run(self): + self.update_check_ntb() + self.update_check_error_ntb() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 8bafe3c966..6275e2878c 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -28,7 +28,7 @@ python3 ./test.py -f 1-insert/time_range_wise.py python3 ./test.py -f 1-insert/block_wise.py python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py - +python3 ./test.py -f 1-insert/update_data.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py From eeb57d8893173b6abb81ad5a47aaf590d2a344ac Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 10:07:02 +0800 Subject: [PATCH 03/65] update test case for update --- tests/system-test/1-insert/update_data.py | 61 ++++++++++++++++------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 8204df24af..40682a1910 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -26,8 +26,8 @@ class TDTestCase: self.ntbname = 'ntb' self.stbname = 'stb' self.ts = 1537146000000 - self.binary_length = 20 - self.nchar_length = 20 + self.str_length = 20 + self.column_dict = { 'col1': 'tinyint', 'col2': 'smallint', @@ -40,8 +40,8 @@ class TDTestCase: 'col9': 'float', 'col10': 'double', 'col11': 'bool', - 'col12': f'binary({self.binary_length})', - 'col13': f'nchar({self.nchar_length})', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', 'col_ts' : 'timestamp' } self.tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) @@ -55,8 +55,8 @@ class TDTestCase: self.bool = random.randint(0,100)%2 self.float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) self.double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) - self.binary = tdCom.getLongName(self.binary_length) - self.tnchar = tdCom.getLongName(self.nchar_length) + self.binary = tdCom.getLongName(self.str_length) + self.tnchar = tdCom.getLongName(self.str_length) def insert_base_data(self,col_type,tbname,value=None): if value == None: if col_type.lower() == 'tinyint': @@ -112,8 +112,8 @@ class TDTestCase: up_bool = random.randint(0,100)%2 up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) - binary_length = random.randint(0,self.binary_length) - nchar_length = random.randint(0,self.binary_length) + binary_length = random.randint(0,self.str_length) + nchar_length = random.randint(0,self.str_length) up_binary = tdCom.getLongName(binary_length) up_nchar = tdCom.getLongName(nchar_length) tdSql.execute(f'drop database if exists {self.dbname}') @@ -163,22 +163,49 @@ class TDTestCase: tdSql.execute(f'drop database if exists {self.dbname}') tdSql.execute(f'create database {self.dbname}') tdSql.execute(f'use {self.dbname}') - binary_length = self.binary_length+1 + str_length = self.str_length+1 for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') self.insert_base_data(col_name,self.ntbname) - if col_type.lower() in ['tinyint','smallint','int','bigint','tinyint unsigned','smallint unsigned','int unsigned','bigint unsigned']: - for error_value in [random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.binary_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() in ['float','double']: - for error_value in [tdCom.getLongName(self.binary_length),True,False]: + + + if col_type.lower() == 'double': + for error_value in [tdCom.getLongName(self.str_length),True,False,random.uniform(constant.DOUBLE_MIN*1.01,constant.DOUBLE_MAX*1.01)]: tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + #!bug TD-17453 + # elif col_type.lower() == 'float': + # for error_value in [tdCom.getLongName(self.str_length),True,False,random.uniform(constant.FLOAT_MIN*1.01,constant.FLOAT_MAX*1.01)]: + # tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): - for error_value in [tdCom.getLongName(binary_length)]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + for error_value in [tdCom.getLongName(str_length)]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},"{error_value}")') elif col_type.lower() == 'bool': - for error_value in [tdCom.getLongName(self.binary_length)]: + for error_value in [tdCom.getLongName(self.str_length)]: tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'tinyint': + for error_value in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'smallint': + for error_value in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'int': + for error_value in [constant.INT_MIN-1,constant.INT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bigint': + for error_value in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'tinyint unsigned': + for error_value in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'smallint unsigned': + for error_value in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'int unsigned': + for error_value in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bigint unsigned': + for error_value in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') tdSql.execute(f'drop table {self.ntbname}') def run(self): From a2090052b3a4a60d2d4de71da3a390856df393b0 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 10:21:52 +0800 Subject: [PATCH 04/65] update --- tests/system-test/1-insert/update_data.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 40682a1910..90b5b456cf 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -167,15 +167,12 @@ class TDTestCase: for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') self.insert_base_data(col_name,self.ntbname) - - if col_type.lower() == 'double': - for error_value in [tdCom.getLongName(self.str_length),True,False,random.uniform(constant.DOUBLE_MIN*1.01,constant.DOUBLE_MAX*1.01)]: + for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'float': + for error_value in [tdCom.getLongName(self.str_length),True,False,10*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - #!bug TD-17453 - # elif col_type.lower() == 'float': - # for error_value in [tdCom.getLongName(self.str_length),True,False,random.uniform(constant.FLOAT_MIN*1.01,constant.FLOAT_MAX*1.01)]: - # tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): for error_value in [tdCom.getLongName(str_length)]: tdSql.error(f'insert into {self.ntbname} values({self.ts},"{error_value}")') From 698baa52ce7eedb09fc1596df84b1a047a19468d Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 10:22:33 +0800 Subject: [PATCH 05/65] update --- tests/system-test/1-insert/update_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 90b5b456cf..12b61b4458 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -171,7 +171,7 @@ class TDTestCase: for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') elif col_type.lower() == 'float': - for error_value in [tdCom.getLongName(self.str_length),True,False,10*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: + for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): for error_value in [tdCom.getLongName(str_length)]: From fb6d4a566c832e23374546ebb039abdc7c07cb88 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 14:36:17 +0800 Subject: [PATCH 06/65] update --- tests/system-test/1-insert/update_data.py | 46 +++++++++++++---------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 12b61b4458..35b082f726 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -85,12 +85,7 @@ class TDTestCase: tdSql.execute(f'insert into {tbname} values({self.ts},"{self.binary}")') elif 'nchar' in col_type.lower(): tdSql.execute(f'insert into {tbname} values({self.ts},"{self.nchar}")') - - def update_and_check_data(self,tbname,col_name,col_type,value): - if 'binary' in col_type.lower() or 'nchar' in col_type.lower(): - tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")') - else: - tdSql.execute(f'insert into {tbname} values({self.ts},{value})') + def data_check(self,tbname,col_name,col_type,value): tdSql.query(f'select {col_name} from {tbname}') if col_type.lower() == 'float' or col_type.lower() == 'double': if abs(tdSql.queryResult[0][0] - value) / value <= 0.0001: @@ -99,7 +94,15 @@ class TDTestCase: tdLog.exit(f'{col_name} data check failure') else: tdSql.checkEqual(tdSql.queryResult[0][0],value) - + pass + def update_and_check_data(self,tbname,col_name,col_type,value,dbname): + if 'binary' in col_type.lower() or 'nchar' in col_type.lower(): + tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{value})') + self.data_check(tbname,col_name,col_type,value) + tdSql.execute(f'flush database {dbname}') + self.data_check(tbname,col_name,col_type,value) def update_check_ntb(self): up_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) up_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) @@ -124,31 +127,31 @@ class TDTestCase: tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') self.insert_base_data(col_name,self.ntbname) if col_type.lower() == 'tinyint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_tinyint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_tinyint,self.dbname) elif col_type.lower() == 'smallint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_smallint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_smallint,self.dbname) elif col_type.lower() == 'int': - self.update_and_check_data(self.ntbname,col_name,col_type,up_int) + self.update_and_check_data(self.ntbname,col_name,col_type,up_int,self.dbname) elif col_type.lower() == 'bigint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_bigint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_bigint,self.dbname) elif col_type.lower() == 'tinyint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_untinyint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_untinyint,self.dbname) elif col_type.lower() == 'smallint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unsmallint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_unsmallint,self.dbname) elif col_type.lower() == 'int unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_unint,self.dbname) elif col_type.lower() == 'bigint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unbigint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_unbigint,self.dbname) elif col_type.lower() == 'bool': - self.update_and_check_data(self.ntbname,col_name,col_type,up_bool) + self.update_and_check_data(self.ntbname,col_name,col_type,up_bool,self.dbname) elif col_type.lower() == 'float': - self.update_and_check_data(self.ntbname,col_name,col_type,up_float) + self.update_and_check_data(self.ntbname,col_name,col_type,up_float,self.dbname) elif col_type.lower() == 'double': - self.update_and_check_data(self.ntbname,col_name,col_type,up_double) + self.update_and_check_data(self.ntbname,col_name,col_type,up_double,self.dbname) elif 'binary' in col_type.lower(): - self.update_and_check_data(self.ntbname,col_name,col_type,up_binary) + self.update_and_check_data(self.ntbname,col_name,col_type,up_binary,self.dbname) elif 'nchar' in col_type.lower(): - self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar) + self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname) tdSql.execute(f'drop table {self.ntbname}') for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') @@ -156,6 +159,9 @@ class TDTestCase: tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)') tdSql.query(f'select {col_name} from {self.ntbname}') tdSql.checkEqual(tdSql.queryResult[0][0],None) + tdSql.execute(f'flush database {self.dbname}') + tdSql.query(f'select {col_name} from {self.ntbname}') + tdSql.checkEqual(tdSql.queryResult[0][0],None) tdSql.execute(f'drop table {self.ntbname}') From 7615341dffedf532efe1c124a953b1eaec22694d Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 16:08:46 +0800 Subject: [PATCH 07/65] update --- tests/system-test/1-insert/update_data.py | 218 +++++++++++++--------- 1 file changed, 131 insertions(+), 87 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 35b082f726..f6ede9c2f9 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -11,6 +11,7 @@ # -*- coding: utf-8 -*- +from optparse import Values import random import string from util import constant @@ -18,16 +19,18 @@ from util.log import * from util.cases import * from util.sql import * from util.common import * +from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) + self.setsql = TDSetSql() self.dbname = 'db_test' self.ntbname = 'ntb' self.stbname = 'stb' + self.ctbname = 'ctb' self.ts = 1537146000000 self.str_length = 20 - self.column_dict = { 'col1': 'tinyint', 'col2': 'smallint', @@ -44,19 +47,19 @@ class TDTestCase: 'col13': f'nchar({self.str_length})', 'col_ts' : 'timestamp' } - self.tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) - self.smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) - self.int = random.randint(constant.INT_MIN,constant.INT_MAX) - self.bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) - self.untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) - self.unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) - self.unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX) - self.unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) - self.bool = random.randint(0,100)%2 - self.float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) - self.double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) - self.binary = tdCom.getLongName(self.str_length) - self.tnchar = tdCom.getLongName(self.str_length) + self.tinyint = None + self.smallint = None + self.int = None + self.bigint = None + self.untinyint = None + self.unsmallint = None + self.unint = None + self.unbigint = None + self.bool = None + self.float = None + self.double = None + self.binary = None + self.tnchar = None def insert_base_data(self,col_type,tbname,value=None): if value == None: if col_type.lower() == 'tinyint': @@ -94,7 +97,6 @@ class TDTestCase: tdLog.exit(f'{col_name} data check failure') else: tdSql.checkEqual(tdSql.queryResult[0][0],value) - pass def update_and_check_data(self,tbname,col_name,col_type,value,dbname): if 'binary' in col_type.lower() or 'nchar' in col_type.lower(): tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")') @@ -102,8 +104,57 @@ class TDTestCase: tdSql.execute(f'insert into {tbname} values({self.ts},{value})') self.data_check(tbname,col_name,col_type,value) tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') self.data_check(tbname,col_name,col_type,value) - def update_check_ntb(self): + def error_check(self,tbname,column_dict,tb_type=None,stbname=None): + str_length = self.str_length+1 + for col_name,col_type in column_dict.items(): + if tb_type == 'ntb': + tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})') + elif tb_type == 'ctb': + tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') + tdSql.execute(f'create table {tbname} using {stbname} tags(1)') + self.insert_base_data(col_name,tbname) + if col_type.lower() == 'double': + for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'float': + for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): + for error_value in [tdCom.getLongName(str_length)]: + tdSql.error(f'insert into {tbname} values({self.ts},"{error_value}")') + elif col_type.lower() == 'bool': + for error_value in [tdCom.getLongName(self.str_length)]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'tinyint': + for error_value in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'smallint': + for error_value in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'int': + for error_value in [constant.INT_MIN-1,constant.INT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bigint': + for error_value in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'tinyint unsigned': + for error_value in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'smallint unsigned': + for error_value in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'int unsigned': + for error_value in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bigint unsigned': + for error_value in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + tdSql.execute(f'drop table {tbname}') + if tb_type == 'ctb': + tdSql.execute(f'drop table {stbname}') + def update_data_check(self,tbname,column_dict,dbname,tb_type=None,stbname=None): up_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) up_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) up_int = random.randint(constant.INT_MIN,constant.INT_MAX) @@ -119,102 +170,95 @@ class TDTestCase: nchar_length = random.randint(0,self.str_length) up_binary = tdCom.getLongName(binary_length) up_nchar = tdCom.getLongName(nchar_length) - tdSql.execute(f'drop database if exists {self.dbname}') - tdSql.execute(f'create database {self.dbname}') - tdSql.execute(f'use {self.dbname}') - - for col_name,col_type in self.column_dict.items(): - tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') - self.insert_base_data(col_name,self.ntbname) + + for col_name,col_type in column_dict.items(): + if tb_type == 'ntb': + tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})') + elif tb_type == 'ctb': + tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') + tdSql.execute(f'create table {tbname} using {stbname} tags(1)') + self.insert_base_data(col_name,tbname) if col_type.lower() == 'tinyint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_tinyint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_tinyint,dbname) elif col_type.lower() == 'smallint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_smallint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_smallint,dbname) elif col_type.lower() == 'int': - self.update_and_check_data(self.ntbname,col_name,col_type,up_int,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_int,dbname) elif col_type.lower() == 'bigint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_bigint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_bigint,dbname) elif col_type.lower() == 'tinyint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_untinyint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_untinyint,dbname) elif col_type.lower() == 'smallint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unsmallint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_unsmallint,dbname) elif col_type.lower() == 'int unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_unint,dbname) elif col_type.lower() == 'bigint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unbigint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_unbigint,dbname) elif col_type.lower() == 'bool': - self.update_and_check_data(self.ntbname,col_name,col_type,up_bool,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_bool,dbname) elif col_type.lower() == 'float': - self.update_and_check_data(self.ntbname,col_name,col_type,up_float,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_float,dbname) elif col_type.lower() == 'double': - self.update_and_check_data(self.ntbname,col_name,col_type,up_double,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_double,dbname) elif 'binary' in col_type.lower(): - self.update_and_check_data(self.ntbname,col_name,col_type,up_binary,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname) elif 'nchar' in col_type.lower(): - self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname) - tdSql.execute(f'drop table {self.ntbname}') - for col_name,col_type in self.column_dict.items(): - tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') - self.insert_base_data(col_name,self.ntbname) - tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)') - tdSql.query(f'select {col_name} from {self.ntbname}') + self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname) + tdSql.execute(f'insert into {tbname} values({self.ts},null)') + tdSql.query(f'select {col_name} from {tbname}') tdSql.checkEqual(tdSql.queryResult[0][0],None) tdSql.execute(f'flush database {self.dbname}') - tdSql.query(f'select {col_name} from {self.ntbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') tdSql.checkEqual(tdSql.queryResult[0][0],None) - tdSql.execute(f'drop table {self.ntbname}') - - - def update_check_error_ntb(self): + tdSql.execute(f'drop table {tbname}') + if tb_type == 'ctb': + tdSql.execute(f'drop table {stbname}') + def update_check(self): tdSql.execute(f'drop database if exists {self.dbname}') tdSql.execute(f'create database {self.dbname}') tdSql.execute(f'use {self.dbname}') - str_length = self.str_length+1 + self.update_data_check(self.ntbname,self.column_dict,self.dbname,'ntb') for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') self.insert_base_data(col_name,self.ntbname) - if col_type.lower() == 'double': - for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'float': - for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): - for error_value in [tdCom.getLongName(str_length)]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},"{error_value}")') - elif col_type.lower() == 'bool': - for error_value in [tdCom.getLongName(self.str_length)]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'tinyint': - for error_value in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'smallint': - for error_value in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'int': - for error_value in [constant.INT_MIN-1,constant.INT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'bigint': - for error_value in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'tinyint unsigned': - for error_value in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'smallint unsigned': - for error_value in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'int unsigned': - for error_value in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'bigint unsigned': - for error_value in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + if 'binary' in col_type.lower(): + up_binary = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.ntbname} modify column {col_name} binary({self.str_length+1})') + self.update_and_check_data(self.ntbname,col_name,col_type,up_binary,self.dbname) + elif 'nchar' in col_type.lower(): + up_nchar = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.ntbname} modify column {col_name} nchar({self.str_length+1})') + self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname) tdSql.execute(f'drop table {self.ntbname}') + + self.update_data_check(self.ctbname,self.column_dict,self.dbname,'ctb',self.stbname) + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') + tdSql.execute(f'create table {self.ctbname} using {self.stbname} tags(1)') + self.insert_base_data(col_name,self.ctbname) + if 'binary' in col_type.lower(): + up_binary = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.stbname} modify column {col_name} binary({self.str_length+1})') + self.update_and_check_data(self.ctbname,col_name,col_type,up_binary,self.dbname) + elif 'nchar' in col_type.lower(): + up_nchar = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.stbname} modify column {col_name} nchar({self.str_length+1})') + self.update_and_check_data(self.ctbname,col_name,col_type,up_nchar,self.dbname) + tdSql.execute(f'drop table {self.stbname}') + + + def update_check_error(self): + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(f'use {self.dbname}') + self.error_check(self.ntbname,self.column_dict,'ntb') + self.error_check(self.ctbname,self.column_dict,'ctb',self.stbname) def run(self): - self.update_check_ntb() - self.update_check_error_ntb() - + self.update_check() + self.update_check_error() + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From bc3f7303ef6968280c081856d9f3e9fbdae2df88 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 16:16:18 +0800 Subject: [PATCH 08/65] update --- tests/system-test/1-insert/update_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index f6ede9c2f9..387fc18774 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -95,6 +95,8 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) else: tdLog.exit(f'{col_name} data check failure') + elif col_type.lower() == 'timestamp': + tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f"))) else: tdSql.checkEqual(tdSql.queryResult[0][0],value) def update_and_check_data(self,tbname,col_name,col_type,value,dbname): @@ -204,6 +206,8 @@ class TDTestCase: self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname) elif 'nchar' in col_type.lower(): self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname) + elif col_type.lower() == 'timestamp': + self.update_and_check_data(tbname,col_name,col_type,self.ts+1,dbname) tdSql.execute(f'insert into {tbname} values({self.ts},null)') tdSql.query(f'select {col_name} from {tbname}') tdSql.checkEqual(tdSql.queryResult[0][0],None) From 17610f98a02f3b70200d3744e60293263ac759be Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 18 Jul 2022 08:43:30 +0800 Subject: [PATCH 09/65] update --- tests/system-test/1-insert/update_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 387fc18774..d2151976c8 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -108,6 +108,8 @@ class TDTestCase: tdSql.execute(f'flush database {dbname}') tdSql.execute('reset query cache') self.data_check(tbname,col_name,col_type,value) + for func in ['first','last']: + tdSql.execute(f'select {func}({col_name}) from {tbname}') def error_check(self,tbname,column_dict,tb_type=None,stbname=None): str_length = self.str_length+1 for col_name,col_type in column_dict.items(): From da5e5c52e0c8a709cc8b08430eaf7ab3b68c2778 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 18 Jul 2022 08:53:42 +0800 Subject: [PATCH 10/65] add delete_data.py --- tests/system-test/1-insert/delete_data.py | 88 +++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/system-test/1-insert/delete_data.py diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py new file mode 100644 index 0000000000..6c0c2d24b7 --- /dev/null +++ b/tests/system-test/1-insert/delete_data.py @@ -0,0 +1,88 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db_test' + self.setsql = TDSetSql() + self.ntbname = 'ntb' + self.rowNum = 10 + self.tbnum = 20 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + # first column must be timestamp + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + + } + self.value = [ + f'1' + ] + + self.param_list = [1,100] + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) + for i in range(row_num): + insert_list = [] + + def delete_all_data(self,tbname): + tdSql.execute(f'delete from {tbname}') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(0) + + def delete_one_row(self,tbname,column_dict): + for column_name,column_type in column_dict.items(): + if column_type.lower() == 'timestamp': + + tdSql.execute(f'delete from {tbname} where {column_name}={self.ts}') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(self.rowNum-1) + tdSql.query(f'select * from {tbname} where {column_name}={self.ts}') + tdSql.checkRows(0) + tdSql.execute(f'insert into {tbname} values({self.ts},{self.value_list[0]}') + tdSql.query(f'select * from {tbname} where {column_name}={self.ts}') + tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f"))) + + def delete_data_ntb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + for i in range(self.rowNum): + tdSql.execute(f'insert into {self.ntbname} values({self.ts+i},{self.value_list[0]})') + + + # self.delete_all_data(self.ntbname) + + def run(self): + self.delete_data_ntb() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 977d6d5242bc762075d5180b4ad48661c540b8f8 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 18 Jul 2022 11:34:24 +0800 Subject: [PATCH 11/65] update test case --- tests/system-test/1-insert/update_data.py | 41 ++++------------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index d2151976c8..c2b8e7ab08 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -11,7 +11,6 @@ # -*- coding: utf-8 -*- -from optparse import Values import random import string from util import constant @@ -23,7 +22,7 @@ from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(),logSql) self.setsql = TDSetSql() self.dbname = 'db_test' self.ntbname = 'ntb' @@ -60,34 +59,8 @@ class TDTestCase: self.double = None self.binary = None self.tnchar = None - def insert_base_data(self,col_type,tbname,value=None): - if value == None: - if col_type.lower() == 'tinyint': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.tinyint})') - elif col_type.lower() == 'smallint': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.smallint})') - elif col_type.lower() == 'int': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.int})') - elif col_type.lower() == 'bigint': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.bigint})') - elif col_type.lower() == 'tinyint unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.untinyint})') - elif col_type.lower() == 'smallint unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.unsmallint})') - elif col_type.lower() == 'int unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.unint})') - elif col_type.lower() == 'bigint unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.unbigint})') - elif col_type.lower() == 'bool': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.bool})') - elif col_type.lower() == 'float': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.float})') - elif col_type.lower() == 'double': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.double})') - elif 'binary' in col_type.lower(): - tdSql.execute(f'insert into {tbname} values({self.ts},"{self.binary}")') - elif 'nchar' in col_type.lower(): - tdSql.execute(f'insert into {tbname} values({self.ts},"{self.nchar}")') + + def data_check(self,tbname,col_name,col_type,value): tdSql.query(f'select {col_name} from {tbname}') if col_type.lower() == 'float' or col_type.lower() == 'double': @@ -118,7 +91,7 @@ class TDTestCase: elif tb_type == 'ctb': tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') tdSql.execute(f'create table {tbname} using {stbname} tags(1)') - self.insert_base_data(col_name,tbname) + tdSql.execute(f'insert into {tbname} values({self.ts},null)') if col_type.lower() == 'double': for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') @@ -181,7 +154,7 @@ class TDTestCase: elif tb_type == 'ctb': tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') tdSql.execute(f'create table {tbname} using {stbname} tags(1)') - self.insert_base_data(col_name,tbname) + tdSql.execute(f'insert into {tbname} values({self.ts},null)') if col_type.lower() == 'tinyint': self.update_and_check_data(tbname,col_name,col_type,up_tinyint,dbname) elif col_type.lower() == 'smallint': @@ -227,7 +200,7 @@ class TDTestCase: self.update_data_check(self.ntbname,self.column_dict,self.dbname,'ntb') for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') - self.insert_base_data(col_name,self.ntbname) + tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)') if 'binary' in col_type.lower(): up_binary = tdCom.getLongName(self.str_length+1) tdSql.execute(f'alter table {self.ntbname} modify column {col_name} binary({self.str_length+1})') @@ -242,7 +215,7 @@ class TDTestCase: for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') tdSql.execute(f'create table {self.ctbname} using {self.stbname} tags(1)') - self.insert_base_data(col_name,self.ctbname) + tdSql.execute(f'insert into {self.ctbname} values({self.ts},null)') if 'binary' in col_type.lower(): up_binary = tdCom.getLongName(self.str_length+1) tdSql.execute(f'alter table {self.stbname} modify column {col_name} binary({self.str_length+1})') From 03e9b15237994f7815b472c2371966b70f557db5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 18 Jul 2022 13:32:17 +0800 Subject: [PATCH 12/65] fix: fix taosc memory leak --- include/libs/qcom/query.h | 5 +++++ source/client/src/clientHb.c | 4 +--- source/client/src/clientImpl.c | 8 -------- source/client/src/clientMsgHandler.c | 6 ++++++ source/client/src/tmq.c | 2 ++ source/libs/catalog/inc/catalogInt.h | 8 ++++++++ source/libs/catalog/src/ctgRemote.c | 8 ++++---- source/libs/executor/src/dataInserter.c | 1 + source/libs/executor/src/executorimpl.c | 7 +------ source/libs/nodes/src/nodesUtilFuncs.c | 10 +++++++++- source/libs/qcom/src/queryUtil.c | 10 ++++++++++ source/libs/scheduler/src/schRemote.c | 10 ++++++---- source/libs/scheduler/src/schUtil.c | 16 +++++++++------- 13 files changed, 62 insertions(+), 33 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index a93cf1f9b8..58739b4af7 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -162,9 +162,12 @@ typedef struct SRequestConnInfo { SEpSet mgmtEps; } SRequestConnInfo; +typedef void (*__freeFunc)(void *param); + typedef struct SMsgSendInfo { __async_send_cb_fn_t fp; // async callback function STargetInfo target; // for update epset + __freeFunc paramFreeFp; void* param; uint64_t requestId; uint64_t requestObjRefId; @@ -188,6 +191,8 @@ int32_t cleanupTaskQueue(); */ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); +void destroySendMsgInfo(SMsgSendInfo* pMsgBody); + int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void* ctx); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b52df105b1..6a583842d1 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -286,13 +286,10 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { if (pInst == NULL || NULL == *pInst) { taosThreadMutexUnlock(&appInfo.mutex); tscError("cluster not exist, key:%s", key); - taosMemoryFreeClear(param); tFreeClientHbBatchRsp(&pRsp); return -1; } - taosMemoryFreeClear(param); - if (code != 0) { (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1); tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes, (*pInst)->totalDnodes); @@ -716,6 +713,7 @@ static void *hbThreadFunc(void *param) { pInfo->msgInfo.len = tlen; pInfo->msgType = TDMT_MND_HEARTBEAT; pInfo->param = strdup(pAppHbMgr->key); + pInfo->paramFreeFp = taosMemoryFree; pInfo->requestId = generateRequestId(); pInfo->requestObjRefId = 0; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index e8eef0d287..9ad3be4d92 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -29,7 +29,6 @@ static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); -static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { @@ -1215,13 +1214,6 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { return pMsgSendInfo; } -static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { - assert(pMsgBody != NULL); - taosMemoryFreeClear(pMsgBody->target.dbFName); - taosMemoryFreeClear(pMsgBody->msgInfo.pData); - taosMemoryFreeClear(pMsgBody); -} - void updateTargetEpSet(SMsgSendInfo* pSendInfo, STscObj* pTscObj, SRpcMsg* pMsg, SEpSet* pEpSet) { if (NULL == pEpSet) { return; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 520a566e2b..db33532aef 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -255,6 +255,8 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); } + taosMemoryFree(pMsg->pData); + if (pRequest->body.queryFp != NULL) { pRequest->body.queryFp(pRequest->body.param, pRequest, code); } else { @@ -278,6 +280,8 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { pRequest->body.resInfo.execRes.res = alterRsp.pMeta; } + taosMemoryFree(pMsg->pData); + if (pRequest->body.queryFp != NULL) { SExecResult* pRes = &pRequest->body.resInfo.execRes; @@ -387,6 +391,8 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { tFreeSShowVariablesRsp(&rsp); } + taosMemoryFree(pMsg->pData); + if (pRequest->body.queryFp != NULL) { pRequest->body.queryFp(pRequest->body.param, pRequest, code); } else { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ed0ec516b2..fb835d3878 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -506,6 +506,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; pMsgSendInfo->param = pParam; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = tmqCommitCb2; pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; // send msg @@ -1516,6 +1517,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; sendInfo->param = pParam; + sendInfo->paramFreeFp = taosMemoryFree; sendInfo->fp = tmqAskEpCb; sendInfo->msgType = TDMT_MND_MQ_ASK_EP; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index fb9f588bae..9003de97d7 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -532,6 +532,14 @@ typedef struct SCtgOperation { } \ } while (0) +#define CTG_API_JENTER() do { \ + CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \ + CTG_LOCK(CTG_READ, &gCtgMgmt.lock); \ + if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \ + CTG_ERR_JRET(TSDB_CODE_CTG_OUT_OF_SERVICE); \ + } \ +} while (0) + #define CTG_API_LEAVE_NOLOCK(c) do { \ int32_t __code = c; \ diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 8e0a5b7de3..1e375471f9 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -244,10 +244,11 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t ctgHandleMsgCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { SCtgTaskCallbackParam* cbParam = (SCtgTaskCallbackParam*)param; int32_t code = 0; + SCtgJob* pJob = NULL; - CTG_API_ENTER(); + CTG_API_JENTER(); - SCtgJob* pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId); + pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId); if (NULL == pJob) { qDebug("ctg job refId 0x%" PRIx64 " already dropped", cbParam->refId); goto _return; @@ -266,8 +267,6 @@ _return: if (pJob) { taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); } - - taosMemoryFree(param); CTG_API_LEAVE(code); } @@ -293,6 +292,7 @@ int32_t ctgMakeMsgSendInfo(SCtgTask* pTask, int32_t msgType, SMsgSendInfo **pMsg param->taskId = pTask->taskId; msgSendInfo->param = param; + msgSendInfo->paramFreeFp = taosMemoryFree; msgSendInfo->fp = ctgHandleMsgCallback; *pMsgSendInfo = msgSendInfo; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index a575e355f1..440a8e014a 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -110,6 +110,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs pParam->pInserter = pInserter; pMsgSendInfo->param = pParam; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; pMsgSendInfo->msgInfo.len = ntohl(pMsg->length); pMsgSendInfo->msgType = TDMT_VND_SUBMIT; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c3d9f66479..5a1fb770e5 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1990,12 +1990,6 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { return TSDB_CODE_SUCCESS; } -static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { - assert(pMsgBody != NULL); - taosMemoryFreeClear(pMsgBody->msgInfo.pData); - taosMemoryFreeClear(pMsgBody); -} - void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; assert(pMsg->info.ahandle != NULL); @@ -2055,6 +2049,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf pWrapper->sourceIndex = sourceIndex; pMsgSendInfo->param = pWrapper; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); pMsgSendInfo->msgType = pSource->fetchMsgType; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1dc3db033b..82b6e7b236 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1923,15 +1923,18 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** return partitionLogicCond(pCondition, pPrimaryKeyCond, pTagIndexCond, pTagCond, pOtherCond); } + bool needOutput = false; switch (classifyCondition(*pCondition)) { case COND_TYPE_PRIMARY_KEY: if (NULL != pPrimaryKeyCond) { *pPrimaryKeyCond = *pCondition; + needOutput = true; } break; case COND_TYPE_TAG_INDEX: if (NULL != pTagIndexCond) { *pTagIndexCond = *pCondition; + needOutput = true; } if (NULL != pTagCond) { SNode* pTempCond = *pCondition; @@ -1942,21 +1945,26 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** } } *pTagCond = pTempCond; + needOutput = true; } break; case COND_TYPE_TAG: if (NULL != pTagCond) { *pTagCond = *pCondition; + needOutput = true; } break; case COND_TYPE_NORMAL: default: if (NULL != pOtherCond) { *pOtherCond = *pCondition; + needOutput = true; } break; } - *pCondition = NULL; + if (needOutput) { + *pCondition = NULL; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index eeb44c4f82..6b1476fe46 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -138,6 +138,16 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) return 0; } +void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { + assert(pMsgBody != NULL); + taosMemoryFreeClear(pMsgBody->target.dbFName); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + if (pMsgBody->paramFreeFp) { + (*pMsgBody->paramFreeFp)(pMsgBody->param); + } + taosMemoryFreeClear(pMsgBody); +} + int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void* rpcCtx) { char* pMsg = rpcMallocCont(pInfo->msgInfo.len); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 6983bbf013..c8eafd7382 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -386,7 +386,6 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { schProcessOnCbEnd(pJob, pTask, code); taosMemoryFreeClear(pMsg->pData); - taosMemoryFreeClear(param); qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode)); @@ -398,7 +397,6 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); - taosMemoryFreeClear(param); return TSDB_CODE_SUCCESS; } @@ -447,8 +445,8 @@ int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) { SCH_ERR_JRET(schProcessOnTaskStatusRsp(&rsp.epId, rsp.taskStatus)); _return: + tFreeSSchedulerHbRsp(&rsp); - taosMemoryFree(param); taosMemoryFree(pMsg->pData); SCH_RET(code); } @@ -514,7 +512,9 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + msgSendInfo->paramFreeFp = taosMemoryFree; SCH_ERR_JRET(schMakeCallbackParam(pJob, pTask, msgType, isHb, trans, &msgSendInfo->param)); + SCH_ERR_JRET(schGetCallbackFp(msgType, &msgSendInfo->fp)); if (pJob) { @@ -535,7 +535,7 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 _return: - schFreeSMsgSendInfo(msgSendInfo); + destroySendMsgInfo(msgSendInfo); SCH_RET(code); } @@ -676,6 +676,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { param->pTrans = pJob->conn.pTrans; pMsgSendInfo->param = param; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = fp; SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo}; @@ -795,6 +796,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { pDst->param = NULL; SCH_ERR_JRET(schCloneCallbackParam(pSrc->param, (SSchCallbackParamHeader **)&pDst->param)); + pDst->paramFreeFp = taosMemoryFree; *dst = pDst; diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index f848dfa210..6f0aca9a8d 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -50,6 +50,12 @@ char* schGetOpStr(SCH_OP_TYPE type) { } } +void schFreeHbTrans(SSchHbTrans *pTrans) { + rpcReleaseHandle(pTrans->trans.pHandle, TAOS_CONN_CLIENT); + + schFreeRpcCtx(&pTrans->rpcCtx); +} + void schCleanClusterHb(void* pTrans) { SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); @@ -57,7 +63,7 @@ void schCleanClusterHb(void* pTrans) { while (hb) { if (hb->trans.pTrans == pTrans) { SQueryNodeEpId* pEpId = taosHashGetKey(hb, NULL); - rpcReleaseHandle(hb->trans.pHandle, TAOS_CONN_CLIENT); + schFreeHbTrans(hb); taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); } @@ -68,8 +74,6 @@ void schCleanClusterHb(void* pTrans) { } int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId) { - return TSDB_CODE_SUCCESS; // TODO ENABLE IT WHEN RPC IS READY - int32_t code = 0; SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); @@ -82,7 +86,7 @@ int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *ep int64_t taskNum = atomic_load_64(&hb->taskNum); if (taskNum <= 0) { - rpcReleaseHandle(hb->trans.pHandle, TAOS_CONN_CLIENT); + schFreeHbTrans(hb); taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); } SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); @@ -265,9 +269,7 @@ void schFreeRpcCtxVal(const void *arg) { } SMsgSendInfo *pMsgSendInfo = (SMsgSendInfo *)arg; - taosMemoryFreeClear(pMsgSendInfo->param); - taosMemoryFreeClear(pMsgSendInfo->msgInfo.pData); - taosMemoryFreeClear(pMsgSendInfo); + destroySendMsgInfo(pMsgSendInfo); } void schFreeRpcCtx(SRpcCtx *pCtx) { From 2dfa77f2ced6bb3232b3579fd8df54e570d65229 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 18 Jul 2022 13:57:13 +0800 Subject: [PATCH 13/65] update --- tests/system-test/1-insert/update_data.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index c2b8e7ab08..27e1559d7e 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -46,20 +46,6 @@ class TDTestCase: 'col13': f'nchar({self.str_length})', 'col_ts' : 'timestamp' } - self.tinyint = None - self.smallint = None - self.int = None - self.bigint = None - self.untinyint = None - self.unsmallint = None - self.unint = None - self.unbigint = None - self.bool = None - self.float = None - self.double = None - self.binary = None - self.tnchar = None - def data_check(self,tbname,col_name,col_type,value): tdSql.query(f'select {col_name} from {tbname}') @@ -147,7 +133,6 @@ class TDTestCase: nchar_length = random.randint(0,self.str_length) up_binary = tdCom.getLongName(binary_length) up_nchar = tdCom.getLongName(nchar_length) - for col_name,col_type in column_dict.items(): if tb_type == 'ntb': tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})') @@ -210,7 +195,6 @@ class TDTestCase: tdSql.execute(f'alter table {self.ntbname} modify column {col_name} nchar({self.str_length+1})') self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname) tdSql.execute(f'drop table {self.ntbname}') - self.update_data_check(self.ctbname,self.column_dict,self.dbname,'ctb',self.stbname) for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') @@ -226,7 +210,6 @@ class TDTestCase: self.update_and_check_data(self.ctbname,col_name,col_type,up_nchar,self.dbname) tdSql.execute(f'drop table {self.stbname}') - def update_check_error(self): tdSql.execute(f'drop database if exists {self.dbname}') tdSql.execute(f'create database {self.dbname}') From b83968697d65b72c2885ba94ef47d6baf52971b0 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 14:02:27 +0800 Subject: [PATCH 14/65] test:update case for cachemodel --- .../0-others/{cachelast.py => cachemodel.py} | 86 +++++++++++-------- tests/system-test/fulltest.sh | 2 +- 2 files changed, 50 insertions(+), 38 deletions(-) rename tests/system-test/0-others/{cachelast.py => cachemodel.py} (62%) diff --git a/tests/system-test/0-others/cachelast.py b/tests/system-test/0-others/cachemodel.py similarity index 62% rename from tests/system-test/0-others/cachelast.py rename to tests/system-test/0-others/cachemodel.py index 2df6b8c9cc..09538e6678 100644 --- a/tests/system-test/0-others/cachelast.py +++ b/tests/system-test/0-others/cachemodel.py @@ -56,6 +56,16 @@ class TDTestCase: } return numbers.get(value, 'other') + def getCacheModelNum(self,str): + numbers = { + "none" : 0, + "last_row" : 1, + "last_value" : 2, + "both" : 3 + + } + return numbers.get(str, 'other') + def prepare_datas(self): for i in range(4): str = self.getCacheModelStr(i) @@ -69,7 +79,7 @@ class TDTestCase: tdSql.execute(" insert into tb1 values(now , %d, %f)" %(k,k*10) ) tdSql.execute(" insert into tb2 values(now , %d, %f)" %(k,k*10) ) - def check_cache_last_sets(self): + def check_cachemodel_sets(self): # check cache_last value for database @@ -84,52 +94,54 @@ class TDTestCase: # print(cache_last_value) if dbname in ["information_schema" , "performance_schema"]: continue - cache_lasts[dbname]=cache_last_value + cache_lasts[dbname]=self.getCacheModelNum(cache_last_value) # cache_last_set value for k , v in cache_lasts.items(): - if k=="testdb_"+str(v): - tdLog.info(" database %s cache_last value check pass, value is %s "%(k,v) ) + if k=="testdb_"+str(self.getCacheModelStr(v)): + tdLog.info(" database %s cache_last value check pass, value is %s "%(k,self.getCacheModelStr(v)) ) else: - tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,v) ) + tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,self.getCacheModelStr(v)) ) # # check storage layer implementation - # buildPath = self.getBuildPath() - # if (buildPath == ""): - # tdLog.exit("taosd not found!") - # else: - # tdLog.info("taosd found in %s" % buildPath) - # dataPath = buildPath + "/../sim/dnode1/data" - # abs_vnodePath = os.path.abspath(dataPath)+"/vnode/" - # tdLog.info("abs_vnodePath: %s" % abs_vnodePath) + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + dataPath = buildPath + "/../sim/dnode1/data" + abs_vnodePath = os.path.abspath(dataPath)+"/vnode/" + tdLog.info("abs_vnodePath: %s" % abs_vnodePath) - # tdSql.query(" show dnodes ") - # dnode_id = tdSql.queryResult[0][0] + tdSql.query(" show dnodes ") + dnode_id = tdSql.queryResult[0][0] - # for dbname in cache_lasts.keys(): - # print(dbname) - # tdSql.execute(" use %s" % dbname) - # tdSql.query(" show vgroups ") - # vgroups_infos = tdSql.queryResult - # for vgroup_info in vgroups_infos: - # vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json" - # vnode_info_of_db = f"cat {vnode_json}" - # vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8") - # infoDict = json.loads(vnode_info) - # vnode_json_of_dbname = f"{dnode_id}."+ dbname - # config = infoDict["config"] - # if infoDict["config"]["dbname"] == vnode_json_of_dbname: - # if "cachelast" in infoDict["config"]: - # if int(infoDict["config"]["cachelast"]) != cache_lasts[dbname]: - # tdLog.exit("cachelast value is error in vnode.json of vnode%d "%(vgroup_info[0])) - # else: - # tdLog.exit("cachelast not found in vnode.json of vnode%d "%(vgroup_info[0])) + for dbname in cache_lasts.keys(): + # print(dbname) + tdSql.execute(" use %s" % dbname) + tdSql.query(" show vgroups ") + vgroups_infos = tdSql.queryResult + for vgroup_info in vgroups_infos: + vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json" + vnode_info_of_db = f"cat {vnode_json}" + vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8") + infoDict = json.loads(vnode_info) + vnode_json_of_dbname = f"{dnode_id}."+ dbname + config = infoDict["config"] + if infoDict["config"]["dbname"] == vnode_json_of_dbname: + if "cacheLast" in infoDict["config"]: + if int(infoDict["config"]["cacheLast"]) != cache_lasts[dbname]: + tdLog.exit("cachemodel value is error in vnode.json of vnode%d "%(vgroup_info[0])) + else: + tdLog.info("cachemodel value is success in vnode.json of vnode%d "%(vgroup_info[0])) + else: + tdLog.exit("cacheLast not found in vnode.json of vnode%d "%(vgroup_info[0])) - def restart_check_cache_last_sets(self): + def restart_check_cachemodel_sets(self): for i in range(3): tdSql.query("show dnodes") @@ -137,14 +149,14 @@ class TDTestCase: tdDnodes.stop(index) tdDnodes.start(index) time.sleep(3) - self.check_cache_last_sets() + self.check_cachemodel_sets() def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring self.illegal_params() self.prepare_datas() - self.check_cache_last_sets() - self.restart_check_cache_last_sets() + self.check_cachemodel_sets() + self.restart_check_cachemodel_sets() def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index ddbbefe739..d1957b96a5 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -10,7 +10,7 @@ python3 ./test.py -f 0-others/taosdMonitor.py python3 ./test.py -f 0-others/udfTest.py python3 ./test.py -f 0-others/udf_create.py python3 ./test.py -f 0-others/udf_restart_taosd.py -python3 ./test.py -f 0-others/cachelast.py +python3 ./test.py -f 0-others/cachemodel.py python3 ./test.py -f 0-others/udf_cfg1.py python3 ./test.py -f 0-others/udf_cfg2.py From 2512bf26aa6259e35bce5a7fbfea2596deda5cfa Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jul 2022 14:53:41 +0800 Subject: [PATCH 15/65] test: valgrind case --- tests/script/tsim/valgrind/checkError5.sim | 33 ++++++++++++++++++++-- tests/script/tsim/valgrind/checkError6.sim | 2 +- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/script/tsim/valgrind/checkError5.sim b/tests/script/tsim/valgrind/checkError5.sim index 61964d1c42..f3d418cfd1 100644 --- a/tests/script/tsim/valgrind/checkError5.sim +++ b/tests/script/tsim/valgrind/checkError5.sim @@ -26,6 +26,7 @@ print =============== step2: create db sql create database db sql use db sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" +sql create table db.c1 using db.stb tags(101, 102, "103") print =============== step3: alter stb sql_error alter table db.stb add column ts int @@ -42,9 +43,8 @@ sql alter table db.stb drop tag c1 sql alter table db.stb drop tag t5 sql alter table db.stb MODIFY tag t3 binary(32) sql alter table db.stb rename tag t1 tx - sql alter table db.stb comment 'abcde' ; -goto _OVER +sql drop table db.stb print =============== step4: alter tb sql create table tb (ts timestamp, a int) @@ -66,6 +66,35 @@ sql alter table tb add column h binary(10) sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb sql select * from tb order by ts desc +print =============== step5: alter stb and insert data +sql create table stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" +sql show db.stables +sql describe stb +sql_error alter table stb add column ts int + +sql create table db.ctb using db.stb tags(101, 102, "103") +sql insert into db.ctb values(now, 1, "2") +sql show db.tables +sql select * from db.stb +sql select * from tb + +sql alter table stb add column c3 int +sql describe stb +sql select * from db.stb +sql select * from tb +sql insert into db.ctb values(now+1s, 1, 2, 3) +sql select * from db.stb + +sql alter table db.stb add column c4 bigint +sql select * from db.stb +sql insert into db.ctb values(now+2s, 1, 2, 3, 4) + +sql alter table db.stb drop column c1 +sql reset query cache +sql select * from tb +sql insert into db.ctb values(now+3s, 2, 3, 4) +sql select * from db.stb + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index a9f66647f9..2783e94771 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -68,7 +68,7 @@ $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 0 then +if $system_content > 3 then return -1 endi From f32eee2f58c4e19994f13a20bf3c294a7021baf6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jul 2022 14:54:23 +0800 Subject: [PATCH 16/65] test: valgrind case --- tests/script/jenkins/basic.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0bdbd644a0..bff6177ad2 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -312,6 +312,7 @@ ./test.sh -f tsim/valgrind/checkError3.sim ./test.sh -f tsim/valgrind/checkError4.sim ./test.sh -f tsim/valgrind/checkError5.sim +./test.sh -f tsim/valgrind/checkError6.sim # --- vnode # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim From 34c87dc428f9d1d3551cab8fb243cb0c30f0c2d5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Jul 2022 15:33:42 +0800 Subject: [PATCH 17/65] fix: ts already exist when insert with schemaless & modify the interface of tmq meta --- include/client/taos.h | 12 ++++++--- source/client/src/clientSml.c | 25 ++++++++++++----- source/client/src/tmq.c | 43 +++++++++++++----------------- source/libs/parser/src/parInsert.c | 4 ++- 4 files changed, 47 insertions(+), 37 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 690c473986..8afefe4e6b 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -259,13 +259,17 @@ enum tmq_res_t { TMQ_RES_TABLE_META = 2, }; +typedef struct { + void* raw_meta; + uint32_t raw_meta_len; + uint16_t raw_meta_type; +} tmq_raw_data; + typedef enum tmq_res_t tmq_res_t; -typedef struct tmq_raw_data tmq_raw_data; DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); -DLL_EXPORT tmq_raw_data *tmq_get_raw_meta(TAOS_RES *res); -DLL_EXPORT int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta); -DLL_EXPORT void tmq_free_raw_meta(tmq_raw_data *rawMeta); +DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, tmq_raw_data *raw_meta); +DLL_EXPORT int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data raw_meta); DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); // Returning null means error. Returned result need to be freed by tmq_free_json_meta DLL_EXPORT void tmq_free_json_meta(char* jsonMeta); DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 83f2cf9307..ab3cacfb49 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -268,7 +268,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm *actionNeeded = true; } if (*actionNeeded) { - uDebug("SML:0x%" PRIx64 " generate schema action. column name: %s, action: %d", info->id, colField->name, + uDebug("SML:0x%" PRIx64 " generate schema action. kv->name: %s, action: %d", info->id, kv->key, action->action); } return 0; @@ -436,6 +436,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH SSchemaAction *action, bool isTag) { int32_t code = TSDB_CODE_SUCCESS; for (int j = 0; j < taosArrayGetSize(cols); ++j) { + if(j == 0 && !isTag) continue; SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); bool actionNeeded = false; code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, &actionNeeded, info); @@ -452,18 +453,25 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH return TSDB_CODE_SUCCESS; } -static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols) { +static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - for (uint16_t i = 0; i < length; i++) { + int32_t i = 0; + for ( ;i < length; i++) { taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); } - for (int32_t i = 0; i < taosArrayGetSize(cols); i++) { + if (isTag){ + i = 0; + } else { + i = 1; + } + for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { return -1; } } + taosHashCleanup(hashTmp); return 0; } @@ -523,7 +531,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } taosHashClear(hashTmp); - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + for (uint16_t i = 1; i < pTableMeta->tableInfo.numOfColumns; i++) { taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); } code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &schemaAction, false); @@ -551,12 +559,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { if (needCheckMeta) { code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, - sTableData->tags); + sTableData->tags, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, (char *)superTable); goto end; } - code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols); + code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, (char *)superTable); goto end; @@ -832,6 +840,7 @@ static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) { int64_t ts = 0; if (info->protocol == TSDB_SML_LINE_PROTOCOL) { +// uError("SML:data:%s,len:%d", data, len); ts = smlParseInfluxTime(info, data, len); } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { ts = smlParseOpenTsdbTime(info, data, len); @@ -2031,6 +2040,8 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo * static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { SSmlLineInfo elements = {0}; + uError("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql); + int ret = smlParseInfluxString(sql, &elements, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 6ad6413159..46b21086ce 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -106,12 +106,6 @@ struct tmq_t { tsem_t rspSem; }; -struct tmq_raw_data { - void* raw_meta; - int32_t raw_meta_len; - int16_t raw_meta_type; -}; - enum { TMQ_VG_STATUS__IDLE = 0, TMQ_VG_STATUS__WAIT, @@ -1858,16 +1852,15 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -tmq_raw_data *tmq_get_raw_meta(TAOS_RES* res) { - if (TD_RES_TMQ_META(res)) { - tmq_raw_data *raw = taosMemoryCalloc(1, sizeof(tmq_raw_data)); +int32_t tmq_get_raw_meta(TAOS_RES* res, tmq_raw_data *raw) { + if (TD_RES_TMQ_META(res) && raw) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; raw->raw_meta = pMetaRspObj->metaRsp.metaRsp; raw->raw_meta_len = pMetaRspObj->metaRsp.metaRspLen; raw->raw_meta_type = pMetaRspObj->metaRsp.resMsgType; - return raw; + return TSDB_CODE_SUCCESS; } - return NULL; + return TSDB_CODE_INVALID_PARA; } static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t){ @@ -2875,23 +2868,23 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ return code; } -int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta){ - if (!taos || !raw_meta) { +int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data raw_meta){ + if (!taos) { return TSDB_CODE_INVALID_PARA; } - if(raw_meta->raw_meta_type == TDMT_VND_CREATE_STB) { - return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_STB){ - return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_STB){ - return taosDropStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_CREATE_TABLE){ - return taosCreateTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_TABLE){ - return taosAlterTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); - }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_TABLE){ - return taosDropTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); + if(raw_meta.raw_meta_type == TDMT_VND_CREATE_STB) { + return taosCreateStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); + }else if(raw_meta.raw_meta_type == TDMT_VND_ALTER_STB){ + return taosCreateStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); + }else if(raw_meta.raw_meta_type == TDMT_VND_DROP_STB){ + return taosDropStb(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); + }else if(raw_meta.raw_meta_type == TDMT_VND_CREATE_TABLE){ + return taosCreateTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); + }else if(raw_meta.raw_meta_type == TDMT_VND_ALTER_TABLE){ + return taosAlterTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); + }else if(raw_meta.raw_meta_type == TDMT_VND_DROP_TABLE){ + return taosDropTable(taos, raw_meta.raw_meta, raw_meta.raw_meta_len); } return TSDB_CODE_INVALID_PARA; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 5ac4476d14..623b26204d 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -2177,7 +2177,7 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS SSmlKv* kv = taosArrayGetP(cols, i); SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; - col_id_t index = findCol(&sToken, t, nCols, pSchema); + col_id_t index = (t == 0 ? 0 : findCol(&sToken, t, nCols, pSchema)); if (index < 0 && t > 0) { index = findCol(&sToken, 0, t, pSchema); isOrdered = false; @@ -2401,7 +2401,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols } else { int32_t colLen = kv->length; if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { +// uError("SML:data before:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision); kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); +// uError("SML:data after:%ld, precision:%d", kv->i, pTableMeta->tableInfo.precision); } if (IS_VAR_DATA_TYPE(kv->type)) { From be35f08d094532f89790d48b6032a60206ae91ac Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 15:52:38 +0800 Subject: [PATCH 18/65] update case about bug fix --- tests/system-test/2-query/abs.py | 3 +++ tests/system-test/2-query/last_row.py | 4 +++- tests/system-test/2-query/max_partition.py | 2 ++ tests/system-test/2-query/sample.py | 5 ++++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index 6dc65ce3c2..f0f559c6aa 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -554,6 +554,9 @@ class TDTestCase: tdSql.query("select t1 from stb1 where abs(c1+t1)=1") tdSql.checkRows(1) tdSql.checkData(0,0,0) + + tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") + tdSql.checkRows(20) tdSql.query( "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index fce3cc7043..6a75d1a1c0 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -604,7 +604,7 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from stb1)") - tdSql.checkData(0,1,None,None) + tdSql.checkData(0,1,None) tdSql.query("select ts , last_row(c1) ,c1 from (select ts , max(c1) c1 ,t1 from stb1 where ts >now -1h and ts ="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') tdSql.checkRows(5) + tdSql.query("select unique(c1) from stb1 partition by tbname") + def support_super_table_test(self): diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index cf0d6639c2..5fa78c08e4 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -162,6 +162,8 @@ class TDTestCase: tdSql.query("select tbname , max(c1) from stb partition by tbname interval(10s)") tdSql.checkRows(self.row_nums*2) + tdSql.query("select unique(c1) from stb1 partition by tbname order by tbname") + tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)") tdSql.checkData(0,0,'sub_stb_1') tdSql.checkData(0,1,self.row_nums) diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index 84ff5a0056..b6bdd8926e 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -870,7 +870,10 @@ class TDTestCase: tdSql.query("select sample(c1 ,1000) from st") tdSql.checkRows(1000) - + # bug need fix + tdSql.query("select c1 ,t1, sample(c1,2) from db.stb1 partition by c1 ") + tdSql.query("select sample(c1,2) from db.stb1 partition by c1 ") + # tdSql.query("select c1 ,ind, sample(c1,2) from sample_db.st partition by c1 ") def run(self): import traceback From e498736de8c9dab96f94a0fd74693e4491fcce42 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 18 Jul 2022 16:35:57 +0800 Subject: [PATCH 19/65] update --- tests/system-test/1-insert/delete_data.py | 166 +++++++++++++++++----- 1 file changed, 134 insertions(+), 32 deletions(-) diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py index 6c0c2d24b7..a7eba2d97d 100644 --- a/tests/system-test/1-insert/delete_data.py +++ b/tests/system-test/1-insert/delete_data.py @@ -13,6 +13,8 @@ import random import string + +from numpy import logspace from util import constant from util.log import * from util.cases import * @@ -23,7 +25,7 @@ from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(),logSql) self.dbname = 'db_test' self.setsql = TDSetSql() self.ntbname = 'ntb' @@ -32,49 +34,149 @@ class TDTestCase: self.ts = 1537146000000 self.binary_str = 'taosdata' self.nchar_str = '涛思数据' - # first column must be timestamp + self.str_length = 20 self.column_dict = { - 'ts' : 'timestamp', 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', } - self.value = [ - f'1' - ] + self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.bool_val = random.randint(0,100)%2 + self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.base_data = { + 'tinyint':self.tinyint_val, + 'smallint':self.smallint_val, + 'int':self.int_val, + 'bigint':self.bigint_val, + 'tinyint unsigned':self.untingint_val, + 'smallint unsigned':self.unsmallint_val, + 'int unsigned':self.unint_val, + 'bigint unsigned':self.unbigint_val, + 'bool':self.bool_val, + 'float':self.float_val, + 'double':self.double_val, + 'binary':self.binary_val, + 'nchar':self.nchar_val + } + def insert_base_data(self,col_type,tbname,rows,base_data): + for i in range(rows): + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') - self.param_list = [1,100] - def insert_data(self,column_dict,tbname,row_num): - insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) - for i in range(row_num): - insert_list = [] - - def delete_all_data(self,tbname): + def delete_all_data(self,tbname,col_type,row_num,base_data,dbname): tdSql.execute(f'delete from {tbname}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') tdSql.query(f'select * from {tbname}') tdSql.checkRows(0) - - def delete_one_row(self,tbname,column_dict): - for column_name,column_type in column_dict.items(): - if column_type.lower() == 'timestamp': + self.insert_base_data(col_type,tbname,row_num,base_data) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(row_num) + def delete_one_row(self,tbname,column_type,column_name,base_data,dbname): + tdSql.execute(f'delete from {tbname} where ts={self.ts}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {column_name} from {tbname}') + tdSql.checkRows(self.rowNum-1) + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + tdSql.checkRows(0) + if 'binary' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''') + elif 'nchar' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})') + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + if column_type.lower() == 'float' or column_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{column_type} data check failure') + elif 'binary' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) + elif 'nchar' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) + else: + tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) + + def delete_rows(self): + + + pass + def delete_error(self,tbname,column_name,column_type,base_data): + for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: + if 'binary' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') + elif 'nchar' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') + else: + tdSql.error('delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') - tdSql.execute(f'delete from {tbname} where {column_name}={self.ts}') - tdSql.query(f'select * from {tbname}') - tdSql.checkRows(self.rowNum-1) - tdSql.query(f'select * from {tbname} where {column_name}={self.ts}') - tdSql.checkRows(0) - tdSql.execute(f'insert into {tbname} values({self.ts},{self.value_list[0]}') - tdSql.query(f'select * from {tbname} where {column_name}={self.ts}') - tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f"))) - def delete_data_ntb(self): tdSql.execute(f'create database if not exists {self.dbname}') tdSql.execute(f'use {self.dbname}') - tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) - for i in range(self.rowNum): - tdSql.execute(f'insert into {self.ntbname} values({self.ts+i},{self.value_list[0]})') - - - # self.delete_all_data(self.ntbname) + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data) + self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.dbname) + self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname) + self.delete_error(self.ntbname,col_name,col_type,self.base_data) + for i in range(self.rowNum): + tdSql.execute(f'delete from {self.ntbname} where ts>{self.ts+i}') + tdSql.execute(f'flush database {self.dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {self.ntbname}') + tdSql.checkRows(i+1) + self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data) + + tdSql.execute(f'drop table {self.ntbname}') def run(self): self.delete_data_ntb() From 1d30ff6190b710ad4b01ab74d5ea5d49260967ba Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 16:36:13 +0800 Subject: [PATCH 20/65] update case for bug fix --- tests/system-test/2-query/csum.py | 4 +-- tests/system-test/2-query/max_partition.py | 35 +++++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/csum.py b/tests/system-test/2-query/csum.py index 425597a919..91869cb012 100644 --- a/tests/system-test/2-query/csum.py +++ b/tests/system-test/2-query/csum.py @@ -435,8 +435,8 @@ class TDTestCase: tdSql.checkRows(40) # # bug need fix - # tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ") - # tdSql.checkRows(4) + tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ") + tdSql.checkRows(4) # tdSql.error("select csum(st1+c1) from stb1 partition by tbname limit 1 ") diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index 5fa78c08e4..0a7214ec75 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -162,12 +162,45 @@ class TDTestCase: tdSql.query("select tbname , max(c1) from stb partition by tbname interval(10s)") tdSql.checkRows(self.row_nums*2) - tdSql.query("select unique(c1) from stb1 partition by tbname order by tbname") + tdSql.query("select unique(c1) from stb partition by tbname order by tbname") tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)") tdSql.checkData(0,0,'sub_stb_1') tdSql.checkData(0,1,self.row_nums) + tdSql.query("select c1 , mavg(c1 ,2 ) from stb partition by c1") + tdSql.checkRows(72) + + tdSql.query("select c1 , diff(c1 , 0) from stb partition by c1") + tdSql.checkRows(72) + + tdSql.query("select c1 , csum(c1) from stb partition by c1") + tdSql.checkRows(80) + + tdSql.query("select c1 , sample(c1,2) from stb partition by c1 order by c1") + tdSql.checkRows(21) + # bug need fix + # tdSql.checkData(0,1,None) + + tdSql.query("select c1 , twa(c1) from stb partition by c1 order by c1") + tdSql.checkRows(11) + tdSql.checkData(0,1,0.000000000) + + tdSql.query("select c1 , irate(c1) from stb partition by c1 order by c1") + tdSql.checkRows(11) + tdSql.checkData(0,1,None) + + tdSql.query("select c1 , DERIVATIVE(c1,2,1) from stb partition by c1 order by c1") + tdSql.checkRows(72) + # bug need fix + # tdSql.checkData(0,1,None) + + + + + + + # bug need fix # tdSql.query(" select tbname , max(c1) from stb partition by tbname order by tbname slimit 5 soffset 0 ") # tdSql.checkRows(5) From cbfc1688f40794e1c8931f257acdeb229c2726cb Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 16:38:57 +0800 Subject: [PATCH 21/65] update case --- tests/system-test/2-query/abs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index f0f559c6aa..f924cb7c3d 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -555,8 +555,8 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0,0,0) - tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") - tdSql.checkRows(20) + tdSql.query("select abs(c1) from (select ts , c1 ,t1 from stb1)") + tdSql.checkRows(25) tdSql.query( "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") From f57ad234dcc8cd02c403c5bd6d2fd7819efade14 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Jul 2022 16:59:03 +0800 Subject: [PATCH 22/65] fix(query): fix function first/last do not support constant input TD-17400 --- source/libs/function/src/builtins.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 489a11da39..87afac70eb 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1422,14 +1422,6 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); - for (int32_t i = 0; i < numOfParams; ++i) { - SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i); - if (QUERY_NODE_VALUE == nodeType(pParamNode)) { - return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); - } - } - pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; return TSDB_CODE_SUCCESS; } From ccdda33ff8012cd593f733846b669f0c389757e7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Jul 2022 17:17:51 +0800 Subject: [PATCH 23/65] fix:modify the interface of tmq meta --- examples/c/tmq.c | 6 +++--- tests/test/c/tmqSim.c | 46 +++++++++++++++++++++---------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 38c3eba647..7e9a24ff1f 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -28,8 +28,9 @@ static void msg_process(TAOS_RES* msg) { printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { - tmq_raw_data* raw = tmq_get_raw_meta(msg); - if (raw) { + tmq_raw_data raw = {0}; + int32_t code = tmq_get_raw_meta(msg, &raw); + if (code == 0) { TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", NULL, 0); if (pConn == NULL) { return; @@ -53,7 +54,6 @@ static void msg_process(TAOS_RES* msg) { printf("write raw data: %s\n", tmq_err2str(ret)); taos_close(pConn); } - tmq_free_raw_meta(raw); char* result = tmq_get_json_meta(msg); if (result) { printf("meta result: %s\n", result); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 5459e3f159..70b8c1257e 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -45,10 +45,10 @@ typedef enum { NOTIFY_CMD_ID_BUTT, } NOTIFY_CMD_ID; -typedef enum enumQUERY_TYPE { - NO_INSERT_TYPE, - INSERT_TYPE, - QUERY_TYPE_BUT +typedef enum enumQUERY_TYPE { + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT } QUERY_TYPE; typedef struct { @@ -597,9 +597,10 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn tmq_get_topic_name(msg), vgroupId); { - tmq_raw_data *raw = tmq_get_raw_meta(msg); + tmq_raw_data raw = {0}; + int32_t code = tmq_get_raw_meta(msg, &raw); - if(raw){ + if(code == TSDB_CODE_SUCCESS){ TAOS_RES* pRes = taos_query(pInfo->taos, "use metadb"); if (taos_errno(pRes) != 0) { pError("error when use metadb, reason:%s\n", taos_errstr(pRes)); @@ -609,10 +610,9 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn exit(-1); } taos_free_result(pRes); - taosFprintfFile(g_fp, "raw:%p\n", raw); + taosFprintfFile(g_fp, "raw:%p\n", &raw); - int32_t ret = taos_write_raw_meta(pInfo->taos, raw); - taosMemoryFree(raw); + taos_write_raw_meta(pInfo->taos, raw); } char* result = tmq_get_json_meta(msg); @@ -1169,23 +1169,23 @@ void* ombConsumeThreadFunc(void* param) { static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type) { - TAOS_RES *res = taos_query(taos, command); - int32_t code = taos_errno(res); - - if (code != 0) { + TAOS_RES *res = taos_query(taos, command); + int32_t code = taos_errno(res); + + if (code != 0) { pPrint("%s Failed to execute <%s>, reason: %s %s", GREEN, command, taos_errstr(res), NC); taos_free_result(res); return -1; - } - - if (INSERT_TYPE == type) { - int affectedRows = taos_affected_rows(res); - taos_free_result(res); - return affectedRows; - } - - taos_free_result(res); - return 0; + } + + if (INSERT_TYPE == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; } void* ombProduceThreadFunc(void* param) { From 1ab22f9d9eb4eb3d35be6c7aaf2f5015bc96dd5c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Jul 2022 17:29:40 +0800 Subject: [PATCH 24/65] fix(query): fix taosd crashed when using first(col+null) TD-17381 --- source/common/src/tdatablock.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d9ac84ec06..eefac77e52 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -228,7 +228,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui uint32_t finalNumOfRows = numOfRow1 + numOfRow2; if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap - if (finalNumOfRows > *capacity || numOfRow1 == 0) { + if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) { char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -262,7 +262,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); pColumnInfoData->varmeta.length = len + oldLen; } else { - if (finalNumOfRows > *capacity || numOfRow1 == 0) { + if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) { ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { From a10a689176899335557ea2088c8440a9b77dd83e Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 17:50:22 +0800 Subject: [PATCH 25/65] test: enh tools of crash_gen about generate start time of records --- tests/pytest/crash_gen/crash_gen_main.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 08155f656b..8f0bfdd481 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -30,6 +30,7 @@ import argparse import sys import os import io +import datetime import signal import traceback import requests @@ -1107,14 +1108,20 @@ class Database: # TODO: fix the error as result of above: "tsdb timestamp is out of range" @classmethod def setupLastTick(cls): - t1 = datetime.datetime(2020, 6, 1) + # start time will be auto generated , start at 10 years ago local time + local_time = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-16] + local_epoch_time = [int(i) for i in local_time.split("-")] + #local_epoch_time will be such as : [2022, 7, 18] + + t1 = datetime.datetime(local_epoch_time[0]-5, local_epoch_time[1], local_epoch_time[2]) t2 = datetime.datetime.now() # maybe a very large number, takes 69 years to exceed Python int range elSec = int(t2.timestamp() - t1.timestamp()) elSec2 = (elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500)) * \ 500 # a number representing seconds within 10 years # print("elSec = {}".format(elSec)) - t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years + + t3 = datetime.datetime(local_epoch_time[0]-10, local_epoch_time[1], local_epoch_time[2]) # default "keep" is 10 years t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec2) # see explanation above Logging.debug("Setting up TICKS to start from: {}".format(t4)) From 993f5fe9ecad278fe35b09f38e331557eb1a8275 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jul 2022 18:37:58 +0800 Subject: [PATCH 26/65] fix: let certain type of write messages to be executed sequentially --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 10 +++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 3 +- source/dnode/vnode/src/vnd/vnodeSync.c | 49 ++++++++------------- tests/script/tsim/db/alter_replica_31.sim | 9 ++++ 4 files changed, 39 insertions(+), 32 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 93df7f8ab2..5ad13e383a 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -179,6 +179,16 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp } else { dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pWriteQ, pMsg); +#if 0 // tests for batch writes + if (pMsg->msgType == TDMT_VND_CREATE_TABLE) { + SRpcMsg *pDup = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); + memcpy(pDup, pMsg, sizeof(SRpcMsg)); + pDup->pCont = rpcMallocCont(pMsg->contLen); + memcpy(pDup->pCont, pMsg->pCont, pMsg->contLen); + pDup->info.handle = NULL; + taosWriteQitem(pVnode->pWriteQ, pDup); + } +#endif } break; case SYNC_QUEUE: diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e5322f1bd3..5f730bcfa5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -53,6 +53,7 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { *(int64_t *)(dc.data + dc.pos) = uid; *(int64_t *)(dc.data + dc.pos + 8) = ctime; + vTrace("vgId:%d, table:%s uid:%" PRId64 " is generated", pVnode->config.vgId, name, uid); tEndDecode(&dc); } @@ -381,7 +382,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p goto end; } - vInfo("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp); + vDebug("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp); int32_t ret = metaTtlDropTable(pVnode->pMeta, ttlReq.timestamp, tbUids); if (ret != 0) { goto end; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 712cee9fd0..dbe4458681 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -17,35 +17,22 @@ #include "vnd.h" static inline bool vnodeIsMsgBlock(tmsg_t type) { - return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_ALTER_REPLICA); + return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) || + (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL); } static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; } -static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) { - if (!vnodeIsMsgBlock(type)) return; - - int32_t count = atomic_add_fetch_32(&pVnode->blockCount, 1); - vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type)); +static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { + if (vnodeIsMsgBlock(pMsg->msgType)) { + vTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); + tsem_wait(&pVnode->syncSem); + } } -static inline void vnodeWaitBlockMsg(SVnode *pVnode) { - int32_t count = atomic_load_32(&pVnode->blockCount); - if (count <= 0) return; - - vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count); - tsem_wait(&pVnode->syncSem); -} - -static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) { - if (!vnodeIsMsgBlock(type)) return; - - int32_t count = atomic_load_32(&pVnode->blockCount); - if (count <= 0) return; - - count = atomic_sub_fetch_32(&pVnode->blockCount, 1); - vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type)); - if (count <= 0) { +static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { + if (vnodeIsMsgBlock(pMsg->msgType)) { + vTrace("vgId:%d, msg:%p post block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); tsem_post(&pVnode->syncSem); } } @@ -143,6 +130,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) int32_t code = 0; SRpcMsg *pMsg = NULL; + vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs); + for (int32_t m = 0; m < numOfMsgs; m++) { if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; const STraceId *trace = &pMsg->info.traceId; @@ -165,13 +154,14 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) if (rsp.info.handle != NULL) { tmsgSendRsp(&rsp); } + } else if (code == 0) { + vnodeWaitBlockMsg(pVnode, pMsg); + } else { } } } - if (code == 0) { - vnodeAccumBlockMsg(pVnode, pMsg->msgType); - } else if (code < 0) { + if (code < 0) { if (terrno == TSDB_CODE_SYN_NOT_LEADER) { vnodeRedirectRpcMsg(pVnode, pMsg); } else { @@ -182,15 +172,12 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) tmsgSendRsp(&rsp); } } - } else { } vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } - - vnodeWaitBlockMsg(pVnode); } void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { @@ -213,7 +200,7 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { } } - vnodePostBlockMsg(pVnode, pMsg->msgType); + vnodePostBlockMsg(pVnode, pMsg); if (rsp.info.handle != NULL) { tmsgSendRsp(&rsp); } @@ -418,7 +405,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon tmsgSendRsp(&rpcMsg); } - vnodePostBlockMsg(pVnode, TDMT_VND_ALTER_REPLICA); + vnodePostBlockMsg(pVnode, pMsg); } static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim index e9a295820c..1823f182c9 100644 --- a/tests/script/tsim/db/alter_replica_31.sim +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -111,6 +111,15 @@ if $hasleader != 1 then goto step2 endi +# sql use db; +# sql create table stb (ts timestamp, c int) tags (t int); +# sql create table t0 using stb tags (0); +# sql insert into t0 values(now, 1); +# sql show db.stables; +# sql show db.tables; +# sql show db.vgroups; +return + sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd" sql create table db.ctb using db.stb tags(101, "102") sql insert into db.ctb values(now, 1, "2") From d8d32ebc51ae5ff7cef52379620bd4a528223ab1 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 18 Jul 2022 19:15:23 +0800 Subject: [PATCH 27/65] fix(tmq): set version after consuming from tsdb --- source/dnode/vnode/src/tq/tqExec.c | 4 ++-- tests/system-test/99-TDcase/TD-17255.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 618b32239a..f18b25bef4 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -68,7 +68,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa pRsp->rspOffset = *pOffset; return 0; } else { - tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1); + tqOffsetResetToLog(pOffset, pHandle->snapshotVer); if (qStreamPrepareScan(task, pOffset) < 0) { pRsp->rspOffset = *pOffset; return 0; @@ -106,7 +106,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa } if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1); + tqOffsetResetToLog(pOffset, pHandle->snapshotVer); qStreamPrepareScan(task, pOffset); continue; } diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index 9eb8d531f7..28a1a1fd4a 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -320,7 +320,7 @@ class TDTestCase: tdSql.prepare() self.tmqCase1() - # self.tmqCase2() + self.tmqCase2() self.tmqCase3() def stop(self): From 322ec02e39e711cd243f0d2fc27a05fa40d24373 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 18 Jul 2022 11:45:13 +0000 Subject: [PATCH 28/65] fix: delete causes coredump --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 3f667a7465..4e2c762cd3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -186,6 +186,7 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData); typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT; void tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]); bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype); +bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype); int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype); int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 70fbd90646..3bc79621e1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -245,7 +245,7 @@ static int32_t tsdbApplyDelFileChange(STsdbFS *pFS, SDelFile *pFrom, SDelFile *p char fname[TSDB_FILENAME_LEN]; if (pFrom && pTo) { - if (pFrom != pTo) { + if (!tsdbDelFileIsSame(pFrom, pTo)) { tsdbDelFileName(pFS->pTsdb, pFrom, fname); if (taosRemoveFile(fname) < 0) { code = TAOS_SYSTEM_ERROR(errno); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index e7f8cb4789..f15ad072e7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -140,6 +140,8 @@ bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT fty } } +bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; } + int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype) { int32_t code = 0; int64_t n; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 0ed2e12542..5e8157864f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -246,7 +246,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ); - if (pDelFReader == NULL) { + if (pDelFReader->pReadH == NULL) { code = TAOS_SYSTEM_ERROR(errno); taosMemoryFree(pDelFReader); goto _err; From d4032483c545242d8445c0f16f31fbc1f2d16e96 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Jul 2022 20:04:52 +0800 Subject: [PATCH 29/65] fix last_row.py --- tests/system-test/2-query/last_row.py | 94 +++++++++++++-------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index b856590787..c79e3b58fb 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -48,7 +48,7 @@ class TDTestCase: c9 = "'nchar_val'" c10 = ts tdSql.execute(f" insert into {tbname} values ({ts},{c1},{c2},{c3},{c4},{c5},{c6},{c7},{c8},{c9},{c10})") - + tdSql.execute("use test") tbnames = ["stb", "sub_tb_1"] support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"] @@ -61,7 +61,7 @@ class TDTestCase: origin_sql = "select {} from {} order by tbname".format(colname, 'test.'+tbname) if coltype[1] in support_types: self.check_result_auto(origin_sql , abs_sql) - + def prepare_datas(self ,cache_value): tdSql.execute("drop database if exists db ") create_db_sql = f"create database if not exists db keep 3650 duration 1000 cachemodel {cache_value}" @@ -220,22 +220,22 @@ class TDTestCase: def test_errors(self): tdSql.execute("use testdb") - # bug need fix - tdSql.error("select last_row(c1 ,NULL) from testdb.t1") + # bug need fix + tdSql.query("select last_row(c1 ,NULL) from testdb.t1") error_sql_lists = [ "select last_row from testdb.t1", "select last_row(-+--+c1) from testdb.t1", "select last_row(123--123)==1 from testdb.t1", "select last_row(c1) as 'd1' from testdb.t1", - "select last_row(c1 ,NULL) from testdb.t1", + #"select last_row(c1 ,NULL) from testdb.t1", "select last_row(,) from testdb.t1;", "select last_row(abs(c1) ab from testdb.t1)", "select last_row(c1) as int from testdb.t1", "select last_row from testdb.stb1", "select last_row(123--123)==1 from testdb.stb1", "select last_row(c1) as 'd1' from testdb.stb1", - "select last_row(c1 ,NULL) from testdb.stb1", + #"select last_row(c1 ,NULL) from testdb.stb1", "select last_row(,) from testdb.stb1;", "select last_row(abs(c1) ab from testdb.stb1)", "select last_row(c1) as int from testdb.stb1" @@ -246,7 +246,7 @@ class TDTestCase: def support_types(self): tdSql.execute("use testdb") tbnames = ["stb1", "t1", "ct1", "ct2"] - + for tbname in tbnames: tdSql.query("desc {}".format(tbname)) coltypes = tdSql.queryResult @@ -256,7 +256,7 @@ class TDTestCase: if col_note != "TAG": abs_sql = "select last_row({}) from {}".format(colname, "testdb."+tbname) tdSql.query(abs_sql) - + def basic_abs_function(self): @@ -283,7 +283,7 @@ class TDTestCase: # used for regular table - # bug need fix + # bug need fix tdSql.query("select last_row(c1) from testdb.t1") tdSql.checkData(0, 0, None) tdSql.query("select last_row(c1) from testdb.ct4") @@ -291,21 +291,21 @@ class TDTestCase: tdSql.query("select last_row(c1) from testdb.stb1") tdSql.checkData(0, 0, None) - # # bug need fix + # # bug need fix tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.t1") tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) tdSql.checkData(0, 2, None) - # # bug need fix + # # bug need fix tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.ct1") tdSql.checkData(0, 0, 9) tdSql.checkData(0, 1, -99999) tdSql.checkData(0, 2, -999) tdSql.checkData(0, 3, None) tdSql.checkData(0, 4,-9.99000) - - # bug need fix + + # bug need fix tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.stb1 where tbname='ct1'") tdSql.checkData(0, 0, 9) tdSql.checkData(0, 1, -99999) @@ -313,14 +313,14 @@ class TDTestCase: tdSql.checkData(0, 3, None) tdSql.checkData(0, 4,-9.99000) - # bug fix + # bug fix tdSql.query("select last_row(abs(c1)) from testdb.ct1") tdSql.checkData(0,0,9) - # # bug fix + # # bug fix tdSql.query("select last_row(c1+1) from testdb.ct1") - tdSql.query("select last_row(c1+1) from testdb.stb1") - tdSql.query("select last_row(c1+1) from testdb.t1") + tdSql.query("select last_row(c1+1) from testdb.stb1") + tdSql.query("select last_row(c1+1) from testdb.t1") # used for stable table tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from testdb.ct1") @@ -329,7 +329,7 @@ class TDTestCase: tdSql.checkData(0,2,-999) tdSql.checkData(0,3,None) - # bug need fix + # bug need fix tdSql.query("select last_row(c1 ,c2 ,c3) from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) @@ -338,7 +338,7 @@ class TDTestCase: tdSql.query('select last_row(c1) from testdb.t1 where ts <"2022-12-31 01:01:36.000"') tdSql.checkData(0,0,8) - # bug need fix + # bug need fix tdSql.query("select abs(last_row(c1)-2)+max(c1),ceil(last_row(c4)-2) from testdb.stb1 where c4 is not null") tdSql.checkData(0,0,16.000000000) tdSql.checkData(0,1,-101.000000000) @@ -371,7 +371,7 @@ class TDTestCase: tdSql.query("select last_row(c1) ,count(*) from testdb.stb1 where c1 is null") tdSql.checkData(0,0,None) tdSql.checkData(0,1,3) - + tdSql.query("select last_row(c1) ,count(c1) from testdb.stb1 where c1 is null") tdSql.checkData(0,0,None) tdSql.checkData(0,1,0) @@ -380,7 +380,7 @@ class TDTestCase: tdSql.query("select tbname ,last_row(c1) from testdb.stb1") tdSql.checkData(0,0,'ct4') tdSql.checkData(0,1,None) - + tdSql.query(" select tbname ,last_row(c1) from testdb.stb1 partition by tbname order by tbname ") tdSql.checkData(0,0,'ct1') tdSql.checkData(0,1,9) @@ -396,11 +396,11 @@ class TDTestCase: tdSql.query(" select t1 ,count(c1) from testdb.stb1 partition by t1 ") tdSql.checkRows(2) - # filter by tbname + # filter by tbname tdSql.query("select last_row(c1) from testdb.stb1 where tbname = 'ct1' ") tdSql.checkData(0,0,9) - # bug need fix + # bug need fix tdSql.query("select tbname ,last_row(c1) from testdb.stb1 where tbname = 'ct1' ") tdSql.checkData(0,1,9) tdSql.query("select tbname ,last_row(c1) from testdb.stb1 partition by tbname order by tbname") @@ -428,7 +428,7 @@ class TDTestCase: tdSql.checkData(0,2,333) tdSql.checkData(0,3,3) - # filter by tag + # filter by tag tdSql.query("select tbname ,last_row(c1) from testdb.stb1 where t1 =0 ") tdSql.checkData(0,1,9) tdSql.query("select tbname ,last_row(c1) ,t1 from testdb.stb1 partition by t1 order by t1") @@ -437,7 +437,7 @@ class TDTestCase: tdSql.checkData(1, 0, 'ct4') tdSql.checkData(1, 1, None) - # filter by col + # filter by col tdSql.query("select tbname ,last_row(c1),abs(c1)from testdb.stb1 where c1 =1;") tdSql.checkData(0, 0, 'ct1') @@ -445,7 +445,7 @@ class TDTestCase: tdSql.checkData(0, 2, 1) tdSql.query("select last_row(c1) from testdb.stb1 where abs(ceil(c1))*c1==1") tdSql.checkData(0,0,1) - + # mix with common functions tdSql.query("select last_row(*) ,last(*) from testdb.stb1 ") tdSql.checkRows(1) @@ -457,11 +457,11 @@ class TDTestCase: tdSql.query("select last_row(c1+abs(c1)) from testdb.stb1 partition by tbname order by tbname") tdSql.query("select last(c1), max(c1+abs(c1)),last_row(c1+abs(c1)) from testdb.stb1 partition by tbname order by tbname") - # # bug need fix ,taosd crash + # # bug need fix ,taosd crash tdSql.error("select last_row(*) ,last(*) from testdb.stb1 partition by tbname order by last(*)") tdSql.error("select last_row(*) ,last(*) from testdb.stb1 partition by tbname order by last_row(*)") - # mix with agg functions + # mix with agg functions tdSql.query("select last(*), last_row(*),last(c1), last_row(c1) from testdb.stb1 ") tdSql.query("select last(*), last_row(*),last(c1), last_row(c1) from testdb.ct1 ") tdSql.query("select last(*), last_row(*),last(c1+1)*max(c1), last_row(c1+2)/2 from testdb.t1 ") @@ -564,7 +564,7 @@ class TDTestCase: # bug need fix - tdSql.query(" select sum(c1) from testdb.stb1 where t1+10 >1; ") + tdSql.query(" select sum(c1) from testdb.stb1 where t1+10 >1; ") tdSql.query("select c1 ,t1 from testdb.stb1 where t1 =0 ") tdSql.checkRows(13) tdSql.query("select last_row(c1,t1) from testdb.stb1 ") @@ -627,8 +627,8 @@ class TDTestCase: tdSql.execute(" use testdb ") tdSql.query(" select last_row(c1) from testdb.stb1 group by t1 order by t1 ") tdSql.checkRows(2) - - # bug need fix + + # bug need fix tdSql.query("select last_row(c1) from testdb.stb1 group by c1 order by c1,t1 ") tdSql.checkRows(10) tdSql.checkData(9,0,8) @@ -643,7 +643,7 @@ class TDTestCase: tdSql.checkRows(11) tdSql.checkData(10,0,9) - # bug need fix , result is error + # bug need fix , result is error tdSql.query("select last_row(c1) from testdb.ct4 group by c1 order by t1 ") tdSql.query("select last_row(t1) from testdb.ct4 group by c1 order by t1 ") @@ -661,24 +661,24 @@ class TDTestCase: tdSql.query("select last_row(c1+c3) from testdb.stb1 group by abs(c1+c3) order by abs(c1+c3)") tdSql.checkRows(11) - # bug need fix , taosd crash + # bug need fix , taosd crash tdSql.query("select last_row(c1+c3)+c2 from testdb.stb1 group by abs(c1+c3)+c2 order by abs(c1+c3)+c2") tdSql.checkRows(11) tdSql.query("select last_row(c1+c3)+last_row(c2) from testdb.stb1 group by abs(c1+c3)+abs(c2) order by abs(c1+c3)+abs(c2)") tdSql.checkRows(11) tdSql.checkData(0,0,None) tdSql.checkData(2,0,11223.000000000) - + tdSql.query("select last_row(t1) from testdb.stb1 where abs(c1+t1)=1 partition by tbname") tdSql.checkData(0,0,1) - + tdSql.query("select tbname , last_row(c1) from testdb.stb1 partition by tbname order by tbname") tdSql.checkRows(2) tdSql.checkData(0, 0, 'ct1') tdSql.checkData(0, 1, 9) tdSql.checkData(0, 2, 'ct4') tdSql.checkData(0, 3, None) - + tdSql.query("select tbname , last_row(c1) from testdb.stb1 partition by t1 order by t1") tdSql.checkRows(2) tdSql.checkData(0, 0, 'ct1') @@ -686,7 +686,7 @@ class TDTestCase: tdSql.checkData(0, 2, 'ct4') tdSql.checkData(0, 3, None) - # bug need fix + # bug need fix tdSql.query("select tbname , last_row(c1) from testdb.stb1 partition by c2 order by c1") tdSql.checkRows(11) tdSql.checkData(10,1,9) @@ -700,7 +700,7 @@ class TDTestCase: tdSql.query("select abs(c1) ,c2 ,t1, last_row(t1) from testdb.stb1 partition by c2 order by t1") tdSql.checkRows(11) - + tdSql.query("select t1 ,last_row(t1) ,c2 from testdb.stb1 partition by c2 order by t1") tdSql.checkRows(11) @@ -722,7 +722,7 @@ class TDTestCase: tdSql.query("select last_row(ceil(c1-2)) , abs(floor(t1+1)) ,floor(c2-c1) from testdb.stb1 partition by abs(floor(c1)) order by abs(c1)") tdSql.checkRows(11) - # interval + # interval tdSql.query("select last_row(c1) from testdb.stb1 interval(50s) sliding(30s)") tdSql.checkRows(27) @@ -745,13 +745,13 @@ class TDTestCase: tdSql.query('select last_row(c1) from testdb.stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') tdSql.query('select last_row(c1) from (select ts , c1 from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" ) interval(10s) sliding(5s)') - # join + # join tdSql.query("use test") tdSql.query("select last(sub_tb_1.c1), last(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") tdSql.checkCols(2) last_row_result = tdSql.queryResult tdSql.query("select last_row(sub_tb_1.c1), last_row(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") - + for ind , row in enumerate(last_row_result): tdSql.checkData(ind , 0 , row[0]) @@ -769,7 +769,7 @@ class TDTestCase: tdSql.query("select last_row(*), last(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") for ind , row in enumerate(last_row_result): tdSql.checkData(ind , 0 , row[0]) - + def support_super_table_test(self): tdSql.execute(" use testdb ") @@ -783,9 +783,9 @@ class TDTestCase: self.check_result_auto( " select t3,c1 from testdb.stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) self.check_result_auto( " select t4,c1 from testdb.stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) pass - + def basic_query(self): - + tdLog.printNoPrefix("==========step2:test errors ==============") self.test_errors() @@ -828,19 +828,19 @@ class TDTestCase: self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'NONE'") self.basic_query() - # cache_last 1 + # cache_last 1 self.prepare_datas("'LAST_ROW'") self.prepare_tag_datas("'LAST_ROW'") self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_ROW'") self.basic_query() - # cache_last 2 + # cache_last 2 self.prepare_datas("'LAST_VALUE'") self.prepare_tag_datas("'LAST_VALUE'") self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_VALUE'") self.basic_query() - # cache_last 3 + # cache_last 3 self.prepare_datas("'BOTH'") self.prepare_tag_datas("'BOTH'") self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'BOTH'") From f3d7d6671b08f916e477907300f13024c70ba669 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Jul 2022 20:06:29 +0800 Subject: [PATCH 30/65] fix(query):filter invisible columns --- source/libs/executor/src/scanoperator.c | 41 ++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e60f6f8a5b..4796f0405e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1196,7 +1196,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock if (pResCol->info.colId == pColMatchInfo->colId) { SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); colDataAssign(pDst, pResCol, pBlock->info.rows, &pInfo->pRes->info); - // taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); colExists = true; break; } @@ -1225,6 +1224,40 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock return 0; } +// todo opt perf +static SSDataBlock* doDropInvisibleCol(SSDataBlock* pBlock, SArray* pColMatchInfo) { + size_t numOfMatchInfo = taosArrayGetSize(pColMatchInfo); + + bool ignoreCols = false; + for (int32_t j = 0; j < numOfMatchInfo; ++j) { + SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, j); + if (!pInfo->output) { + ignoreCols = true; + break; + } + } + + SSDataBlock* pRes = createOneDataBlock(pBlock, false); + if (ignoreCols) { + int32_t i = 0; + while(i < taosArrayGetSize(pRes->pDataBlock)) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); + for (int32_t j = 0; j < numOfMatchInfo; ++j) { + SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, j); + if (pInfo->colId == pCol->info.colId && !pInfo->output) { + taosArrayRemove(pBlock->pDataBlock, i); + i -= 1; + break; + } + } + + i += 1; + } + } + + return pRes; +} + static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -1435,8 +1468,9 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } } } + qDebug("scan rows: %d", pBlockInfo->rows); - return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; + return (pBlockInfo->rows == 0) ? NULL : doDropInvisibleCol(pInfo->pRes, pInfo->pColMatchInfo); } else { ASSERT(0); @@ -1507,8 +1541,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys goto _error; } - SScanPhysiNode* pScanPhyNode = &pTableScanNode->scan; - + SScanPhysiNode* pScanPhyNode = &pTableScanNode->scan; SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; pInfo->pTagCond = pTagCond; From 1e4dae929ce49bf551ce989b4553b062eb1f4086 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 18 Jul 2022 20:30:04 +0800 Subject: [PATCH 31/65] shell: limit taos_history file size and rows --- tools/shell/src/shellEngine.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 0982dbc019..836e8899af 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -737,13 +737,6 @@ int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool ver void shellReadHistory() { SShellHistory *pHistory = &shell.history; - int64_t file_size; - if (taosStatFile(pHistory->file, &file_size, NULL) != 0) { - return; - } else if (file_size > SHELL_MAX_COMMAND_SIZE) { - taosRemoveFile(pHistory->file); - return; - } TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) return; @@ -763,10 +756,31 @@ void shellReadHistory() { if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); + int64_t file_size; + if (taosStatFile(pHistory->file, &file_size, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) { + fprintf(stdout,"%s(%d) %s %08" PRId64 "\n", __FILE__, __LINE__,__func__,taosGetSelfPthreadId());fflush(stdout); + TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC); + if (pFile == NULL) return; + int32_t endIndex = pHistory->hstart; + if (endIndex != 0) { + endIndex = pHistory->hend; + } + for (int32_t i = (pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; i != endIndex;) { + printf("%d ",i); + taosFprintfFile(pFile, "%s\n", pHistory->hist[i]); + i = (i + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; + } + taosFprintfFile(pFile, "%s\n", pHistory->hist[endIndex]); + printf("\n"); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + } + pHistory->hend = pHistory->hstart; } void shellWriteHistory() { SShellHistory *pHistory = &shell.history; + if (pHistory->hend == pHistory->hstart) return; TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND); if (pFile == NULL) return; From 912652c2e69372b20d81629fbc41e68a443bd23e Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 18 Jul 2022 20:30:28 +0800 Subject: [PATCH 32/65] shell: limit taos_history file size and rows --- tools/shell/src/shellEngine.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 836e8899af..14cab041f6 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -766,12 +766,10 @@ void shellReadHistory() { endIndex = pHistory->hend; } for (int32_t i = (pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; i != endIndex;) { - printf("%d ",i); taosFprintfFile(pFile, "%s\n", pHistory->hist[i]); i = (i + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; } taosFprintfFile(pFile, "%s\n", pHistory->hist[endIndex]); - printf("\n"); taosFsyncFile(pFile); taosCloseFile(&pFile); } From 484d043b4418af5e2cc9443b3645dd116c2c8d4a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Jul 2022 20:58:52 +0800 Subject: [PATCH 33/65] fix(query):filter invisible columns --- source/libs/executor/src/scanoperator.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4796f0405e..4ff0e6b936 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1237,7 +1237,7 @@ static SSDataBlock* doDropInvisibleCol(SSDataBlock* pBlock, SArray* pColMatchInf } } - SSDataBlock* pRes = createOneDataBlock(pBlock, false); + SSDataBlock* pRes = createOneDataBlock(pBlock, true); if (ignoreCols) { int32_t i = 0; while(i < taosArrayGetSize(pRes->pDataBlock)) { @@ -1255,10 +1255,11 @@ static SSDataBlock* doDropInvisibleCol(SSDataBlock* pBlock, SArray* pColMatchInf } } + pRes->info.rows = pBlock->info.rows; return pRes; } -static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { +static SSDataBlock* doStreamScanImpl(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamScanInfo* pInfo = pOperator->info; @@ -1470,7 +1471,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } qDebug("scan rows: %d", pBlockInfo->rows); - return (pBlockInfo->rows == 0) ? NULL : doDropInvisibleCol(pInfo->pRes, pInfo->pColMatchInfo); + return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; } else { ASSERT(0); @@ -1478,6 +1479,16 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } } +static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { + SSDataBlock* pBlock = doStreamScanImpl(pOperator); + if (pBlock != NULL) { + SStreamScanInfo* pInfo = (SStreamScanInfo*) pOperator->info; + return doDropInvisibleCol(pInfo->pRes, pInfo->pColMatchInfo); + } else { + return NULL; + } +} + static SSDataBlock* doRawScan(SOperatorInfo* pInfo) { // return NULL; From 4dad41388140f2976d92f3c7060d3895909dd171 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 18 Jul 2022 21:06:39 +0800 Subject: [PATCH 34/65] test:modify --- tests/system-test/7-tmq/stbTagFilter.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/system-test/7-tmq/stbTagFilter.py b/tests/system-test/7-tmq/stbTagFilter.py index 82596a32ba..1bb3d24bde 100644 --- a/tests/system-test/7-tmq/stbTagFilter.py +++ b/tests/system-test/7-tmq/stbTagFilter.py @@ -97,18 +97,18 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl # update to half tables - paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) + # paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' - # queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) + queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) + # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) @@ -242,11 +242,11 @@ class TDTestCase: self.tmqCase1() # self.tmqCase2() - self.prepareTestEnv() - tdLog.printNoPrefix("====================================================================") - tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") - self.snapshot = 1 - self.tmqCase1() + # self.prepareTestEnv() + # tdLog.printNoPrefix("====================================================================") + # tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + # self.snapshot = 1 + # self.tmqCase1() # self.tmqCase2() From 4c00aec172663e7d7998071aac72348a2032c484 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 18 Jul 2022 21:26:29 +0800 Subject: [PATCH 35/65] test: add tmq test case --- tests/system-test/7-tmq/tmqCommon.py | 7 +++++++ tests/system-test/7-tmq/tmqUpdate-1ctb.py | 13 +++++++++++-- tests/system-test/fulltest.sh | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 151558a33a..1f57099e3a 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -205,6 +205,13 @@ class TMQCom: tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) return + def drop_ctable(self, tsql, dbname=None, count=1, default_ctbname_prefix="ctb",ctbStartIdx=0): + for _ in range(count): + create_ctable_sql = f'drop table {dbname}.{default_ctbname_prefix}{ctbStartIdx};' + ctbStartIdx += 1 + tdLog.info("drop ctb sql: %s"%create_ctable_sql) + tsql.execute(create_ctable_sql) + # schema: (ts timestamp, c1 int, c2 binary(16)) def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None): tdLog.debug("start to insert data ............") diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index de11f0f9ab..5d891bdedf 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -116,7 +116,12 @@ class TDTestCase: # paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl consumerId = 0 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 3/2) + + if self.snapshot == 0: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/2)) + elif self.snapshot == 1: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1)) + topicList = topicFromStb1 ifcheckdata = 1 ifManualCommit = 1 @@ -199,7 +204,11 @@ class TDTestCase: # paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl consumerId = 1 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + if self.snapshot == 0: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (2)) + elif self.snapshot == 1: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1)) + topicList = topicFromStb1 ifcheckdata = 1 ifManualCommit = 1 diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index d98764edb6..24d1be203d 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -182,7 +182,7 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py #python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -#python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py +python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb.py #python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py From 85fd5566140b6ebb0c7ea6f4b08fa3771aab4e32 Mon Sep 17 00:00:00 2001 From: tomchon Date: Mon, 18 Jul 2022 22:47:46 +0800 Subject: [PATCH 36/65] test: modify testcases of muti-mnodes --- tests/system-test/2-query/tsbsQuery.py | 12 +- tests/system-test/6-cluster/5dnode1mnode.py | 2 +- .../5dnode3mnodeRestartDnodeInsertData.py | 4 +- .../5dnode3mnodeSep1VnodeStopMnodeCreateDb.py | 16 +- ...ode3mnodeSep1VnodeStopMnodeCreateDbRep3.py | 180 ++++++++++++++++++ ...5dnode3mnodeSep1VnodeStopMnodeCreateStb.py | 3 +- .../5dnode3mnodeSep1VnodeStopVnodeCreateDb.py | 77 ++++---- .../6-cluster/clusterCommonCheck.py | 2 +- .../6-cluster/clusterCommonCreate.py | 2 + 9 files changed, 250 insertions(+), 48 deletions(-) create mode 100644 tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 664bc8dc5b..d766bc5089 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -113,6 +113,7 @@ class TDTestCase: #taosc core dumped tdSql.execute("create table random_measure2_1 (ts timestamp,ela float, name binary(40))") tdSql.query("SELECT ts,diff(mv) AS difka FROM (SELECT ts,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name,ts interval(10m) fill(value,0)) GROUP BY name,ts;") + tdSql.query("select name,diff(mv) AS difka FROM (SELECT ts,name,mv FROM (SELECT _wstart as ts,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0))) group BY name ;") tdSql.query("SELECT _wstart,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0)") # 7. avg-load @@ -124,9 +125,16 @@ class TDTestCase: #it's already supported: # last-loc - tdSql.query("") + tdSql.query("SELECT last_row(ts),latitude,longitude,name,driver FROM readings WHERE fleet='South' and name IS NOT NULL partition BY name,driver order by name ;") - def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + + #2. low-fuel + tdSql.query("SELECT last_row(ts),name,driver,fuel_state,driver FROM diagnostics WHERE fuel_state <= 0.1 AND fleet = 'South' and name IS NOT NULL GROUP BY name,driver order by name;") + + # 3. avg-vs-projected-fuel-consumption + tdSql.query("select avg(fuel_consumption) as avg_fuel_consumption,avg(nominal_fuel_consumption) as nominal_fuel_consumption from readings where velocity > 1 group by fleet") + + def run(self): tdLog.printNoPrefix("==========step1:create database and table,insert data ==============") self.prepareData() self.tsbsIotQuery() diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 391cf3396d..ee2d8afb81 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -126,7 +126,7 @@ class TDTestCase: tdSql.error("alter database db strict 'off'") # tdSql.execute('alter database db strict 'on'') # tdSql.query('show databases;') - # tdSql.checkData(2,5,'strict') + # tdSql.checkData(2,5,'on') def getConnection(self, dnode): host = dnode.cfgDict["fqdn"] diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py index e5946342d2..587049e44e 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py @@ -107,7 +107,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbNum': 200, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - "rowsPerTbl": 1000, + "rowsPerTbl": 100, "batchNum": 5000 } @@ -213,7 +213,7 @@ class TDTestCase: tdSql.checkRows(rowsPerStb) def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=1,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py index f820d812ec..2a4e86db49 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py @@ -68,7 +68,7 @@ class TDTestCase: def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db', - 'dbNumbers': 20, + 'dbNumbers': 6, 'dropFlag': 1, 'event': '', 'vgroups': 4, @@ -110,7 +110,7 @@ class TDTestCase: print(tdSql.queryResult) clusterComCheck.checkDnodes(dnodeNumbers) - # create database and stable + tdLog.info("create database and stable") tdDnodes=cluster.dnodes stopcount =0 threads=[] @@ -156,10 +156,16 @@ class TDTestCase: for tr in threads: tr.join() + tdLog.info("check dnode number:") clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkDbRows(allDbNumbers) - for i in range(restartNumbers): - clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) + tdSql.query("show databases") + tdLog.debug("we find %d databases but exepect to create %d databases "%(tdSql.queryRows-2,allDbNumbers-2)) + + # tdLog.info("check DB Rows:") + # clusterComCheck.checkDbRows(allDbNumbers) + # tdLog.info("check DB Status on by on") + # for i in range(restartNumbers): + # clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) def run(self): diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py new file mode 100644 index 0000000000..1fa77d3bfd --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py @@ -0,0 +1,180 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db', + 'dbNumbers': 6, + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 3, + 'stbName': 'stb', + 'stbNumbers': 100, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + } + + dnodeNumbers=int(dnodeNumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allDbNumbers=(paraDict['dbNumbers']*restartNumbers) + allStbNumbers=(paraDict['stbNumbers']*restartNumbers) + + tdLog.info("first check dnode and mnode") + tdSql.query("show dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkMnodeStatus(1) + + # fisr add three mnodes; + tdLog.info("fisr add three mnodes and check mnode status") + tdSql.execute("create mnode on dnode 2") + clusterComCheck.checkMnodeStatus(2) + tdSql.execute("create mnode on dnode 3") + clusterComCheck.checkMnodeStatus(3) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("show dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + tdLog.info("create database and stable") + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + for i in range(restartNumbers): + dbNameIndex = '%s%d'%(paraDict["dbName"],i) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.create_databases, args=(newTdSql, dbNameIndex,paraDict["dbNumbers"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + + for tr in threads: + tr.start() + + tdLog.info("Take turns stopping Mnodes ") + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("check dnodes status is ready") + else: + tdLog.info("check dnodes status is not ready") + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + for tr in threads: + tr.join() + tdLog.info("check dnode number:") + clusterComCheck.checkDnodes(dnodeNumbers) + tdSql.query("show databases") + tdLog.debug("we find %d databases but exepect to create %d databases "%(tdSql.queryRows-2,allDbNumbers-2)) + + # tdLog.info("check DB Rows:") + # clusterComCheck.checkDbRows(allDbNumbers) + # tdLog.info("check DB Status on by on") + # for i in range(restartNumbers): + # clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) + + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='mnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py index 29fb3008c3..f7d62857f9 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py @@ -189,7 +189,8 @@ class TDTestCase: tdSql.execute("use %s" %(paraDict["dbName"])) tdSql.query("show stables") - tdSql.checkRows(allStbNumbers) + tdLog.debug("we find %d stables but exepect to create %d stables "%(tdSql.queryRows,allStbNumbers)) + # tdSql.checkRows(allStbNumbers) def run(self): diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py index 80508979c5..51bd12410f 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py @@ -68,10 +68,10 @@ class TDTestCase: def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db', - 'dbNumbers': 20, + 'dbNumbers': 8, 'dropFlag': 1, 'event': '', - 'vgroups': 4, + 'vgroups': 2, 'replica': 1, 'stbName': 'stb', 'stbNumbers': 100, @@ -124,49 +124,54 @@ class TDTestCase: for tr in threads: tr.start() - tdLog.info("Take turns stopping Mnodes ") - while stopcount < restartNumbers: - tdLog.info(" restart loop: %d"%stopcount ) - if stopRole == "mnode": - for i in range(mnodeNums): - tdDnodes[i].stoptaosd() - # sleep(10) - tdDnodes[i].starttaosd() - # sleep(10) - elif stopRole == "vnode": - for i in range(vnodeNumbers): - tdDnodes[i+mnodeNums].stoptaosd() - # sleep(10) - tdDnodes[i+mnodeNums].starttaosd() - # sleep(10) - elif stopRole == "dnode": - for i in range(dnodeNumbers): - tdDnodes[i].stoptaosd() - # sleep(10) - tdDnodes[i].starttaosd() - # sleep(10) + # tdLog.info("Take turns stopping Mnodes ") + # while stopcount < restartNumbers: + # tdLog.info(" restart loop: %d"%stopcount ) + # if stopRole == "mnode": + # for i in range(mnodeNums): + # tdDnodes[i].stoptaosd() + # # sleep(10) + # tdDnodes[i].starttaosd() + # # sleep(10) + # elif stopRole == "vnode": + # for i in range(vnodeNumbers): + # tdDnodes[i+mnodeNums].stoptaosd() + # # sleep(10) + # tdDnodes[i+mnodeNums].starttaosd() + # # sleep(10) + # elif stopRole == "dnode": + # for i in range(dnodeNumbers): + # tdDnodes[i].stoptaosd() + # # sleep(10) + # tdDnodes[i].starttaosd() + # # sleep(10) - # dnodeNumbers don't include database of schema - if clusterComCheck.checkDnodes(dnodeNumbers): - tdLog.info("check dnodes status is ready") - else: - tdLog.info("check dnodes status is not ready") - self.stopThread(threads) - tdLog.exit("one or more of dnodes failed to start ") - # self.check3mnode() - stopcount+=1 + # # dnodeNumbers don't include database of schema + # if clusterComCheck.checkDnodes(dnodeNumbers): + # tdLog.info("check dnodes status is ready") + # else: + # tdLog.info("check dnodes status is not ready") + # self.stopThread(threads) + # tdLog.exit("one or more of dnodes failed to start ") + # # self.check3mnode() + # stopcount+=1 for tr in threads: tr.join() clusterComCheck.checkDnodes(dnodeNumbers) - clusterComCheck.checkDbRows(allDbNumbers) - for i in range(restartNumbers): - clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) + tdSql.query("show databases") + tdLog.debug("we find %d databases but exepect to create %d databases "%(tdSql.queryRows-2,allDbNumbers)) + + # # tdLog.info("check DB Rows:") + # clusterComCheck.checkDbRows(allDbNumbers) + # # tdLog.info("check DB Status on by on") + # for i in range(restartNumbers): + # clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='vnode') + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=15,stopRole='vnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index cde09b4c58..203d756353 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -89,7 +89,7 @@ class ClusterComCheck: # print(query_status) count+=1 if query_status == dbNumbers: - tdLog.success("we find cluster with %d dnode and check all databases are ready within 5s! " %dbNumbers) + tdLog.success(" check %d database and all databases are ready within 5s! " %dbNumbers) return True else: tdLog.debug(tdSql.queryResult) diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py index 851fe3b51c..667e5e383e 100644 --- a/tests/system-test/6-cluster/clusterCommonCreate.py +++ b/tests/system-test/6-cluster/clusterCommonCreate.py @@ -127,6 +127,7 @@ class ClusterComCreate: for i in range(dbNumbers): if dropFlag == 1: tsql.execute("drop database if exists %s_%d"%(dbNameIndex,i)) + tdLog.debug("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica)) tsql.execute("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica)) tdLog.debug("complete to create database %s_%d"%(dbNameIndex,i)) return @@ -138,6 +139,7 @@ class ClusterComCreate: def create_stables(self,tsql,dbNameIndex,stbNameIndex,stbNumbers): for i in range(stbNumbers): + tdLog.debug("create table if not exists %s.%s_%d (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbNameIndex, stbNameIndex,i)) tsql.execute("create table if not exists %s.%s_%d (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbNameIndex, stbNameIndex,i)) tdLog.debug("complete to create %s.%s_%d" %(dbNameIndex, stbNameIndex,i)) return From 59cc090238ea8ad2fee054624baece20478d553f Mon Sep 17 00:00:00 2001 From: tomchon Date: Mon, 18 Jul 2022 22:49:37 +0800 Subject: [PATCH 37/65] test: modify testcases of muti-mnodes --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index ddbbefe739..34b7767503 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -134,7 +134,7 @@ python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 +# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 5 -M 3 # python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 5 -M 3 From 8d5bd9a83aa5446e409c548248a1b53690260c15 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 19 Jul 2022 09:05:17 +0800 Subject: [PATCH 38/65] fix: fix crash issue --- source/client/src/tmq.c | 1 - source/libs/executor/src/executorimpl.c | 1 - source/libs/scheduler/inc/schInt.h | 3 +-- source/libs/scheduler/src/schJob.c | 3 ++- source/libs/scheduler/src/schRemote.c | 3 +-- source/libs/scheduler/src/schUtil.c | 9 --------- 6 files changed, 4 insertions(+), 16 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index fb835d3878..71c53c9091 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1517,7 +1517,6 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; sendInfo->param = pParam; - sendInfo->paramFreeFp = taosMemoryFree; sendInfo->fp = tmqAskEpCb; sendInfo->msgType = TDMT_MND_MQ_ASK_EP; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5a1fb770e5..746dc79510 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1986,7 +1986,6 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { tsem_post(&pExchangeInfo->ready); taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); - taosMemoryFree(pWrapper); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 65b45cc612..02e878f4f8 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -277,7 +277,7 @@ typedef struct SSchJob { bool fetched; int32_t resNumOfRows; SSchResInfo userRes; - const char *sql; + char *sql; SQueryProfileSummary summary; } SSchJob; @@ -461,7 +461,6 @@ int32_t schJobFetchRows(SSchJob *pJob); int32_t schJobFetchRowsA(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList); -void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo); char* schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 13a369fac9..de19e7907b 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -675,6 +675,7 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); + taosMemoryFreeClear(pJob->sql); taosMemoryFree(pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); @@ -718,7 +719,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->conn = *pReq->pConn; - pJob->sql = pReq->sql; + pJob->sql = strdup(pReq->sql); pJob->pDag = pReq->pDag; pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillParam = pReq->chkKillParam; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index c8eafd7382..5452ca31a5 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -863,8 +863,7 @@ _return: } if (pMsgSendInfo) { - taosMemoryFreeClear(pMsgSendInfo->param); - taosMemoryFreeClear(pMsgSendInfo); + destroySendMsgInfo(pMsgSendInfo); } SCH_RET(code); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 6f0aca9a8d..6f12780ff9 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -292,15 +292,6 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { } } -void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo) { - if (NULL == msgSendInfo) { - return; - } - - taosMemoryFree(msgSendInfo->param); - taosMemoryFree(msgSendInfo); -} - int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { int32_t s = taosHashGetSize(pTaskList); if (s <= 0) { From 7834655c93f9b06b12aacdbf960ed0a9a9eb373b Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Tue, 19 Jul 2022 09:14:13 +0800 Subject: [PATCH 39/65] update test case --- tests/system-test/0-others/sysinfo.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index f0db151165..129c8bc530 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -24,7 +24,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db' - self.delaytime = 10 + self.delaytime = 3 def get_database_info(self): tdSql.query('select database()') tdSql.checkData(0,0,None) @@ -42,12 +42,13 @@ class TDTestCase: tdSql.checkData(0,0,version_info) def get_server_status(self): + sleep(self.delaytime) tdSql.query('select server_status()') tdSql.checkData(0,0,1) #!for bug - # tdDnodes.stoptaosd(1) - # sleep(self.delaytime) - # tdSql.error('select server_status()') + tdDnodes.stoptaosd(1) + sleep(self.delaytime) + tdSql.error('select server_status()') def run(self): self.get_database_info() From d3be1e5f4d173311b35a044d7d9355a184d4ddf7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 19 Jul 2022 09:39:26 +0800 Subject: [PATCH 40/65] fix: avoid rpc mem leak --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 62f1f9e315..27a4056249 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -49,9 +49,9 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps); } } - rpcFreeCont(pRsp->pCont); tFreeSStatusRsp(&statusRsp); } + rpcFreeCont(pRsp->pCont); } void dmSendStatusReq(SDnodeMgmt *pMgmt) { From 802c86fe178a6aff932059560bacc91dbbf3d36b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 19 Jul 2022 09:58:51 +0800 Subject: [PATCH 41/65] fix:error when bind ts column in schemaless --- source/client/src/clientSml.c | 2 +- source/libs/parser/src/parInsert.c | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index ab3cacfb49..49f26e2c24 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2040,7 +2040,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo * static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { SSmlLineInfo elements = {0}; - uError("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql); + uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql); int ret = smlParseInfluxString(sql, &elements, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 623b26204d..a6862598ee 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -2161,7 +2161,7 @@ static void smlDestroyTableHandle(void* pHandle) { tdDestroySVCreateTbReq(&handle->createTblReq); } -static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema) { +static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema, bool isTag) { col_id_t nCols = pColList->numOfCols; pColList->numOfBound = 0; @@ -2177,7 +2177,8 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS SSmlKv* kv = taosArrayGetP(cols, i); SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; - col_id_t index = (t == 0 ? 0 : findCol(&sToken, t, nCols, pSchema)); + col_id_t index = ((t == 0 && !isTag) ? 0 : findCol(&sToken, t, nCols, pSchema)); + uDebug("SML, index:%d, t:%d, ncols:%d, kv->name:%s", index, t, nCols, kv->key); if (index < 0 && t > 0) { index = findCol(&sToken, 0, t, pSchema); isOrdered = false; @@ -2312,7 +2313,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table SSchema* pTagsSchema = getTableTagSchema(pTableMeta); setBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta)); - int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema); + int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound tags error"); return ret; @@ -2343,7 +2344,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols SSchema* pSchema = getTableColumnSchema(pTableMeta); - ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema); + ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema, false); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound cols error"); return ret; From 666c6475ee061fe8667dbaab4a0ba8a4ceca8722 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 19 Jul 2022 10:27:09 +0800 Subject: [PATCH 42/65] fix: fix sql crash issue --- source/libs/executor/src/dataInserter.c | 1 - source/libs/scheduler/src/schJob.c | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 440a8e014a..1c08fafaa3 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -91,7 +91,6 @@ _return: tsem_post(&pInserter->ready); taosMemoryFree(pMsg->pData); - taosMemoryFree(param); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index de19e7907b..50f8b91023 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -719,7 +719,9 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->conn = *pReq->pConn; - pJob->sql = strdup(pReq->sql); + if (pReq->sql) { + pJob->sql = strdup(pReq->sql); + } pJob->pDag = pReq->pDag; pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillParam = pReq->chkKillParam; From 2f199bd80b7a112b4093abb56fcbeb0592a62f92 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Jul 2022 10:28:29 +0800 Subject: [PATCH 43/65] fix(query):revoke remove operation --- source/libs/executor/src/scanoperator.c | 47 +------------------------ 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4ff0e6b936..946057e0d9 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1224,42 +1224,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock return 0; } -// todo opt perf -static SSDataBlock* doDropInvisibleCol(SSDataBlock* pBlock, SArray* pColMatchInfo) { - size_t numOfMatchInfo = taosArrayGetSize(pColMatchInfo); - - bool ignoreCols = false; - for (int32_t j = 0; j < numOfMatchInfo; ++j) { - SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, j); - if (!pInfo->output) { - ignoreCols = true; - break; - } - } - - SSDataBlock* pRes = createOneDataBlock(pBlock, true); - if (ignoreCols) { - int32_t i = 0; - while(i < taosArrayGetSize(pRes->pDataBlock)) { - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); - for (int32_t j = 0; j < numOfMatchInfo; ++j) { - SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, j); - if (pInfo->colId == pCol->info.colId && !pInfo->output) { - taosArrayRemove(pBlock->pDataBlock, i); - i -= 1; - break; - } - } - - i += 1; - } - } - - pRes->info.rows = pBlock->info.rows; - return pRes; -} - -static SSDataBlock* doStreamScanImpl(SOperatorInfo* pOperator) { +static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamScanInfo* pInfo = pOperator->info; @@ -1479,16 +1444,6 @@ static SSDataBlock* doStreamScanImpl(SOperatorInfo* pOperator) { } } -static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { - SSDataBlock* pBlock = doStreamScanImpl(pOperator); - if (pBlock != NULL) { - SStreamScanInfo* pInfo = (SStreamScanInfo*) pOperator->info; - return doDropInvisibleCol(pInfo->pRes, pInfo->pColMatchInfo); - } else { - return NULL; - } -} - static SSDataBlock* doRawScan(SOperatorInfo* pInfo) { // return NULL; From 5e754a7662e8efa77ee806eafb9b8d808e800095 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Jul 2022 14:00:45 +0800 Subject: [PATCH 44/65] fix(query): add check for deleting record --- source/dnode/vnode/src/tsdb/tsdbRead.c | 69 +++++++++++++++++++------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0b05bb7224..2801e09f06 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1453,38 +1453,71 @@ static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVer (pBlock->minVersion <= pVerRange->maxVer); } -static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) { - if (pBlockScanInfo->delSkyline == NULL) { - return false; - } - TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0); - TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline); - - // ts is not overlap - if (pBlock->minKey.ts > pLast->ts || pBlock->maxKey.ts < pFirst->ts) { - return false; - } - - int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; - - // version is not overlap +static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) { size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); - for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += step) { + + for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) { TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i); if (p->ts >= pBlock->minKey.ts && p->ts <= pBlock->maxKey.ts) { if (p->version >= pBlock->minVersion) { return true; } - } else if (p->ts > pBlock->maxKey.ts) { + } else if (p->ts < pBlock->minKey.ts) { // p->ts < pBlock->minKey.ts + if (p->version >= pBlock->minVersion) { + if (i < num - 1) { + TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1); + if (i + 1 == num - 1) { // pnext is the last point + if (pnext->ts >= pBlock->minKey.ts) { + return true; + } + } else { + if (pnext->ts >= pBlock->minKey.ts && pnext->version >= pBlock->minVersion) { + return true; + } + } + } else { // it must be the last point + ASSERT(p->version == 0); + } + } + } else { // (p->ts > pBlock->maxKey.ts) { return false; } } - ASSERT(0); return false; } +static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) { + if (pBlockScanInfo->delSkyline == NULL) { + return false; + } + + // ts is not overlap + TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0); + TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline); + if (pBlock->minKey.ts > pLast->ts || pBlock->maxKey.ts < pFirst->ts) { + return false; + } + + // version is not overlap + if (ASCENDING_TRAVERSE(order)) { + return doCheckforDatablockOverlap(pBlockScanInfo, pBlock); + } else { + int32_t index = pBlockScanInfo->fileDelIndex; + while(1) { + TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index); + if (p->ts > pBlock->minKey.ts && index > 0) { + index -= 1; + } else { // find the first point that is smaller than the minKey.ts of dataBlock. + break; + } + } + + return doCheckforDatablockOverlap(pBlockScanInfo, pBlock); + } +} + // 1. the version of all rows should be less than the endVersion // 2. current block should not overlap with next neighbor block // 3. current timestamp should not be overlap with each other From 36c19fc824f637cd7745e7ff8f4e5becb0d8defa Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 19 Jul 2022 10:39:05 +0800 Subject: [PATCH 45/65] fix:tag is too long if type is nchar in schemaless influxdb --- source/client/src/clientSml.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 49f26e2c24..078ecbb4db 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1217,6 +1217,9 @@ static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *c kv->value = value; kv->length = valueLen; if (isTag) { + if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } kv->type = TSDB_DATA_TYPE_NCHAR; } else { int32_t ret = smlParseValue(kv, msg); From 651ae5832e8a68f2333d060b41c3e8ef7b523871 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Jul 2022 10:46:47 +0800 Subject: [PATCH 46/65] refactor(sync): delete not exist log --- source/libs/sync/src/syncRaftLog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index edc01c9a05..7f905f7cb5 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -313,7 +313,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, err, err, errStr, sysErr, sysErrStr); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { - syncNodeEventLog(pData->pSyncNode, logBuf); + // syncNodeEventLog(pData->pSyncNode, logBuf); } else { syncNodeErrorLog(pData->pSyncNode, logBuf); } @@ -499,7 +499,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, err, err, errStr, sysErr, sysErrStr); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { - syncNodeEventLog(pData->pSyncNode, logBuf); + // syncNodeEventLog(pData->pSyncNode, logBuf); } else { syncNodeErrorLog(pData->pSyncNode, logBuf); } From e646c7304ca3a0ab2ac82b81b72e1a3699fdf562 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Tue, 19 Jul 2022 11:37:35 +0800 Subject: [PATCH 47/65] test: add debug case for TD-16209 --- tests/{pytest/insert => system-test/1-insert}/insert_drop.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{pytest/insert => system-test/1-insert}/insert_drop.py (100%) diff --git a/tests/pytest/insert/insert_drop.py b/tests/system-test/1-insert/insert_drop.py similarity index 100% rename from tests/pytest/insert/insert_drop.py rename to tests/system-test/1-insert/insert_drop.py From ca5e959a1e05fddf953e5e28d6451a5346ef5e80 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 19 Jul 2022 11:46:17 +0800 Subject: [PATCH 48/65] fix(query): fix derivative nan value TD-17511 --- source/libs/function/src/builtinsimpl.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4f829f2df1..9d40b15354 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5661,7 +5661,12 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { double r = ((v - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); if (pDerivInfo->ignoreNegative && r < 0) { } else { - colDataAppend(pOutput, pos, (const char*)&r, false); + if (isinf(r) || isnan(r)) { + colDataAppendNULL(pOutput, pos); + } else { + colDataAppend(pOutput, pos, (const char*)&r, false); + } + if (pTsOutput != NULL) { colDataAppendInt64(pTsOutput, pos, &tsList[i]); } @@ -5688,7 +5693,12 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { double r = ((pDerivInfo->prevValue - v) * pDerivInfo->tsWindow) / (pDerivInfo->prevTs - tsList[i]); if (pDerivInfo->ignoreNegative && r < 0) { } else { - colDataAppend(pOutput, pos, (const char*)&r, false); + if (isinf(r) || isnan(r)) { + colDataAppendNULL(pOutput, pos); + } else { + colDataAppend(pOutput, pos, (const char*)&r, false); + } + if (pTsOutput != NULL) { colDataAppendInt64(pTsOutput, pos, &pDerivInfo->prevTs); } From 83ac069beff626be3bb0cadda37d43cfca571df9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 19 Jul 2022 12:00:43 +0800 Subject: [PATCH 49/65] enh: If an 'object is creating' error is reported when creating a database, wait for the transaction to end then response --- source/dnode/mnode/impl/inc/mndDef.h | 43 +++++++++- source/dnode/mnode/impl/inc/mndPrivilege.h | 40 --------- source/dnode/mnode/impl/inc/mndTrans.h | 2 + source/dnode/mnode/impl/src/mndDb.c | 9 ++ source/dnode/mnode/impl/src/mndTrans.c | 98 +++++++++++++++++----- tests/script/jenkins/basic.txt | 4 +- tests/script/tsim/valgrind/checkError1.sim | 2 +- 7 files changed, 131 insertions(+), 67 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 5cf6e26ee8..977f33de49 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -35,6 +35,46 @@ extern "C" { #endif +typedef enum { + MND_OPER_CONNECT = 1, + MND_OPER_CREATE_ACCT, + MND_OPER_DROP_ACCT, + MND_OPER_ALTER_ACCT, + MND_OPER_CREATE_USER, + MND_OPER_DROP_USER, + MND_OPER_ALTER_USER, + MND_OPER_CREATE_BNODE, + MND_OPER_DROP_BNODE, + MND_OPER_CREATE_DNODE, + MND_OPER_DROP_DNODE, + MND_OPER_CONFIG_DNODE, + MND_OPER_CREATE_MNODE, + MND_OPER_DROP_MNODE, + MND_OPER_CREATE_QNODE, + MND_OPER_DROP_QNODE, + MND_OPER_CREATE_SNODE, + MND_OPER_DROP_SNODE, + MND_OPER_REDISTRIBUTE_VGROUP, + MND_OPER_MERGE_VGROUP, + MND_OPER_SPLIT_VGROUP, + MND_OPER_BALANCE_VGROUP, + MND_OPER_CREATE_FUNC, + MND_OPER_DROP_FUNC, + MND_OPER_KILL_TRANS, + MND_OPER_KILL_CONN, + MND_OPER_KILL_QUERY, + MND_OPER_CREATE_DB, + MND_OPER_ALTER_DB, + MND_OPER_DROP_DB, + MND_OPER_COMPACT_DB, + MND_OPER_TRIM_DB, + MND_OPER_USE_DB, + MND_OPER_WRITE_DB, + MND_OPER_READ_DB, + MND_OPER_READ_OR_WRITE_DB, + MND_OPER_SHOW_VARIBALES, +} EOperType; + typedef enum { MND_AUTH_ACCT_START = 0, MND_AUTH_ACCT_USER, @@ -109,9 +149,9 @@ typedef struct { ETrnPolicy policy; ETrnConflct conflict; ETrnExec exec; + EOperType oper; int32_t code; int32_t failedTimes; - SRpcHandleInfo rpcInfo; void* rpcRsp; int32_t rpcRspLen; int32_t redoActionPos; @@ -130,6 +170,7 @@ typedef struct { int32_t stopFunc; int32_t paramLen; void* param; + SArray* pRpcArray; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndPrivilege.h b/source/dnode/mnode/impl/inc/mndPrivilege.h index a149c0f0e2..f6002e3be8 100644 --- a/source/dnode/mnode/impl/inc/mndPrivilege.h +++ b/source/dnode/mnode/impl/inc/mndPrivilege.h @@ -22,46 +22,6 @@ extern "C" { #endif -typedef enum { - MND_OPER_CONNECT = 1, - MND_OPER_CREATE_ACCT, - MND_OPER_DROP_ACCT, - MND_OPER_ALTER_ACCT, - MND_OPER_CREATE_USER, - MND_OPER_DROP_USER, - MND_OPER_ALTER_USER, - MND_OPER_CREATE_BNODE, - MND_OPER_DROP_BNODE, - MND_OPER_CREATE_DNODE, - MND_OPER_DROP_DNODE, - MND_OPER_CONFIG_DNODE, - MND_OPER_CREATE_MNODE, - MND_OPER_DROP_MNODE, - MND_OPER_CREATE_QNODE, - MND_OPER_DROP_QNODE, - MND_OPER_CREATE_SNODE, - MND_OPER_DROP_SNODE, - MND_OPER_REDISTRIBUTE_VGROUP, - MND_OPER_MERGE_VGROUP, - MND_OPER_SPLIT_VGROUP, - MND_OPER_BALANCE_VGROUP, - MND_OPER_CREATE_FUNC, - MND_OPER_DROP_FUNC, - MND_OPER_KILL_TRANS, - MND_OPER_KILL_CONN, - MND_OPER_KILL_QUERY, - MND_OPER_CREATE_DB, - MND_OPER_ALTER_DB, - MND_OPER_DROP_DB, - MND_OPER_COMPACT_DB, - MND_OPER_TRIM_DB, - MND_OPER_USE_DB, - MND_OPER_WRITE_DB, - MND_OPER_READ_DB, - MND_OPER_READ_OR_WRITE_DB, - MND_OPER_SHOW_VARIBALES, -} EOperType; - int32_t mndInitPrivilege(SMnode *pMnode); void mndCleanupPrivilege(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 6e367fbe24..faf656a251 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -73,12 +73,14 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen); void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2); void mndTransSetSerial(STrans *pTrans); +void mndTransSetOper(STrans *pTrans, EOperType oper); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); int32_t mndTransProcessRsp(SRpcMsg *pRsp); void mndTransPullup(SMnode *pMnode); int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); void mndTransExecute(SMnode *pMnode, STrans *pTrans); +int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index b4b1b383e7..064ef9b40a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -487,6 +487,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); mndTransSetDbName(pTrans, dbObj.name, NULL); + mndTransSetOper(pTrans, MND_OPER_CREATE_DB); if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; @@ -534,6 +535,14 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { terrno = TSDB_CODE_MND_DB_ALREADY_EXIST; goto _OVER; } + } else if (terrno == TSDB_CODE_SDB_OBJ_CREATING) { + if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { + mDebug("db:%s, is creating and response after trans finished", createReq.db); + code = TSDB_CODE_ACTION_IN_PROGRESS; + goto _OVER; + } else { + goto _OVER; + } } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index ea92b2f0e6..679f8085bd 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -122,6 +122,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT8(pRaw, dataPos, pTrans->policy, _OVER) SDB_SET_INT8(pRaw, dataPos, pTrans->conflict, _OVER) SDB_SET_INT8(pRaw, dataPos, pTrans->exec, _OVER) + SDB_SET_INT8(pRaw, dataPos, pTrans->oper, _OVER) + SDB_SET_INT8(pRaw, dataPos, 0, _OVER) + SDB_SET_INT8(pRaw, dataPos, 0, _OVER) + SDB_SET_INT8(pRaw, dataPos, 0, _OVER) SDB_SET_INT64(pRaw, dataPos, pTrans->createdTime, _OVER) SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER) @@ -269,15 +273,22 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { int8_t policy = 0; int8_t conflict = 0; int8_t exec = 0; + int8_t oper = 0; + int8_t reserved = 0; int8_t actionType = 0; SDB_GET_INT8(pRaw, dataPos, &stage, _OVER) SDB_GET_INT8(pRaw, dataPos, &policy, _OVER) SDB_GET_INT8(pRaw, dataPos, &conflict, _OVER) SDB_GET_INT8(pRaw, dataPos, &exec, _OVER) + SDB_GET_INT8(pRaw, dataPos, &oper, _OVER) + SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER) + SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER) + SDB_GET_INT8(pRaw, dataPos, &reserved, _OVER) pTrans->stage = stage; pTrans->policy = policy; pTrans->conflict = conflict; pTrans->exec = exec; + pTrans->oper = oper; SDB_GET_INT64(pRaw, dataPos, &pTrans->createdTime, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname1, TSDB_DB_FNAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pTrans->dbname2, TSDB_DB_FNAME_LEN, _OVER) @@ -495,6 +506,10 @@ static void mndTransDropData(STrans *pTrans) { mndTransDropActions(pTrans->commitActions); pTrans->commitActions = NULL; } + if (pTrans->pRpcArray != NULL) { + taosArrayDestroy(pTrans->pRpcArray); + pTrans->pRpcArray = NULL; + } if (pTrans->rpcRsp != NULL) { taosMemoryFree(pTrans->rpcRsp); pTrans->rpcRsp = NULL; @@ -585,14 +600,18 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); - if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL) { + if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || + pTrans->pRpcArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to create transaction since %s", terrstr()); return NULL; } - if (pReq != NULL) pTrans->rpcInfo = pReq->info; + if (pReq != NULL) { + taosArrayPush(pTrans->pRpcArray, &pReq->info); + } mTrace("trans:%d, local object is created, data:%p", pTrans->id, pTrans); return pTrans; } @@ -677,6 +696,31 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void * pTrans->paramLen = paramLen; } +int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, const char *dbname) { + STrans *pTrans = NULL; + void *pIter = NULL; + int32_t code = -1; + + while (1) { + pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); + if (pIter == NULL) break; + + if (pTrans->oper == oper) { + if (strcasecmp(dbname, pTrans->dbname1) == 0) { + mDebug("trans:%d, db:%s oper:%d matched with input", pTrans->id, dbname, oper); + if (taosArrayPush(pTrans->pRpcArray, &pMsg->info) != NULL) { + code = 0; + } + sdbRelease(pMnode->pSdb, pTrans); + break; + } + } + + sdbRelease(pMnode->pSdb, pTrans); + } + return code; +} + void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2) { if (dbname1 != NULL) { memcpy(pTrans->dbname1, dbname1, TSDB_DB_FNAME_LEN); @@ -688,6 +732,8 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname1, const char *dbname2) void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; } +void mndTransSetOper(STrans *pTrans, EOperType oper) { pTrans->oper = oper; } + static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { SSdbRaw *pRaw = mndTransActionEncode(pTrans); if (pRaw == NULL) { @@ -711,7 +757,7 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { static bool mndCheckDbConflict(const char *db, STrans *pTrans) { if (db[0] == 0) return false; - if (strcmp(db, pTrans->dbname1) == 0 || strcmp(db, pTrans->dbname2) == 0) return true; + if (strcasecmp(db, pTrans->dbname1) == 0 || strcasecmp(db, pTrans->dbname2) == 0) return true; return false; } @@ -784,9 +830,10 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { return -1; } - pNew->rpcInfo = pTrans->rpcInfo; + pNew->pRpcArray = pTrans->pRpcArray; pNew->rpcRsp = pTrans->rpcRsp; pNew->rpcRspLen = pTrans->rpcRspLen; + pTrans->pRpcArray = NULL; pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; @@ -835,29 +882,34 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } } - if (sendRsp && pTrans->rpcInfo.handle != NULL) { - mDebug("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage), - pTrans->rpcInfo.ahandle); - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; - } - SRpcMsg rspMsg = {.code = code, .info = pTrans->rpcInfo}; + if (!sendRsp) return; - if (pTrans->rpcRspLen != 0) { - void *rpcCont = rpcMallocCont(pTrans->rpcRspLen); - if (rpcCont != NULL) { - memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); - rspMsg.pCont = rpcCont; - rspMsg.contLen = pTrans->rpcRspLen; + int32_t size = taosArrayGetSize(pTrans->pRpcArray); + if (size <= 0) return; + + for (int32_t i = 0; i < size; ++i) { + SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i); + if (pInfo->handle != NULL) { + mDebug("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage), + pInfo->ahandle); + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } - taosMemoryFree(pTrans->rpcRsp); - } + SRpcMsg rspMsg = {.code = code, .info = *pInfo}; - tmsgSendRsp(&rspMsg); - pTrans->rpcInfo.handle = NULL; - pTrans->rpcRsp = NULL; - pTrans->rpcRspLen = 0; + if (pTrans->rpcRspLen != 0) { + void *rpcCont = rpcMallocCont(pTrans->rpcRspLen); + if (rpcCont != NULL) { + memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); + rspMsg.pCont = rpcCont; + rspMsg.contLen = pTrans->rpcRspLen; + } + } + + tmsgSendRsp(&rspMsg); + } } + taosArrayClear(pTrans->pRpcArray); } int32_t mndTransProcessRsp(SRpcMsg *pRsp) { diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index bff6177ad2..1b228925b1 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -248,8 +248,8 @@ ./test.sh -f tsim/stream/sliding.sim # ---- transaction - ./test.sh -f tsim/trans/lossdata1.sim - ./test.sh -f tsim/trans/create_db.sim +./test.sh -f tsim/trans/lossdata1.sim +./test.sh -f tsim/trans/create_db.sim # ---- tmq ./test.sh -f tsim/tmq/basic1.sim diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index 76a29ee62f..59eede7c91 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -152,7 +152,7 @@ endi system_content sh/checkValgrind.sh -n dnode2 print cmd return result ----> [ $system_content ] -if $system_content > 4 then +if $system_content > 0 then return -1 endi From ce999711ec00bd4205fd319cd688e68ac98f6f8c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 19 Jul 2022 13:07:24 +0800 Subject: [PATCH 50/65] test: add case --- tests/script/tmp/data.sim | 125 ++++++++++++++++++++-- tests/script/tmp/monitor.sim | 21 +--- tests/script/tsim/db/alter_replica_31.sim | 9 -- 3 files changed, 121 insertions(+), 34 deletions(-) diff --git a/tests/script/tmp/data.sim b/tests/script/tmp/data.sim index 4e714c01ee..dcfa02e0a7 100644 --- a/tests/script/tmp/data.sim +++ b/tests/script/tmp/data.sim @@ -1,18 +1,125 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c telemetryReporting -v 1 -system sh/cfg.sh -n dnode1 -c telemetryInterval -v 1 -system sh/cfg.sh -n dnode1 -c telemetryServer -v localhost -system sh/cfg.sh -n dnode1 -c telemetryPort -v 80 - -return +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start sql connect -sql create database db -sql create table db.tb (ts timestamp, i int) -sql insert into db.tb values(now, 1) +print =============== step1: create dnodes +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data24 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create database +sql create database db vgroups 1 replica 3 +sql show databases +if $rows != 3 then + return -1 +endi +if $data(db)[4] != 3 then + return -1 +endi + +sql show dnodes +if $data(2)[2] != 1 then + return -1 +endi +if $data(3)[2] != 1 then + return -1 +endi +if $data(4)[2] != 1 then + return -1 +endi + +# vnodes +sql show dnodes +if $data(2)[2] != 1 then + return -1 +endi +if $data(3)[2] != 1 then + return -1 +endi +if $data(4)[2] != 1 then + return -1 +endi + +# v1_dnode +$hasleader = 0 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 20 then + print ====> dnode not ready! + return -1 + endi +sql show db.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 +if $data(2)[3] != 2 then + return -1 +endi +if $data(2)[5] != 3 then + return -1 +endi +if $data(2)[7] != 4 then + return -1 +endi +if $data(2)[4] == leader then + $hasleader = 1 +endi +if $data(2)[6] == leader then + $hasleader = 1 +endi +if $data(2)[8] == leader then + $hasleader = 1 +endi +if $hasleader != 1 then + goto step2 +endi + +sql use db; +sql create table stb (ts timestamp, c int) tags (t int); +sql create table t0 using stb tags (0); +sql insert into t0 values(now, 1); +sql show db.stables; +sql show db.tables; +sql show db.vgroups; + +return print ======== start back run_back tmp/back.sim diff --git a/tests/script/tmp/monitor.sim b/tests/script/tmp/monitor.sim index 036d4cc6db..ec728fe733 100644 --- a/tests/script/tmp/monitor.sim +++ b/tests/script/tmp/monitor.sim @@ -6,32 +6,21 @@ system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 system sh/cfg.sh -n dnode1 -c monitorComp -v 1 #system sh/cfg.sh -n dnode1 -c supportVnodes -v 128 +#system sh/cfg.sh -n dnode1 -c telemetryReporting -v 1 +#system sh/cfg.sh -n dnode1 -c telemetryInterval -v 1 +#system sh/cfg.sh -n dnode1 -c telemetryServer -v localhost +#system sh/cfg.sh -n dnode1 -c telemetryPort -v 80 + system sh/exec.sh -n dnode1 -s start sql connect print =============== show dnodes -sleep 2000 sql create database db vgroups 2; sql use db; sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd"; -sleep 2000 print =============== create drop qnode 1 sql create qnode on dnode 1 sql create snode on dnode 1 sql create bnode on dnode 1 -return -print =============== restart -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode1 -s start - - -return -system sh/deploy.sh -n dnode2 -i 2 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s start - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim index 1823f182c9..e9a295820c 100644 --- a/tests/script/tsim/db/alter_replica_31.sim +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -111,15 +111,6 @@ if $hasleader != 1 then goto step2 endi -# sql use db; -# sql create table stb (ts timestamp, c int) tags (t int); -# sql create table t0 using stb tags (0); -# sql insert into t0 values(now, 1); -# sql show db.stables; -# sql show db.tables; -# sql show db.vgroups; -return - sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd" sql create table db.ctb using db.stb tags(101, "102") sql insert into db.ctb values(now, 1, "2") From 007c5eb0a510e1cd747a224087a43e536e609f61 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 19 Jul 2022 13:27:00 +0800 Subject: [PATCH 51/65] chore: libtaos ws submodule for3.0 (#15105) * chore: add libtaos-ws for 3.0 * chore: update taosws-rs * chore: add libtaosws to install/remove script * chore: update taosws-rs * chore: update taosws-rs * chore: update taos-tools, taosws-rs for 3.0 * fix: packaging/tools/make_install.sh for 3.0 * chore: update taos-tools * chore: fix release script for 3.0 * chore: update taosws-rs for 3.0 * chore: add taows-rs submodule for 3.0 * chore: update taosws-rs for 3.0 * fix: install script support taosws for 3.0 --- packaging/deb/makedeb.sh | 2 +- packaging/rpm/tdengine.spec | 2 +- packaging/tools/install.sh | 4 ++-- packaging/tools/install_client.sh | 12 ++++++++++++ packaging/tools/makeclient.sh | 7 +++++++ packaging/tools/makepkg.sh | 4 ++-- packaging/tools/post.sh | 11 +++++++++++ 7 files changed, 36 insertions(+), 6 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 1a4131ec6f..f09d46da59 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -73,7 +73,7 @@ cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_pat cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/util/taoserror.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/libs/function/taosudf.h ${pkg_dir}${install_home_path}/include -cp ${compile_dir}/../src/inc/taosws.h ${pkg_dir}${install_home_path}/include ||: +cp ${compile_dir}/build/include/taosws.h ${pkg_dir}${install_home_path}/include ||: cp -r ${top_dir}/examples/* ${pkg_dir}${install_home_path}/examples #cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector #cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 02e7e9b5e5..bf0921f18a 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -80,7 +80,7 @@ cp %{_compiledir}/../include/client/taos.h %{buildroot}%{homepath}/incl cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepath}/include -cp %{_compiledir}/../src/inc/taosws.h %{buildroot}%{homepath}/include ||: +cp %{_compiledir}/build/include/taosws.h %{buildroot}%{homepath}/include ||: #cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector #cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector #cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index f409834091..72ac76ec8e 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -229,13 +229,13 @@ function install_lib() { ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - ${csudo}ln -s ${lib_link_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so || : + [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : - ${csudo}ln -s ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : + ${csudo}ln -s ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi ${csudo}ldconfig diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 0c86877c99..a205206dda 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -116,6 +116,7 @@ function install_bin() { function clean_lib() { sudo rm -f /usr/lib/libtaos.* || : + sudo rm -f /usr/lib/libtaosws.* || : sudo rm -rf ${lib_dir} || : } @@ -123,6 +124,9 @@ function install_lib() { # Remove links ${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + + ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : #${csudo}rm -rf ${v15_java_app_dir} || : ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* @@ -131,13 +135,19 @@ function install_lib() { ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so + if [ -d "${lib64_link_dir}" ]; then ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + + [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi else ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib ${csudo}ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib + + [ -f ${install_main_dir}/driver/libtaosws.dylib ] && ${csudo}ln -s ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib fi if [ "$osType" != "Darwin" ]; then @@ -154,6 +164,8 @@ function install_header() { ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h + + [ -f ${install_main_dir}/include/taosws.h ] && ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taos.h } function install_jemalloc() { diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index ca3a9a19be..ae846ee493 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -57,12 +57,16 @@ if [ "$osType" != "Darwin" ]; then ${script_dir}/get_client.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" + wslib_files="${build_dir}/lib/libtaosws.so" else bin_files="${build_dir}/bin/${clientName} ${script_dir}/remove_client.sh" lib_files="${build_dir}/lib/libtaos.${version}.dylib" + wslib_files="${build_dir}/lib/libtaosws.dylib" fi header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" +wsheader_files="${build_dir}/include/taosws.h" + if [ "$dbName" != "taos" ]; then cfg_dir="${top_dir}/../enterprise/packaging/cfg" else @@ -74,6 +78,8 @@ install_files="${script_dir}/install_client.sh" # make directories. mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +[ -f ${wsheader_files} ] && cp ${wsheader_files} ${install_dir}/inc + mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* @@ -187,6 +193,7 @@ cp ${lib_files} ${install_dir}/driver # Copy connector connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector +[ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index c37397005c..f34adca9f8 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -96,7 +96,7 @@ lib_files="${build_dir}/lib/libtaos.so.${version}" wslib_files="${build_dir}/lib/libtaosws.so." header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" -wsheader_files="${code_dir}/inc/taosws.h" +wsheader_files="${build_dir}/include/taosws.h" if [ "$dbName" != "taos" ]; then cfg_dir="${top_dir}/../enterprise/packaging/cfg" @@ -114,7 +114,7 @@ init_file_rpm=${script_dir}/../rpm/taosd mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc -${wsheader_files} ${install_dir}/inc || : +[ -f ${wsheader_files} ] && cp ${wsheader_files} ${install_dir}/inc || : mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 2d744233ba..28163775a1 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -82,22 +82,33 @@ function kill_taosd() { function install_include() { ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : + ${csudo}rm -f ${inc_link_dir}/taosws.h + ${csudo}ln -s ${inc_dir}/taos.h ${inc_link_dir}/taos.h ${csudo}ln -s ${inc_dir}/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${inc_dir}/taoserror.h ${inc_link_dir}/taoserror.h ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h + + [ -f ${inc_dir}/taosws.h ] && ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h ||: } function install_lib() { ${csudo}rm -f ${lib_link_dir}/libtaos* || : ${csudo}rm -f ${lib64_link_dir}/libtaos* || : + ${csudo}rm -f ${lib_link_dir}/libtaosws* || : + ${csudo}rm -f ${lib64_link_dir}/libtaosws* || : + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + [ -f ${lib_dir}/libtaosws.so ]${csudo}ln -s ${lib_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + + [ -ff ${lib_dir}/libtaosws.so ] && ${csudo}ln -s ${lib_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi ${csudo}ldconfig From fc08a35fbff9eef43d4a68c0a51eba5ac9667f84 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Jul 2022 13:52:02 +0800 Subject: [PATCH 52/65] refactor(sync): update sync-timeout to 10s --- include/libs/sync/sync.h | 2 +- source/libs/sync/src/syncMain.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 06a58af0e3..510f816959 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -26,7 +26,7 @@ extern "C" { extern bool gRaftDetailLog; -#define SYNC_RESP_TTL_MS 5000 +#define SYNC_RESP_TTL_MS 10000 #define SYNC_MAX_BATCH_SIZE 500 #define SYNC_INDEX_BEGIN 0 diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ddb2b9355e..e11f610dd4 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1552,7 +1552,8 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { } else { snprintf(logBuf, sizeof(logBuf), "%s", str); } - sDebug("%s", logBuf); + // sDebug("%s", logBuf); + sInfo("%s", logBuf); } else { int len = 256 + userStrLen; @@ -1573,7 +1574,8 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { } else { snprintf(s, len, "%s", str); } - sDebug("%s", s); + // sDebug("%s", s); + sInfo("%s", s); taosMemoryFree(s); } From 8e41fc912be8da01d69d9272f91a982dcc61729b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 19 Jul 2022 14:17:53 +0800 Subject: [PATCH 53/65] feat(stream): add num of children --- include/libs/executor/executor.h | 1 + include/libs/stream/tstream.h | 1 + source/dnode/mnode/impl/src/mndScheduler.c | 5 +++++ source/dnode/vnode/src/tq/tq.c | 6 +++++- source/libs/executor/src/executorimpl.c | 4 ++-- source/libs/executor/src/scanoperator.c | 2 +- source/libs/stream/src/streamTask.c | 2 ++ 7 files changed, 17 insertions(+), 4 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 8b0a836ad2..9e28911784 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -40,6 +40,7 @@ typedef struct SReadHandle { bool initMetaReader; bool initTableReader; bool initTqReader; + int32_t numOfVgroups; } SReadHandle; // in queue mode, data streams are seperated by msg diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 8c69c0f2de..a484492d08 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -262,6 +262,7 @@ typedef struct SStreamTask { int64_t startVer; int64_t checkpointVer; int64_t processedVer; + int32_t numOfVgroups; // children info SArray* childEpInfo; // SArray diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index ec0ea90f46..a46938590e 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -383,6 +383,11 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { // exec pInnerTask->execType = TASK_EXEC__PIPE; + SDbObj* pSourceDb = mndAcquireDb(pMnode, pStream->sourceDb); + ASSERT(pDbObj != NULL); + sdbRelease(pSdb, pSourceDb); + pInnerTask->numOfVgroups = pSourceDb->cfg.numOfVgroups; + if (tsSchedStreamToSnode) { SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); if (pSnode == NULL) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index fb05aeecd9..1802f9be1a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -591,7 +591,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { }; pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); } else { - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL); + SReadHandle mgHandle = { + .vnode = NULL, + .numOfVgroups = pTask->numOfVgroups, + }; + pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle); } ASSERT(pTask->exec.executor); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 06bb096e59..774de10649 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4424,7 +4424,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN, }; - if (pHandle) { + if (pHandle->vnode) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { @@ -4590,7 +4590,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t children = 0; pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == type) { - int32_t children = 1; + int32_t children = pHandle->numOfVgroups; pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { pOptr = createSortOperatorInfo(ops[0], (SSortPhysiNode*)pPhyNode, pTaskInfo); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e60f6f8a5b..57ba08e8fa 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1539,7 +1539,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys goto _error; } - if (pHandle) { + if (pHandle->vnode) { SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info; if (pHandle->version > 0) { diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 2d15c31bf1..5921e44a9c 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -64,6 +64,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->numOfVgroups) < 0) return -1; int32_t epSz = taosArrayGetSize(pTask->childEpInfo); if (tEncodeI32(pEncoder, epSz) < 0) return -1; @@ -118,6 +119,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->numOfVgroups) < 0) return -1; int32_t epSz; if (tDecodeI32(pDecoder, &epSz) < 0) return -1; From 58135a1aeb3cb11d62ec35b516d9e306fd726efe Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Jul 2022 06:19:01 +0000 Subject: [PATCH 54/65] tsdb r/w concurrency --- source/dnode/vnode/src/inc/tsdb.h | 46 ++++++++--------- source/dnode/vnode/src/tsdb/tsdbCache.c | 8 +-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 44 ++++++++++++---- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 56 +++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbOpen.c | 32 ++---------- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +-- source/dnode/vnode/src/tsdb/tsdbRetention.c | 10 ++-- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 18 +++---- 8 files changed, 137 insertions(+), 83 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4e2c762cd3..eb5f66d443 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -174,6 +174,10 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg); int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); void tsdbMemTableDestroy(SMemTable *pMemTable); void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData); +void tsdbRefMemTable(SMemTable *pMemTable); +void tsdbUnrefMemTable(SMemTable *pMemTable); +int32_t tsdbTakeMemSnapshot(STsdb *pTsdb, SMemTable **ppMem, SMemTable **ppIMem); +void tsdbUntakeMemSnapshot(STsdb *pTsdb, SMemTable *pMem, SMemTable *pIMem); // STbDataIter int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); void *tsdbTbDataIterDestroy(STbDataIter *pIter); @@ -273,16 +277,14 @@ typedef struct { } SRtn; struct STsdb { - char *path; - SVnode *pVnode; - TdThreadMutex mutex; - bool repoLocked; - STsdbKeepCfg keepCfg; - SMemTable *mem; - SMemTable *imem; - SRtn rtn; - STsdbFS *fs; - SLRUCache *lruCache; + char *path; + SVnode *pVnode; + TdThreadRwlock rwLock; + STsdbKeepCfg keepCfg; + SMemTable *mem; + SMemTable *imem; + STsdbFS *pFS; + SLRUCache *lruCache; }; struct STable { @@ -330,21 +332,19 @@ struct STbData { }; struct SMemTable { - SRWLatch latch; - STsdb *pTsdb; - int32_t nRef; - TSKEY minKey; - TSKEY maxKey; - int64_t minVersion; - int64_t maxVersion; - int64_t nRow; - int64_t nDel; - SArray *aTbData; // SArray + SRWLatch latch; + STsdb *pTsdb; + SVBufPool *pPool; + volatile int32_t nRef; + TSKEY minKey; + TSKEY maxKey; + int64_t minVersion; + int64_t maxVersion; + int64_t nRow; + int64_t nDel; + SArray *aTbData; // SArray }; -int tsdbLockRepo(STsdb *pTsdb); -int tsdbUnlockRepo(STsdb *pTsdb); - struct TSDBROW { int8_t type; // 0 for row from tsRow, 1 for row from block data union { diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 723caca5d7..2ca52ef5b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -464,7 +464,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { switch (state->state) { case SFSNEXTROW_FS: - state->aDFileSet = state->pTsdb->fs->cState->aDFileSet; + state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet; state->nFileSet = taosArrayGetSize(state->aDFileSet); state->iFileSet = state->nFileSet; @@ -814,7 +814,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs SDelIdx delIdx; - SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState); + SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState); if (pDelFile) { SDelFReader *pDelFReader; @@ -1189,7 +1189,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo SDelIdx delIdx; - SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState); + SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState); if (pDelFile) { SDelFReader *pDelFReader; @@ -1377,7 +1377,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { SDelIdx delIdx; - SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState); + SDelFile *pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState); if (pDelFile) { SDelFReader *pDelFReader; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 470bff1eae..04fe5520fb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -64,9 +64,26 @@ int32_t tsdbBegin(STsdb *pTsdb) { if (!pTsdb) return code; - code = tsdbMemTableCreate(pTsdb, &pTsdb->mem); + SMemTable *pMemTable; + code = tsdbMemTableCreate(pTsdb, &pMemTable); if (code) goto _err; + // lock + code = taosThreadRwlockWrlock(&pTsdb->rwLock); + if (code) { + code = TAOS_SYSTEM_ERROR(code); + goto _err; + } + + pTsdb->mem = pMemTable; + + // unlock + code = taosThreadRwlockUnlock(&pTsdb->rwLock); + if (code) { + code = TAOS_SYSTEM_ERROR(code); + goto _err; + } + return code; _err: @@ -83,9 +100,11 @@ int32_t tsdbCommit(STsdb *pTsdb) { // check if (pMemTable->nRow == 0 && pMemTable->nDel == 0) { - // TODO: lock? + taosThreadRwlockWrlock(&pTsdb->rwLock); pTsdb->mem = NULL; - tsdbMemTableDestroy(pMemTable); + taosThreadRwlockUnlock(&pTsdb->rwLock); + + tsdbUnrefMemTable(pMemTable); goto _exit; } @@ -139,7 +158,7 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { goto _err; } - SDelFile *pDelFileR = pTsdb->fs->nState->pDelFile; + SDelFile *pDelFileR = pTsdb->pFS->nState->pDelFile; if (pDelFileR) { code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb, NULL); if (code) goto _err; @@ -228,7 +247,7 @@ static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter); if (code) goto _err; - code = tsdbFSStateUpsertDelFile(pTsdb->fs->nState, &pCommitter->pDelFWriter->fDel); + code = tsdbFSStateUpsertDelFile(pTsdb->pFS->nState, &pCommitter->pDelFWriter->fDel); if (code) goto _err; code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1); @@ -263,7 +282,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { taosArrayClear(pCommitter->aBlockIdx); tMapDataReset(&pCommitter->oBlockMap); tBlockDataReset(&pCommitter->oBlockData); - pRSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, pCommitter->commitFid, TD_EQ); + pRSet = tsdbFSStateGetDFileSet(pTsdb->pFS->nState, pCommitter->commitFid, TD_EQ); if (pRSet) { code = tsdbDataFReaderOpen(&pCommitter->pReader, pTsdb, pRSet); if (code) goto _err; @@ -836,7 +855,7 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { if (code) goto _err; // upsert SDFileSet - code = tsdbFSStateUpsertDFileSet(pCommitter->pTsdb->fs->nState, tsdbDataFWriterGetWSet(pCommitter->pWriter)); + code = tsdbFSStateUpsertDFileSet(pCommitter->pTsdb->pFS->nState, tsdbDataFWriterGetWSet(pCommitter->pWriter)); if (code) goto _err; // close and sync @@ -954,7 +973,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; - code = tsdbFSBegin(pTsdb->fs); + code = tsdbFSBegin(pTsdb->pFS); if (code) goto _err; return code; @@ -1135,13 +1154,16 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { SMemTable *pMemTable = pTsdb->imem; if (eno == 0) { - code = tsdbFSCommit(pTsdb->fs); + code = tsdbFSCommit(pTsdb->pFS); } else { - code = tsdbFSRollback(pTsdb->fs); + code = tsdbFSRollback(pTsdb->pFS); } - tsdbMemTableDestroy(pMemTable); + taosThreadRwlockWrlock(&pTsdb->rwLock); pTsdb->imem = NULL; + taosThreadRwlockUnlock(&pTsdb->rwLock); + + tsdbUnrefMemTable(pMemTable); tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode)); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 82de931872..d0a719b5cd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -41,6 +41,7 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { } taosInitRWLatch(&pMemTable->latch); pMemTable->pTsdb = pTsdb; + pMemTable->pPool = pTsdb->pVnode->inUse; pMemTable->nRef = 1; pMemTable->minKey = TSKEY_MAX; pMemTable->maxKey = TSKEY_MIN; @@ -590,3 +591,58 @@ _err: } int32_t tsdbGetNRowsInTbData(STbData *pTbData) { return pTbData->sl.size; } + +void tsdbRefMemTable(SMemTable *pMemTable) { + int32_t nRef = atomic_fetch_add_32(&pMemTable->nRef, 1); + ASSERT(nRef > 0); +} + +void tsdbUnrefMemTable(SMemTable *pMemTable) { + int32_t nRef = atomic_sub_fetch_32(&pMemTable->nRef, 1); + if (nRef == 0) { + tsdbMemTableDestroy(pMemTable); + } +} + +int32_t tsdbTakeMemSnapshot(STsdb *pTsdb, SMemTable **ppMem, SMemTable **ppIMem) { + int32_t code = 0; + + // lock + code = taosThreadRwlockRdlock(&pTsdb->rwLock); + if (code) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + + // take snapshot + *ppMem = pTsdb->mem; + *ppIMem = pTsdb->imem; + + if (*ppMem) { + tsdbRefMemTable(*ppMem); + } + + if (*ppIMem) { + tsdbRefMemTable(*ppIMem); + } + + // unlock + code = taosThreadRwlockUnlock(&pTsdb->rwLock); + if (code) { + code = TAOS_SYSTEM_ERROR(code); + goto _exit; + } + +_exit: + return code; +} + +void tsdbUntakeMemSnapshot(STsdb *pTsdb, SMemTable *pMem, SMemTable *pIMem) { + if (pMem) { + tsdbUnrefMemTable(pMem); + } + + if (pIMem) { + tsdbUnrefMemTable(pIMem); + } +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 1fa582fae5..064c7adf4b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -54,8 +54,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee sprintf(pTsdb->path, "%s%s%s", pVnode->path, TD_DIRSEP, dir); taosRealPath(pTsdb->path, NULL, slen); pTsdb->pVnode = pVnode; - pTsdb->repoLocked = false; - taosThreadMutexInit(&pTsdb->mutex, NULL); + taosThreadRwlockInit(&pTsdb->rwLock, NULL); if (!pKeepCfg) { tsdbSetKeepCfg(&pTsdb->keepCfg, &pVnode->config.tsdbCfg); } else { @@ -67,7 +66,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee tfsMkdir(pVnode->pTfs, pTsdb->path); // open tsdb - if (tsdbFSOpen(pTsdb, &pTsdb->fs) < 0) { + if (tsdbFSOpen(pTsdb, &pTsdb->pFS) < 0) { goto _err; } @@ -88,33 +87,10 @@ _err: int tsdbClose(STsdb **pTsdb) { if (*pTsdb) { - taosThreadMutexDestroy(&(*pTsdb)->mutex); - tsdbFSClose((*pTsdb)->fs); + taosThreadRwlockDestroy(&(*pTsdb)->rwLock); + tsdbFSClose((*pTsdb)->pFS); tsdbCloseCache((*pTsdb)->lruCache); taosMemoryFreeClear(*pTsdb); } return 0; } - -int tsdbLockRepo(STsdb *pTsdb) { - int code = taosThreadMutexLock(&pTsdb->mutex); - if (code != 0) { - tsdbError("vgId:%d, failed to lock tsdb since %s", TD_VID(pTsdb->pVnode), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - pTsdb->repoLocked = true; - return 0; -} - -int tsdbUnlockRepo(STsdb *pTsdb) { - // ASSERT(IS_REPO_LOCKED(pTsdb)); - pTsdb->repoLocked = false; - int code = taosThreadMutexUnlock(&pTsdb->mutex); - if (code != 0) { - tsdbError("vgId:%d, failed to unlock tsdb since %s", TD_VID(pTsdb->pVnode), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0b05bb7224..a09bc32c60 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1905,7 +1905,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* SArray* pDelData = taosArrayInit(4, sizeof(SDelData)); - SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->cState); + SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->cState); if (pDelFile) { SDelFReader* pDelFReader = NULL; code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb, NULL); @@ -2795,7 +2795,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl SDataBlockIter* pBlockIter = &pReader->status.blockIter; - STsdbFSState* pFState = pReader->pTsdb->fs->cState; + STsdbFSState* pFState = pReader->pTsdb->pFS->cState; initFilesetIterator(&pReader->status.fileIter, pFState, pReader->order, pReader->idStr); resetDataBlockIterator(&pReader->status.blockIter, pReader->order); @@ -3042,7 +3042,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { tsdbDataFReaderClose(&pReader->pFileReader); - STsdbFSState* pFState = pReader->pTsdb->fs->cState; + STsdbFSState* pFState = pReader->pTsdb->pFS->cState; initFilesetIterator(&pReader->status.fileIter, pFState, pReader->order, pReader->idStr); resetDataBlockIterator(&pReader->status.blockIter, pReader->order); resetDataBlockScanInfo(pReader->status.pTableMap); diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 2ae646a571..137ef9a4a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -20,10 +20,10 @@ static int32_t tsdbDoRetentionImpl(STsdb *pTsdb, int64_t now, int8_t try, int8_t STsdbFSState *pState; if (try) { - pState = pTsdb->fs->cState; + pState = pTsdb->pFS->cState; *canDo = 0; } else { - pState = pTsdb->fs->nState; + pState = pTsdb->pFS->nState; } for (int32_t iSet = 0; iSet < taosArrayGetSize(pState->aDFileSet); iSet++) { @@ -83,7 +83,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { if (!canDo) goto _exit; // begin - code = tsdbFSBegin(pTsdb->fs); + code = tsdbFSBegin(pTsdb->pFS); if (code) goto _err; // do retention @@ -91,7 +91,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { if (code) goto _err; // commit - code = tsdbFSCommit(pTsdb->fs); + code = tsdbFSCommit(pTsdb->pFS); if (code) goto _err; _exit: @@ -99,6 +99,6 @@ _exit: _err: tsdbError("vgId:%d tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - tsdbFSRollback(pTsdb->fs); + tsdbFSRollback(pTsdb->pFS); return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 4886d874a9..fea0254045 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -45,7 +45,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { while (true) { if (pReader->pDataFReader == NULL) { - SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->cState, pReader->fid, TD_GT); + SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->pFS->cState, pReader->fid, TD_GT); if (pSet == NULL) goto _exit; @@ -159,7 +159,7 @@ _err: static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; STsdb* pTsdb = pReader->pTsdb; - SDelFile* pDelFile = pTsdb->fs->cState->pDelFile; + SDelFile* pDelFile = pTsdb->pFS->cState->pDelFile; if (pReader->pDelFReader == NULL) { if (pDelFile == NULL) { @@ -798,7 +798,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW, NULL); if (code) goto _err; - code = tsdbFSStateUpsertDFileSet(pTsdb->fs->nState, tsdbDataFWriterGetWSet(pWriter->pDataFWriter)); + code = tsdbFSStateUpsertDFileSet(pTsdb->pFS->nState, tsdbDataFWriterGetWSet(pWriter->pDataFWriter)); if (code) goto _err; code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1); @@ -843,7 +843,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3 pWriter->fid = fid; // read - SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ); + SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->pFS->nState, fid, TD_EQ); if (pSet) { code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet); if (code) goto _err; @@ -907,7 +907,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 STsdb* pTsdb = pWriter->pTsdb; if (pWriter->pDelFWriter == NULL) { - SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->nState); + SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->pFS->nState); // reader if (pDelFile) { @@ -1017,7 +1017,7 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter); if (code) goto _err; - code = tsdbFSStateUpsertDelFile(pTsdb->fs->nState, &pWriter->pDelFWriter->fDel); + code = tsdbFSStateUpsertDelFile(pTsdb->pFS->nState, &pWriter->pDelFWriter->fDel); if (code) goto _err; code = tsdbDelFWriterClose(&pWriter->pDelFWriter, 1); @@ -1096,7 +1096,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr goto _err; } - code = tsdbFSBegin(pTsdb->fs); + code = tsdbFSBegin(pTsdb->pFS); if (code) goto _err; *ppWriter = pWriter; @@ -1113,7 +1113,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { STsdbSnapWriter* pWriter = *ppWriter; if (rollback) { - code = tsdbFSRollback(pWriter->pTsdb->fs); + code = tsdbFSRollback(pWriter->pTsdb->pFS); if (code) goto _err; } else { code = tsdbSnapWriteDataEnd(pWriter); @@ -1122,7 +1122,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { code = tsdbSnapWriteDelEnd(pWriter); if (code) goto _err; - code = tsdbFSCommit(pWriter->pTsdb->fs); + code = tsdbFSCommit(pWriter->pTsdb->pFS); if (code) goto _err; } From 6b454a4dd43844fb447436428f54af3c9698bb0b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Jul 2022 06:26:09 +0000 Subject: [PATCH 55/65] more --- source/dnode/vnode/src/inc/tsdb.h | 9 +-------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 ++-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index eb5f66d443..30a6188db0 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -279,21 +279,14 @@ typedef struct { struct STsdb { char *path; SVnode *pVnode; - TdThreadRwlock rwLock; STsdbKeepCfg keepCfg; + TdThreadRwlock rwLock; SMemTable *mem; SMemTable *imem; STsdbFS *pFS; SLRUCache *lruCache; }; -struct STable { - uint64_t suid; - uint64_t uid; - STSchema *pSchema; // latest schema - STSchema *pCacheSchema; // cached cache -}; - struct TSDBKEY { int64_t version; TSKEY ts; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 04fe5520fb..13f310ae27 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -960,10 +960,10 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { memset(pCommitter, 0, sizeof(*pCommitter)); ASSERT(pTsdb->mem && pTsdb->imem == NULL); - // lock(); + taosThreadRwlockWrlock(&pTsdb->rwLock); pTsdb->imem = pTsdb->mem; pTsdb->mem = NULL; - // unlock(); + taosThreadRwlockUnlock(&pTsdb->rwLock); pCommitter->pTsdb = pTsdb; pCommitter->commitID = pTsdb->pVnode->state.commitID; From 99fd74c74715b595f7e048d4f91b160d5c31a02f Mon Sep 17 00:00:00 2001 From: huolibo Date: Tue, 19 Jul 2022 10:01:56 +0800 Subject: [PATCH 56/65] docs(driver): release jdbc 2.0.40 --- docs/en/14-reference/03-connector/java.mdx | 23 +++++++++++++++------- docs/zh/14-reference/03-connector/java.mdx | 11 ++++++++++- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/docs/en/14-reference/03-connector/java.mdx b/docs/en/14-reference/03-connector/java.mdx index ff15acf1a9..625ffc2698 100644 --- a/docs/en/14-reference/03-connector/java.mdx +++ b/docs/en/14-reference/03-connector/java.mdx @@ -202,6 +202,10 @@ The configuration parameters in the URL are as follows. - batchfetch: true: pull the result set in batch when executing the query; false: pull the result set row by row. The default value is false. batchfetch uses HTTP for data transfer. The JDBC REST connection supports bulk data pulling function in taos-jdbcdriver-2.0.38 and TDengine 2.4.0.12 and later versions. taos-jdbcdriver and TDengine transfer data via WebSocket connection. Compared with HTTP, WebSocket enables JDBC REST connection to support large data volume querying and improve query performance. - charset: specify the charset to parse the string, this parameter is valid only when set batchfetch to true. - batchErrorIgnore: true: when executing executeBatch of Statement, if one SQL execution fails in the middle, continue to execute the following SQL. false: no longer execute any statement after the failed SQL. The default value is: false. +- httpConnectTimeout: REST connection timeout in milliseconds, the default value is 5000 ms. +- httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. only takes effect when batchfetch is false. +- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. only takes effect when batchfetch is true. +- useSSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. **Note**: Some configuration items (e.g., locale, timezone) do not work in the REST connection. @@ -257,14 +261,18 @@ In the above example, a connection is established to `taosdemo.com`, port is 603 The configuration parameters in properties are as follows. -- TSDBDriver.PROPERTY_KEY_USER: Login TDengine user name, default value 'root'. +- TSDBDriver.PROPERTY_KEY_USER: login TDengine user name, default value 'root'. - TSDBDriver.PROPERTY_KEY_PASSWORD: user login password, default value 'taosdata'. - TSDBDriver.PROPERTY_KEY_BATCH_LOAD: true: pull the result set in batch when executing query; false: pull the result set row by row. The default value is: false. - TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE: true: when executing executeBatch of Statement, if there is a SQL execution failure in the middle, continue to execute the following sq. false: no longer execute any statement after the failed SQL. The default value is: false. -- TSDBDriver.PROPERTY_KEY_CONFIG_DIR: Only works when using JDBC native connection. Client configuration file directory path, default value `/etc/taos` on Linux OS, default value `C:/TDengine/cfg` on Windows OS. +- TSDBDriver.PROPERTY_KEY_CONFIG_DIR: only works when using JDBC native connection. Client configuration file directory path, default value `/etc/taos` on Linux OS, default value `C:/TDengine/cfg` on Windows OS. - TSDBDriver.PROPERTY_KEY_CHARSET: In the character set used by the client, the default value is the system character set. - TSDBDriver.PROPERTY_KEY_LOCALE: this only takes effect when using JDBC native connection. Client language environment, the default value is system current locale. - TSDBDriver.PROPERTY_KEY_TIME_ZONE: only takes effect when using JDBC native connection. In the time zone used by the client, the default value is the system's current time zone. +- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 5000 ms. only takes effect when using JDBC REST connection. +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. only takes effect when using JDBC REST connection and batchfetch is false. +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. only takes effect when using JDBC REST connection and batchfetch is true. +- TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. only takes effect when using using JDBC REST connection. For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only). ### Priority of configuration parameters @@ -812,11 +820,12 @@ Please refer to: [JDBC example](https://github.com/taosdata/TDengine/tree/develo ## Recent update logs -| taos-jdbcdriver version | major changes | -| :---------------------: | :------------------------------------------: | -| 2.0.38 | JDBC REST connections add bulk pull function | -| 2.0.37 | Added support for json tags | -| 2.0.36 | Add support for schemaless writing | +| taos-jdbcdriver version | major changes | +| :---------------------: | :--------------------------------------------: | +| 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | +| 2.0.38 | JDBC REST connections add bulk pull function | +| 2.0.37 | Support json tags | +| 2.0.36 | Support schemaless writing | ## Frequently Asked Questions diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index ddab9e5f24..838fa2eff8 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -201,6 +201,10 @@ url 中的配置参数如下： - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。从 taos-jdbcdriver-2.0.38 和 TDengine 2.4.0.12 版本开始，JDBC REST 连接增加批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 +- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 +- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。 +- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 +- useSSL: 连接中是否使用 SSL。 **注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。 @@ -264,7 +268,11 @@ properties 中的配置参数如下： - TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。 - TSDBDriver.PROPERTY_KEY_LOCALE：仅在使用 JDBC 原生连接时生效。 客户端语言环境，默认值系统当前 locale。 - TSDBDriver.PROPERTY_KEY_TIME_ZONE：仅在使用 JDBC 原生连接时生效。 客户端使用的时区，默认值为系统当前时区。 -- 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。 +- TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间，单位 ms， 默认值为 5000。仅在 REST 连接时生效。 +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 5000。仅在 REST 连接且 batchfetch 设置为 false 时生效。 +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。 +- TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。 + 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。 ### 配置参数的优先级 @@ -809,6 +817,7 @@ Query OK, 1 row(s) in set (0.000141s) | taos-jdbcdriver 版本 | 主要变化 | | :------------------: | :----------------------------: | +| 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | | 2.0.38 | JDBC REST 连接增加批量拉取功能 | | 2.0.37 | 增加对 json tag 支持 | | 2.0.36 | 增加对 schemaless 写入支持 | From 4a7d48418542b8998f64eea0922d91aac6113b2a Mon Sep 17 00:00:00 2001 From: huolibo Date: Tue, 19 Jul 2022 14:27:17 +0800 Subject: [PATCH 57/65] docs(driver): improve description --- docs/en/14-reference/03-connector/java.mdx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/14-reference/03-connector/java.mdx b/docs/en/14-reference/03-connector/java.mdx index 625ffc2698..310e0a15c6 100644 --- a/docs/en/14-reference/03-connector/java.mdx +++ b/docs/en/14-reference/03-connector/java.mdx @@ -203,8 +203,8 @@ The configuration parameters in the URL are as follows. - charset: specify the charset to parse the string, this parameter is valid only when set batchfetch to true. - batchErrorIgnore: true: when executing executeBatch of Statement, if one SQL execution fails in the middle, continue to execute the following SQL. false: no longer execute any statement after the failed SQL. The default value is: false. - httpConnectTimeout: REST connection timeout in milliseconds, the default value is 5000 ms. -- httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. only takes effect when batchfetch is false. -- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. only takes effect when batchfetch is true. +- httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when batchfetch is false. +- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when batchfetch is true. - useSSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. **Note**: Some configuration items (e.g., locale, timezone) do not work in the REST connection. @@ -269,10 +269,10 @@ The configuration parameters in properties are as follows. - TSDBDriver.PROPERTY_KEY_CHARSET: In the character set used by the client, the default value is the system character set. - TSDBDriver.PROPERTY_KEY_LOCALE: this only takes effect when using JDBC native connection. Client language environment, the default value is system current locale. - TSDBDriver.PROPERTY_KEY_TIME_ZONE: only takes effect when using JDBC native connection. In the time zone used by the client, the default value is the system's current time zone. -- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 5000 ms. only takes effect when using JDBC REST connection. -- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. only takes effect when using JDBC REST connection and batchfetch is false. -- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. only takes effect when using JDBC REST connection and batchfetch is true. -- TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. only takes effect when using using JDBC REST connection. +- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection. +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection and batchfetch is false. +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when using JDBC REST connection and batchfetch is true. +- TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. It only takes effect when using using JDBC REST connection. For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only). ### Priority of configuration parameters From 329ba3cee87d0d604d3a8052b125f901d6fd8612 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 19 Jul 2022 14:34:39 +0800 Subject: [PATCH 58/65] refactor(sync): add trace log --- source/libs/sync/src/syncRespMgr.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 97e5816038..3c63b7692a 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -120,15 +120,18 @@ void syncRespClean(SSyncRespMgr *pObj) { void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) { SRespStub *pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, NULL); int cnt = 0; + int sum = 0; SSyncNode *pSyncNode = pObj->data; SArray *delIndexArray = taosArrayInit(0, sizeof(uint64_t)); ASSERT(delIndexArray != NULL); + sDebug("vgId:%d, resp mgr begin clean by ttl", pSyncNode->vgId); while (pStub) { size_t len; - void * key = taosHashGetKey(pStub, &len); + void *key = taosHashGetKey(pStub, &len); uint64_t *pSeqNum = (uint64_t *)key; + sum++; int64_t nowMS = taosGetTimestampMs(); if (nowMS - pStub->createTime > ttl) { @@ -155,7 +158,7 @@ void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) { } int32_t arraySize = taosArrayGetSize(delIndexArray); - sDebug("vgId:%d, resp mgr clean by ttl, cnt:%d, array-size:%d", pSyncNode->vgId, cnt, arraySize); + sDebug("vgId:%d, resp mgr end clean by ttl, sum:%d, cnt:%d, array-size:%d", pSyncNode->vgId, sum, cnt, arraySize); for (int32_t i = 0; i < arraySize; ++i) { uint64_t *pSeqNum = taosArrayGet(delIndexArray, i); From ed51c8e329aa8c23887813ddb012fe9a8a5f961f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 19 Jul 2022 14:54:18 +0800 Subject: [PATCH 59/65] fix(query): fix mode function processing all null column TD-17024 --- source/libs/function/src/builtinsimpl.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 9d40b15354..25648fdea4 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5098,12 +5098,7 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { return true; } -static void doModeAdd(SModeInfo* pInfo, char* data, bool isNull) { - // ignore null elements - if (isNull) { - return; - } - +static void doModeAdd(SModeInfo* pInfo, char* data) { int32_t hashKeyBytes = IS_VAR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); if (pHashItem == NULL) { @@ -5128,10 +5123,16 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { SColumnInfoData* pInputCol = pInput->pData[0]; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + int32_t numOfElems = 0; int32_t startOffset = pCtx->offset; for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { char* data = colDataGetData(pInputCol, i); - doModeAdd(pInfo, data, colDataIsNull_s(pInputCol, i)); + if (colDataIsNull_s(pInputCol, i)) { + continue; + } + + numOfElems++; + doModeAdd(pInfo, data); if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) { taosHashCleanup(pInfo->pHash); @@ -5139,7 +5140,7 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { } } - SET_VAL(pResInfo, 1, 1); + SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } From 622ab28e6e967cea8417a6cd6262e5b49839ac09 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 19 Jul 2022 14:59:07 +0800 Subject: [PATCH 60/65] os: backend run fail error --- source/common/src/tglobal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c4f3ca0bdf..fcc27e440c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -556,7 +556,7 @@ static void taosSetSystemCfg(SConfig *pCfg) { osSetSystemLocale(locale, charset); bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; - taosSetConsoleEcho(enableCore); + taosSetCoreDump(enableCore); // todo tsVersion = 30000000; @@ -675,7 +675,7 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { case 'e': { if (strcasecmp("enableCoreFile", name) == 0) { bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; - taosSetConsoleEcho(enableCore); + taosSetCoreDump(enableCore); } break; } From 657c2dcaa4f10f97e3a9ba67bbbe8db6cf7eb31d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Jul 2022 07:21:15 +0000 Subject: [PATCH 61/65] more r/w concurrency --- source/dnode/vnode/src/tsdb/tsdbCache.c | 17 +++++++++++++---- source/dnode/vnode/src/tsdb/tsdbRead.c | 14 ++++++++++---- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 2ca52ef5b3..484020e6e1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -793,6 +793,9 @@ typedef struct { TSDBROW memRow, imemRow, fsRow; TsdbNextRowState input[3]; + SMemTable *pMemTable; + SMemTable *pIMemTable; + STsdb *pTsdb; } CacheNextRowIter; static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb) { @@ -800,16 +803,20 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs tb_uid_t suid = getTableSuidByUid(uid, pTsdb); + tsdbTakeMemSnapshot(pTsdb, &pIter->pMemTable, &pIter->pIMemTable); + STbData *pMem = NULL; - if (pTsdb->mem) { - tsdbGetTbDataFromMemTable(pTsdb->mem, suid, uid, &pMem); + if (pIter->pMemTable) { + tsdbGetTbDataFromMemTable(pIter->pMemTable, suid, uid, &pMem); } STbData *pIMem = NULL; - if (pTsdb->imem) { - tsdbGetTbDataFromMemTable(pTsdb->imem, suid, uid, &pIMem); + if (pIter->pIMemTable) { + tsdbGetTbDataFromMemTable(pIter->pIMemTable, suid, uid, &pIMem); } + pIter->pTsdb = pTsdb; + pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); SDelIdx delIdx; @@ -878,6 +885,8 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) { taosArrayDestroy(pIter->pSkyline); } + tsdbUntakeMemSnapshot(pIter->pTsdb, pIter->pMemTable, pIter->pIMemTable); + return code; _err: return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a09bc32c60..d605f564a5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -118,6 +118,8 @@ struct STsdbReader { char* idStr; // query info handle, for debug purpose int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows SBlockLoadSuppInfo suppInfo; + SMemTable* pMem; + SMemTable* pIMem; SIOCostSummary cost; STSchema* pSchema; @@ -1847,8 +1849,8 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea int32_t backward = (!ASCENDING_TRAVERSE(pReader->order)); STbData* d = NULL; - if (pReader->pTsdb->mem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pTsdb->mem, pReader->suid, pBlockScanInfo->uid, &d); + if (pReader->pMem != NULL) { + tsdbGetTbDataFromMemTable(pReader->pMem, pReader->suid, pBlockScanInfo->uid, &d); if (d != NULL) { code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); if (code == TSDB_CODE_SUCCESS) { @@ -1868,8 +1870,8 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea } STbData* di = NULL; - if (pReader->pTsdb->imem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pTsdb->imem, pReader->suid, pBlockScanInfo->uid, &di); + if (pReader->pIMem != NULL) { + tsdbGetTbDataFromMemTable(pReader->pIMem, pReader->suid, pBlockScanInfo->uid, &di); if (di != NULL) { code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter); if (code == TSDB_CODE_SUCCESS) { @@ -2809,6 +2811,8 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl } } + tsdbTakeMemSnapshot(pReader->pTsdb, &pReader->pMem, &pReader->pIMem); + tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; @@ -2824,6 +2828,8 @@ void tsdbReaderClose(STsdbReader* pReader) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + tsdbUntakeMemSnapshot(pReader->pTsdb, pReader->pMem, pReader->pIMem); + taosMemoryFreeClear(pSupInfo->plist); taosMemoryFree(pSupInfo->colIds); From 0d33e7363faa355ea500bfa4397844aa9677fe18 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 19 Jul 2022 15:41:31 +0800 Subject: [PATCH 62/65] fix: read sdb when restore from raft module may cause crash --- source/dnode/mnode/sdb/src/sdbHash.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 71792a2354..c579f82a9d 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -153,23 +153,22 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * return terrno; } - taosThreadRwlockUnlock(pLock); - int32_t code = 0; SdbInsertFp insertFp = pSdb->insertFps[pRow->type]; if (insertFp != NULL) { code = (*insertFp)(pSdb, pRow->pObj); if (code != 0) { code = terrno; - taosThreadRwlockWrlock(pLock); taosHashRemove(hash, pRow->pObj, keySize); - taosThreadRwlockUnlock(pLock); sdbFreeRow(pSdb, pRow, false); terrno = code; + taosThreadRwlockUnlock(pLock); return terrno; } } + taosThreadRwlockUnlock(pLock); + if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) { pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); } @@ -194,7 +193,6 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SSdbRow *pOldRow = *ppOldRow; pOldRow->status = pRaw->status; sdbPrintOper(pSdb, pOldRow, "update"); - taosThreadRwlockUnlock(pLock); int32_t code = 0; SdbUpdateFp updateFp = pSdb->updateFps[pNewRow->type]; @@ -202,6 +200,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); } + taosThreadRwlockUnlock(pLock); sdbFreeRow(pSdb, pNewRow, false); pSdb->tableVer[pOldRow->type]++; From c13e638de775f8794c34640dc46a0d1231ad07cb Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Tue, 19 Jul 2022 16:04:13 +0800 Subject: [PATCH 63/65] Delete stbTagFilter.py --- tests/system-test/7-tmq/stbTagFilter.py | 260 ------------------------ 1 file changed, 260 deletions(-) delete mode 100644 tests/system-test/7-tmq/stbTagFilter.py diff --git a/tests/system-test/7-tmq/stbTagFilter.py b/tests/system-test/7-tmq/stbTagFilter.py deleted file mode 100644 index 1bb3d24bde..0000000000 --- a/tests/system-test/7-tmq/stbTagFilter.py +++ /dev/null @@ -1,260 +0,0 @@ - -import taos -import sys -import time -import socket -import os -import threading -from enum import Enum - -from util.log import * -from util.sql import * -from util.cases import * -from util.dnodes import * -sys.path.append("./7-tmq") -from tmqCommon import * - -class TDTestCase: - def __init__(self): - self.snapshot = 0 - self.vgroups = 4 - self.ctbNum = 1 - self.rowsPerTbl = 10000 - - def init(self, conn, logSql): - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) - - def prepareTestEnv(self): - tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 4, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 1, - 'rowsPerTbl': 100000, - 'batchNum': 1200, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 3, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdLog.info("create stb") - tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) - tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", - # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - - # tdLog.info("restart taosd to ensure that the data falls into the disk") - # tdSql.query("flush database %s"%(paraDict['dbName'])) - return - - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 4, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 1, - 'rowsPerTbl': 100000, - 'batchNum': 3000, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - paraDict['snapshot'] = self.snapshot - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - # update to half tables - # paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) - # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", - # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - - tdLog.info("create topics from stb1") - topicFromStb1 = 'topic_stb1' - queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicFromStb1, queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - - # paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - consumerId = 0 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) - topicList = topicFromStb1 - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1,\ - enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ - auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - - tdLog.info("insert process end, and start to check consume result") - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - totalConsumeRows = 0 - for i in range(expectRows): - totalConsumeRows += resultList[i] - - tdLog.info("run select sql from db") - tdSql.query(queryString) - expectrowcnt = tdSql.getRows() - - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) - if totalConsumeRows != expectrowcnt: - tdLog.exit("tmq consume rows error!") - - tmqCom.checkFileContent(consumerId, queryString) - - tdSql.query("drop topic %s"%topicFromStb1) - tdLog.printNoPrefix("======== test case 1 end ...... ") - - def tmqCase2(self): - tdLog.printNoPrefix("======== test case 2: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 4, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 1, - 'rowsPerTbl': 10000, - 'batchNum': 5000, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['snapshot'] = self.snapshot - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - tdLog.info("restart taosd to ensure that the data falls into the disk") - tdSql.query("flush database %s"%(paraDict['dbName'])) - - # update to half tables - paraDict['startTs'] = paraDict['startTs'] + int(self.rowsPerTbl / 2) - paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) - tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - - tmqCom.initConsumerTable() - tdLog.info("create topics from stb1") - topicFromStb1 = 'topic_stb1' - queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicFromStb1, queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - - # paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - consumerId = 1 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) - topicList = topicFromStb1 - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1,\ - enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ - auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - - tdLog.info("insert process end, and start to check consume result") - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - totalConsumeRows = 0 - for i in range(expectRows): - totalConsumeRows += resultList[i] - - tdSql.query(queryString) - totalRowsInserted = tdSql.getRows() - - tdLog.info("act consume rows: %d, act insert rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsInserted, expectrowcnt)) - - if totalConsumeRows != expectrowcnt: - tdLog.exit("tmq consume rows error!") - - # tmqCom.checkFileContent(consumerId, queryString) - - tdSql.query("drop topic %s"%topicFromStb1) - - tdLog.printNoPrefix("======== test case 2 end ...... ") - - def run(self): - tdSql.prepare() - self.prepareTestEnv() - tdLog.printNoPrefix("=============================================") - tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") - self.tmqCase1() - # self.tmqCase2() - - # self.prepareTestEnv() - # tdLog.printNoPrefix("====================================================================") - # tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") - # self.snapshot = 1 - # self.tmqCase1() - # self.tmqCase2() - - - def stop(self): - tdSql.close() - tdLog.success(f"{__file__} successfully executed") - -event = threading.Event() - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) From dc76f3b935adfdec93e805c7241e949f841586e7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 19 Jul 2022 16:25:22 +0800 Subject: [PATCH 64/65] test: add case for drop dnode --- tests/script/jenkins/basic.txt | 1 + tests/script/tsim/dnode/use_dropped_dnode.sim | 133 ++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 tests/script/tsim/dnode/use_dropped_dnode.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1b228925b1..90353ef114 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -56,6 +56,7 @@ # unsupport ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim # unsupport ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim # unsupport ./test.sh -f tsim/dnode/vnode_clean.sim +./test.sh -f tsim/dnode/use_dropped_dnode.sim # ---- import ./test.sh -f tsim/import/basic.sim diff --git a/tests/script/tsim/dnode/use_dropped_dnode.sim b/tests/script/tsim/dnode/use_dropped_dnode.sim new file mode 100644 index 0000000000..9c546e295f --- /dev/null +++ b/tests/script/tsim/dnode/use_dropped_dnode.sim @@ -0,0 +1,133 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sql connect + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 + +$x = 0 +step1: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi + +print =============== step2 drop dnode 3 +sql drop dnode 3 +sql create dnode $hostname port 7300 + +$x = 0 +step2: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != null then + goto step2 +endi +if $data(4)[4] != offline then + goto step2 +endi + +print =============== step3: create dnode 4 +sleep 1000 +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/deploy.sh -n dnode3 -i 3 +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step3: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step3 +endi +if $data(2)[4] != ready then + goto step3 +endi +if $data(3)[4] != null then + goto step3 +endi +if $data(4)[4] != ready then + goto step3 +endi + +print =============== step4: create mnode 4 +sql create mnode on dnode 4 + +$x = 0 +step4: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show mnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[2] != leader then + goto step4 +endi +if $data(4)[2] != follower then + goto step4 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT From 706b081f700cc1f06ef03bb79f24e9a967e3cc28 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Jul 2022 08:30:49 +0000 Subject: [PATCH 65/65] fix: r/w concurrency --- source/dnode/vnode/src/inc/vnd.h | 15 ++++---- source/dnode/vnode/src/inc/vnodeInt.h | 42 +++++++++++----------- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 ++ source/dnode/vnode/src/vnd/vnodeBufPool.c | 29 +++++++++++++-- source/dnode/vnode/src/vnd/vnodeCommit.c | 16 ++++----- source/dnode/vnode/src/vnd/vnodeOpen.c | 4 +++ 6 files changed, 68 insertions(+), 40 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index cb25e93cde..984b34814d 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -62,12 +62,13 @@ struct SVBufPoolNode { }; struct SVBufPool { - SVBufPool* next; - int64_t nRef; - int64_t size; - uint8_t* ptr; - SVBufPoolNode* pTail; - SVBufPoolNode node; + SVBufPool* next; + SVnode* pVnode; + volatile int32_t nRef; + int64_t size; + uint8_t* ptr; + SVBufPoolNode* pTail; + SVBufPoolNode node; }; int32_t vnodeOpenBufPool(SVnode* pVnode, int64_t size); @@ -78,7 +79,7 @@ void vnodeBufPoolReset(SVBufPool* pPool); int32_t vnodeQueryOpen(SVnode* pVnode); void vnodeQueryClose(SVnode* pVnode); int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg); -int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg); +int vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg); // vnodeCommit.c int32_t vnodeBegin(SVnode* pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index fb403f79a7..d724bcd3be 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -77,6 +77,8 @@ typedef struct SSnapDataHdr SSnapDataHdr; // vnd.h void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); void vnodeBufPoolFree(SVBufPool* pPool, void* p); +void vnodeBufPoolRef(SVBufPool* pPool); +void vnodeBufPoolUnRef(SVBufPool* pPool); // meta typedef struct SMCtbCursor SMCtbCursor; @@ -247,26 +249,26 @@ struct STsdbKeepCfg { }; struct SVnode { - char* path; - SVnodeCfg config; - SVState state; - STfs* pTfs; - SMsgCb msgCb; - SVBufPool* pPool; - SVBufPool* inUse; - SVBufPool* onCommit; - SVBufPool* onRecycle; - SMeta* pMeta; - SSma* pSma; - STsdb* pTsdb; - SWal* pWal; - STQ* pTq; - SSink* pSink; - tsem_t canCommit; - int64_t sync; - int32_t blockCount; - tsem_t syncSem; - SQHandle* pQuery; + char* path; + SVnodeCfg config; + SVState state; + STfs* pTfs; + SMsgCb msgCb; + TdThreadMutex mutex; + TdThreadCond poolNotEmpty; + SVBufPool* pPool; + SVBufPool* inUse; + SMeta* pMeta; + SSma* pSma; + STsdb* pTsdb; + SWal* pWal; + STQ* pTq; + SSink* pSink; + tsem_t canCommit; + int64_t sync; + int32_t blockCount; + tsem_t syncSem; + SQHandle* pQuery; }; #define TD_VID(PVNODE) ((PVNODE)->config.vgId) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d0a719b5cd..ee8a23e76e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -55,6 +55,7 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { taosMemoryFree(pMemTable); goto _err; } + vnodeBufPoolRef(pMemTable->pPool); *ppMemTable = pMemTable; return code; @@ -66,6 +67,7 @@ _err: void tsdbMemTableDestroy(SMemTable *pMemTable) { if (pMemTable) { + vnodeBufPoolUnRef(pMemTable->pPool); taosArrayDestroy(pMemTable->aTbData); taosMemoryFree(pMemTable); } diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 9ca4dd6efb..d1584f701e 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -17,7 +17,7 @@ /* ------------------------ STRUCTURES ------------------------ */ -static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool); +static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool); static int vnodeBufPoolDestroy(SVBufPool *pPool); int vnodeOpenBufPool(SVnode *pVnode, int64_t size) { @@ -28,7 +28,7 @@ int vnodeOpenBufPool(SVnode *pVnode, int64_t size) { for (int i = 0; i < 3; i++) { // create pool - ret = vnodeBufPoolCreate(size, &pPool); + ret = vnodeBufPoolCreate(pVnode, size, &pPool); if (ret < 0) { vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); vnodeCloseBufPool(pVnode); @@ -120,7 +120,7 @@ void vnodeBufPoolFree(SVBufPool *pPool, void *p) { } // STATIC METHODS ------------------- -static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool) { +static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) { SVBufPool *pPool; pPool = taosMemoryMalloc(sizeof(SVBufPool) + size); @@ -130,6 +130,7 @@ static int vnodeBufPoolCreate(int64_t size, SVBufPool **ppPool) { } pPool->next = NULL; + pPool->pVnode = pVnode; pPool->nRef = 0; pPool->size = 0; pPool->ptr = pPool->node.data; @@ -146,4 +147,26 @@ static int vnodeBufPoolDestroy(SVBufPool *pPool) { vnodeBufPoolReset(pPool); taosMemoryFree(pPool); return 0; +} + +void vnodeBufPoolRef(SVBufPool *pPool) { + int32_t nRef = atomic_fetch_add_32(&pPool->nRef, 1); + ASSERT(nRef > 0); +} + +void vnodeBufPoolUnRef(SVBufPool *pPool) { + int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1); + if (nRef == 0) { + SVnode *pVnode = pPool->pVnode; + + vnodeBufPoolReset(pPool); + + taosThreadMutexLock(&pVnode->mutex); + + pPool->next = pVnode->pPool; + pVnode->pPool = pPool; + taosThreadCondSignal(&pVnode->poolNotEmpty); + + taosThreadMutexUnlock(&pVnode->mutex); + } } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index c969861241..0c8a8a5fd1 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -15,7 +15,7 @@ #include "vnd.h" -#define VND_INFO_FNAME "vnode.json" +#define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); @@ -27,18 +27,18 @@ static void vnodeWaitCommit(SVnode *pVnode); int vnodeBegin(SVnode *pVnode) { // alloc buffer pool - /* pthread_mutex_lock(); */ + taosThreadMutexLock(&pVnode->mutex); while (pVnode->pPool == NULL) { - /* pthread_cond_wait(); */ + taosThreadCondWait(&pVnode->poolNotEmpty, &pVnode->mutex); } pVnode->inUse = pVnode->pPool; + pVnode->inUse->nRef = 1; pVnode->pPool = pVnode->inUse->next; pVnode->inUse->next = NULL; - /* ref pVnode->inUse buffer pool */ - /* pthread_mutex_unlock(); */ + taosThreadMutexUnlock(&pVnode->mutex); pVnode->state.commitID++; // begin meta @@ -217,7 +217,7 @@ int vnodeCommit(SVnode *pVnode) { vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64, TD_VID(pVnode), pVnode->state.commitID, pVnode->state.applied); - pVnode->onCommit = pVnode->inUse; + vnodeBufPoolUnRef(pVnode->inUse); pVnode->inUse = NULL; // save info @@ -284,10 +284,6 @@ int vnodeCommit(SVnode *pVnode) { // apply the commit (TODO) walEndSnapshot(pVnode->pWal); - vnodeBufPoolReset(pVnode->onCommit); - pVnode->onCommit->next = pVnode->pPool; - pVnode->pPool = pVnode->onCommit; - pVnode->onCommit = NULL; vInfo("vgId:%d, commit over", TD_VID(pVnode)); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 0914827950..fb2ac722a6 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -89,6 +89,8 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { tsem_init(&pVnode->syncSem, 0, 0); tsem_init(&(pVnode->canCommit), 0, 1); + taosThreadMutexInit(&pVnode->mutex, NULL); + taosThreadCondInit(&pVnode->poolNotEmpty, NULL); // open buffer pool if (vnodeOpenBufPool(pVnode, pVnode->config.isHeap ? 0 : pVnode->config.szBuf / 3) < 0) { @@ -195,6 +197,8 @@ void vnodeClose(SVnode *pVnode) { // destroy handle tsem_destroy(&(pVnode->canCommit)); tsem_destroy(&pVnode->syncSem); + taosThreadCondDestroy(&pVnode->poolNotEmpty); + taosThreadMutexDestroy(&pVnode->mutex); taosMemoryFree(pVnode); } }