From 8337b4474bbc2bdf5f3d9dbaca997201d06d34a9 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 15 Jul 2022 17:43:45 +0800 Subject: [PATCH 01/88] update case --- tests/system-test/2-query/last_row.py | 152 +++++++++++++++----------- 1 file changed, 87 insertions(+), 65 deletions(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 449f5c7806..4d6fe08c62 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -488,7 +488,14 @@ class TDTestCase: # bug fix for compute tdSql.query("select last_row(c1) -0 ,last(c1)-0 ,last(c1)+last_row(c1) from ct4 ") - tdSql.query(" select c1, abs(c1) -0 ,last_row(c1-0.1)-0.1 from ct4") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(0,2,None) + + tdSql.query(" select c1, abs(c1) -0 ,last_row(c1-0.1)-0.1 from ct1") + tdSql.checkData(0,0,9) + tdSql.checkData(0,1,9.000000000) + tdSql.checkData(0,2,8.800000000) def abs_func_filter(self): tdSql.execute("use db") @@ -616,7 +623,7 @@ class TDTestCase: tdSql.error("select last_row(c1) ,c1 from (select count(c1) c1 from stb1 where ts >now -1h and ts ="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s) fill(NULL)') - # tdSql.checkRows(40) - # tdSql.checkData(0,0,None) - tdSql.query('select max(c1) from stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') - tdSql.checkRows(5) + tdSql.query("select last_row(c1) from stb1 interval(50s) sliding(30s)") + tdSql.checkRows(27) + tdSql.query("select last_row(c1) from ct1 interval(50s) sliding(30s)") + tdSql.checkRows(5) + last_row_result = tdSql.queryResult + tdSql.query("select last(c1) from ct1 interval(50s) sliding(30s)") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + + # bug need fix + tdSql.query('select max(c1) from t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" interval(50d) sliding(30d) fill(NULL)') + tdSql.checkRows(8) + tdSql.checkData(7,0,None) + + tdSql.query('select last_row(c1) from t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" interval(50d) sliding(30d) fill(value ,2 )') + tdSql.checkRows(8) + tdSql.checkData(7,0,2) + + tdSql.query('select last_row(c1) from stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') + tdSql.query('select last_row(c1) from (select ts , c1 from t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" ) interval(10s) sliding(5s)') + + # join + tdSql.query("use test") + tdSql.query("select last(sub_tb_1.c1), last(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + tdSql.checkCols(2) + last_row_result = tdSql.queryResult + tdSql.query("select last_row(sub_tb_1.c1), last_row(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + + tdSql.query("select last(*), last(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + + last_row_result = tdSql.queryResult + tdSql.query("select last_row(*), last_row(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + def support_super_table_test(self): From c45f6c4ba95f8c537d26e34c6e21e650f99c86a4 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Fri, 15 Jul 2022 19:51:53 +0800 Subject: [PATCH 02/88] test:test case for update_iotdata --- tests/system-test/1-insert/update_data.py | 193 ++++++++++++++++++++++ tests/system-test/fulltest.sh | 2 +- 2 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/1-insert/update_data.py diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py new file mode 100644 index 0000000000..8204df24af --- /dev/null +++ b/tests/system-test/1-insert/update_data.py @@ -0,0 +1,193 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db_test' + self.ntbname = 'ntb' + self.stbname = 'stb' + self.ts = 1537146000000 + self.binary_length = 20 + self.nchar_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.binary_length})', + 'col13': f'nchar({self.nchar_length})', + 'col_ts' : 'timestamp' + } + self.tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.bool = random.randint(0,100)%2 + self.float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.binary = tdCom.getLongName(self.binary_length) + self.tnchar = tdCom.getLongName(self.nchar_length) + def insert_base_data(self,col_type,tbname,value=None): + if value == None: + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.tinyint})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.smallint})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.int})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.bigint})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.untinyint})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.unsmallint})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.unint})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.unbigint})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.bool})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.float})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts},{self.double})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'insert into {tbname} values({self.ts},"{self.binary}")') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'insert into {tbname} values({self.ts},"{self.nchar}")') + + def update_and_check_data(self,tbname,col_name,col_type,value): + if 'binary' in col_type.lower() or 'nchar' in col_type.lower(): + tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{value})') + tdSql.query(f'select {col_name} from {tbname}') + if col_type.lower() == 'float' or col_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - value) / value <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{col_name} data check failure') + else: + tdSql.checkEqual(tdSql.queryResult[0][0],value) + + def update_check_ntb(self): + up_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + up_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + up_int = random.randint(constant.INT_MIN,constant.INT_MAX) + up_bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + up_untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + up_unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + up_unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + up_unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + up_bool = random.randint(0,100)%2 + up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + binary_length = random.randint(0,self.binary_length) + nchar_length = random.randint(0,self.binary_length) + up_binary = tdCom.getLongName(binary_length) + up_nchar = tdCom.getLongName(nchar_length) + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(f'use {self.dbname}') + + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_name,self.ntbname) + if col_type.lower() == 'tinyint': + self.update_and_check_data(self.ntbname,col_name,col_type,up_tinyint) + elif col_type.lower() == 'smallint': + self.update_and_check_data(self.ntbname,col_name,col_type,up_smallint) + elif col_type.lower() == 'int': + self.update_and_check_data(self.ntbname,col_name,col_type,up_int) + elif col_type.lower() == 'bigint': + self.update_and_check_data(self.ntbname,col_name,col_type,up_bigint) + elif col_type.lower() == 'tinyint unsigned': + self.update_and_check_data(self.ntbname,col_name,col_type,up_untinyint) + elif col_type.lower() == 'smallint unsigned': + self.update_and_check_data(self.ntbname,col_name,col_type,up_unsmallint) + elif col_type.lower() == 'int unsigned': + self.update_and_check_data(self.ntbname,col_name,col_type,up_unint) + elif col_type.lower() == 'bigint unsigned': + self.update_and_check_data(self.ntbname,col_name,col_type,up_unbigint) + elif col_type.lower() == 'bool': + self.update_and_check_data(self.ntbname,col_name,col_type,up_bool) + elif col_type.lower() == 'float': + self.update_and_check_data(self.ntbname,col_name,col_type,up_float) + elif col_type.lower() == 'double': + self.update_and_check_data(self.ntbname,col_name,col_type,up_double) + elif 'binary' in col_type.lower(): + self.update_and_check_data(self.ntbname,col_name,col_type,up_binary) + elif 'nchar' in col_type.lower(): + self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar) + tdSql.execute(f'drop table {self.ntbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_name,self.ntbname) + tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)') + tdSql.query(f'select {col_name} from {self.ntbname}') + tdSql.checkEqual(tdSql.queryResult[0][0],None) + tdSql.execute(f'drop table {self.ntbname}') + + + def update_check_error_ntb(self): + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(f'use {self.dbname}') + binary_length = self.binary_length+1 + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_name,self.ntbname) + if col_type.lower() in ['tinyint','smallint','int','bigint','tinyint unsigned','smallint unsigned','int unsigned','bigint unsigned']: + for error_value in [random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.binary_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() in ['float','double']: + for error_value in [tdCom.getLongName(self.binary_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): + for error_value in [tdCom.getLongName(binary_length)]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bool': + for error_value in [tdCom.getLongName(self.binary_length)]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + tdSql.execute(f'drop table {self.ntbname}') + + def run(self): + self.update_check_ntb() + self.update_check_error_ntb() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 8bafe3c966..6275e2878c 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -28,7 +28,7 @@ python3 ./test.py -f 1-insert/time_range_wise.py python3 ./test.py -f 1-insert/block_wise.py python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py - +python3 ./test.py -f 1-insert/update_data.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py From 3417cf469c19788597b75fb60efb191360054d33 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 15 Jul 2022 20:00:00 +0800 Subject: [PATCH 03/88] fix:raw block datalen if json data --- source/client/src/clientImpl.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d846cb93af..6185413f22 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1612,11 +1612,13 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; + int32_t totalLen = 0; int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); memcpy(p1, p, len); p += len; p1 += len; + totalLen += len; len = sizeof(int32_t) * numOfCols; int32_t* colLength = (int32_t*)p; @@ -1624,6 +1626,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int memcpy(p1, p, len); p += len; p1 += len; + totalLen += len; char* pStart = p; char* pStart1 = p1; @@ -1639,6 +1642,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int memcpy(pStart1, pStart, len); pStart += len; pStart1 += len; + totalLen += len; len = 0; for (int32_t j = 0; j < numOfRows; ++j) { @@ -1683,24 +1687,30 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int len += varDataTLen(dst); } colLen1 = len; + totalLen += colLen1; colLength1[i] = htonl(len); } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { len = numOfRows * sizeof(int32_t); memcpy(pStart1, pStart, len); pStart += len; pStart1 += len; + totalLen += len; + totalLen += colLen; memcpy(pStart1, pStart, colLen); } else { len = BitmapLen(pResultInfo->numOfRows); memcpy(pStart1, pStart, len); pStart += len; pStart1 += len; + totalLen += len; + totalLen += colLen; memcpy(pStart1, pStart, colLen); } pStart += colLen; pStart1 += colLen1; } + *(int32_t*)(pResultInfo->convertJson) = totalLen; pResultInfo->pData = pResultInfo->convertJson; return TSDB_CODE_SUCCESS; } From eeb57d8893173b6abb81ad5a47aaf590d2a344ac Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 10:07:02 +0800 Subject: [PATCH 04/88] update test case for update --- tests/system-test/1-insert/update_data.py | 61 ++++++++++++++++------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 8204df24af..40682a1910 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -26,8 +26,8 @@ class TDTestCase: self.ntbname = 'ntb' self.stbname = 'stb' self.ts = 1537146000000 - self.binary_length = 20 - self.nchar_length = 20 + self.str_length = 20 + self.column_dict = { 'col1': 'tinyint', 'col2': 'smallint', @@ -40,8 +40,8 @@ class TDTestCase: 'col9': 'float', 'col10': 'double', 'col11': 'bool', - 'col12': f'binary({self.binary_length})', - 'col13': f'nchar({self.nchar_length})', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', 'col_ts' : 'timestamp' } self.tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) @@ -55,8 +55,8 @@ class TDTestCase: self.bool = random.randint(0,100)%2 self.float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) self.double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) - self.binary = tdCom.getLongName(self.binary_length) - self.tnchar = tdCom.getLongName(self.nchar_length) + self.binary = tdCom.getLongName(self.str_length) + self.tnchar = tdCom.getLongName(self.str_length) def insert_base_data(self,col_type,tbname,value=None): if value == None: if col_type.lower() == 'tinyint': @@ -112,8 +112,8 @@ class TDTestCase: up_bool = random.randint(0,100)%2 up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) - binary_length = random.randint(0,self.binary_length) - nchar_length = random.randint(0,self.binary_length) + binary_length = random.randint(0,self.str_length) + nchar_length = random.randint(0,self.str_length) up_binary = tdCom.getLongName(binary_length) up_nchar = tdCom.getLongName(nchar_length) tdSql.execute(f'drop database if exists {self.dbname}') @@ -163,22 +163,49 @@ class TDTestCase: tdSql.execute(f'drop database if exists {self.dbname}') tdSql.execute(f'create database {self.dbname}') tdSql.execute(f'use {self.dbname}') - binary_length = self.binary_length+1 + str_length = self.str_length+1 for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') self.insert_base_data(col_name,self.ntbname) - if col_type.lower() in ['tinyint','smallint','int','bigint','tinyint unsigned','smallint unsigned','int unsigned','bigint unsigned']: - for error_value in [random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.binary_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() in ['float','double']: - for error_value in [tdCom.getLongName(self.binary_length),True,False]: + + + if col_type.lower() == 'double': + for error_value in [tdCom.getLongName(self.str_length),True,False,random.uniform(constant.DOUBLE_MIN*1.01,constant.DOUBLE_MAX*1.01)]: tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + #!bug TD-17453 + # elif col_type.lower() == 'float': + # for error_value in [tdCom.getLongName(self.str_length),True,False,random.uniform(constant.FLOAT_MIN*1.01,constant.FLOAT_MAX*1.01)]: + # tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): - for error_value in [tdCom.getLongName(binary_length)]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + for error_value in [tdCom.getLongName(str_length)]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},"{error_value}")') elif col_type.lower() == 'bool': - for error_value in [tdCom.getLongName(self.binary_length)]: + for error_value in [tdCom.getLongName(self.str_length)]: tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'tinyint': + for error_value in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'smallint': + for error_value in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'int': + for error_value in [constant.INT_MIN-1,constant.INT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bigint': + for error_value in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'tinyint unsigned': + for error_value in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'smallint unsigned': + for error_value in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'int unsigned': + for error_value in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bigint unsigned': + for error_value in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') tdSql.execute(f'drop table {self.ntbname}') def run(self): From a2090052b3a4a60d2d4de71da3a390856df393b0 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 10:21:52 +0800 Subject: [PATCH 05/88] update --- tests/system-test/1-insert/update_data.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 40682a1910..90b5b456cf 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -167,15 +167,12 @@ class TDTestCase: for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') self.insert_base_data(col_name,self.ntbname) - - if col_type.lower() == 'double': - for error_value in [tdCom.getLongName(self.str_length),True,False,random.uniform(constant.DOUBLE_MIN*1.01,constant.DOUBLE_MAX*1.01)]: + for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: + tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + elif col_type.lower() == 'float': + for error_value in [tdCom.getLongName(self.str_length),True,False,10*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - #!bug TD-17453 - # elif col_type.lower() == 'float': - # for error_value in [tdCom.getLongName(self.str_length),True,False,random.uniform(constant.FLOAT_MIN*1.01,constant.FLOAT_MAX*1.01)]: - # tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): for error_value in [tdCom.getLongName(str_length)]: tdSql.error(f'insert into {self.ntbname} values({self.ts},"{error_value}")') From 698baa52ce7eedb09fc1596df84b1a047a19468d Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 10:22:33 +0800 Subject: [PATCH 06/88] update --- tests/system-test/1-insert/update_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 90b5b456cf..12b61b4458 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -171,7 +171,7 @@ class TDTestCase: for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') elif col_type.lower() == 'float': - for error_value in [tdCom.getLongName(self.str_length),True,False,10*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: + for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): for error_value in [tdCom.getLongName(str_length)]: From 98178c6090ee95eb0c7f3bd6c38d0e32ac615b2f Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Sat, 16 Jul 2022 11:02:01 +0800 Subject: [PATCH 07/88] update --- tests/system-test/2-query/last_row.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 4d6fe08c62..a23ee8a86a 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -759,8 +759,15 @@ class TDTestCase: tdSql.query("select last_row(*), last_row(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") for ind , row in enumerate(last_row_result): tdSql.checkData(ind , 0 , row[0]) - + tdSql.query("select last(*), last_row(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + + tdSql.query("select last_row(*), last(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") + for ind , row in enumerate(last_row_result): + tdSql.checkData(ind , 0 , row[0]) + def support_super_table_test(self): tdSql.execute(" use testdb ") From 2f0b6e0086ca5697830e85e77c89d6cb78c9619b Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Sat, 16 Jul 2022 14:00:36 +0800 Subject: [PATCH 08/88] add test case for last_row --- tests/system-test/2-query/last_row.py | 201 ++++++++++++++------------ 1 file changed, 108 insertions(+), 93 deletions(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index a23ee8a86a..a6398fb76f 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -216,16 +216,11 @@ class TDTestCase: tdSql.execute("use testdb") # bug need fix - # tdSql.query("select last_row(c1 ,NULL) from t1") + tdSql.error("select last_row(c1 ,NULL) from t1") error_sql_lists = [ "select last_row from t1", "select last_row(-+--+c1) from t1", - # "select +-last_row(c1) from t1", - # "select ++-last_row(c1) from t1", - # "select ++--last_row(c1) from t1", - # "select - -last_row(c1)*0 from t1", - # "select last_row(tbname+1) from t1 ", "select last_row(123--123)==1 from t1", "select last_row(c1) as 'd1' from t1", "select last_row(c1 ,NULL) from t1", @@ -233,15 +228,8 @@ class TDTestCase: "select last_row(abs(c1) ab from t1)", "select last_row(c1) as int from t1", "select last_row from stb1", - # "select last_row(-+--+c1) from stb1", - # "select +-last_row(c1) from stb1", - # "select ++-last_row(c1) from stb1", - # "select ++--last_row(c1) from stb1", - # "select - -last_row(c1)*0 from stb1", - # "select last_row(tbname+1) from stb1 ", "select last_row(123--123)==1 from stb1", "select last_row(c1) as 'd1' from stb1", - # "select last_row(c1 ,c2 ) from stb1", "select last_row(c1 ,NULL) from stb1", "select last_row(,) from stb1;", "select last_row(abs(c1) ab from stb1)", @@ -313,12 +301,12 @@ class TDTestCase: tdSql.checkData(0, 4,-9.99000) # bug need fix - # tdSql.query("select last_row(c1), c2, c3 , c4, c5 from stb1 where tbname='ct1'") - # tdSql.checkData(0, 0, 9) - # tdSql.checkData(0, 1, -99999) - # tdSql.checkData(0, 2, -999) - # tdSql.checkData(0, 3, None) - # tdSql.checkData(0, 4,-9.99000) + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from stb1 where tbname='ct1'") + tdSql.checkData(0, 0, 9) + tdSql.checkData(0, 1, -99999) + tdSql.checkData(0, 2, -999) + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4,-9.99000) # bug fix tdSql.query("select last_row(abs(c1)) from ct1") @@ -342,13 +330,13 @@ class TDTestCase: tdSql.checkData(0,1,None) tdSql.checkData(0,2,None) - # tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from stb1 where ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts Date: Sat, 16 Jul 2022 14:16:48 +0800 Subject: [PATCH 09/88] add test case for last_row --- tests/system-test/2-query/last_row.py | 61 +++++++++++++++++++-------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index a6398fb76f..de8ed20fb3 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -22,10 +22,11 @@ class TDTestCase: self.ts = 1434938400000 self.time_step = 1000 - def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ): + def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ,cache_value ): + tdSql.execute("drop database if exists test ") tdLog.info(" prepare datas for auto check abs function ") - tdSql.execute(" create database test cachelast 1 ") + tdSql.execute(f" create database test cachelast {cache_value} ") tdSql.execute(" use test ") tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)") @@ -62,8 +63,10 @@ class TDTestCase: self.check_result_auto(origin_sql , abs_sql) - def prepare_datas(self): - tdSql.execute("create database if not exists db keep 3650 duration 1000 cachelast 1") + def prepare_datas(self ,cache_value): + tdSql.execute("drop database if exists db ") + create_db_sql = f"create database if not exists db keep 3650 duration 1000 cachelast {cache_value}" + tdSql.execute(create_db_sql) tdSql.execute("use db") tdSql.execute( '''create table stb1 @@ -121,10 +124,12 @@ class TDTestCase: ''' ) - def prepare_tag_datas(self): + def prepare_tag_datas(self,cache_value): + + tdSql.execute("drop database if exists testdb ") # prepare datas tdSql.execute( - "create database if not exists testdb keep 3650 duration 1000 cachelast 1") + f"create database if not exists testdb keep 3650 duration 1000 cachelast {cache_value}") tdSql.execute(" use testdb ") tdSql.execute(f" create stable stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp , uc1 int unsigned,\ @@ -796,16 +801,9 @@ class TDTestCase: self.check_result_auto( " select t3,c1 from stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from stb1 where c1 > 0 order by tbname" ) self.check_result_auto( " select t4,c1 from stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from stb1 where c1 > 0 order by tbname" ) pass - - - def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - # tdSql.prepare() - - tdLog.printNoPrefix("==========step1:create table ==============") - - self.prepare_datas() - self.prepare_tag_datas() - + + def basic_query(self): + tdLog.printNoPrefix("==========step2:test errors ==============") self.test_errors() @@ -832,12 +830,41 @@ class TDTestCase: tdLog.printNoPrefix("==========step7: check result of query ============") - self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step) tdLog.printNoPrefix("==========step8: check abs result of stable query ============") self.support_super_table_test() + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + # tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table ==============") + + # cache_last 0 + self.prepare_datas(0) + self.prepare_tag_datas(0) + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,0) + self.basic_query() + + # cache_last 1 + self.prepare_datas(1) + self.prepare_tag_datas(1) + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,1) + self.basic_query() + + # cache_last 2 + self.prepare_datas(2) + self.prepare_tag_datas(2) + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,2) + self.basic_query() + + # cache_last 3 + self.prepare_datas(3) + self.prepare_tag_datas(3) + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,3) + self.basic_query() + + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") From 2c1a36e7208a7866f25d52eb6ef1c0fc3b911dec Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Sat, 16 Jul 2022 14:19:40 +0800 Subject: [PATCH 10/88] add test case for Ci --- tests/system-test/fulltest.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 8bafe3c966..683dbde3c8 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -122,7 +122,7 @@ python3 ./test.py -f 2-query/count_partition.py python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 2-query/queryQnode.py python3 ./test.py -f 2-query/max_partition.py - +python3 ./test.py -f 2-query/last_row.py python3 ./test.py -f 6-cluster/5dnode1mnode.py #BUG python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 @@ -270,6 +270,7 @@ python3 ./test.py -f 2-query/irate.py -Q 2 python3 ./test.py -f 2-query/function_null.py -Q 2 python3 ./test.py -f 2-query/count_partition.py -Q 2 python3 ./test.py -f 2-query/max_partition.py -Q 2 +python3 ./test.py -f 2-query/last_row.py -Q 2 #------------querPolicy 3----------- @@ -357,3 +358,4 @@ python3 ./test.py -f 2-query/irate.py -Q 3 python3 ./test.py -f 2-query/function_null.py -Q 3 python3 ./test.py -f 2-query/count_partition.py -Q 3 python3 ./test.py -f 2-query/max_partition.py -Q 3 +python3 ./test.py -f 2-query/last_row.py -Q 3 From fb6d4a566c832e23374546ebb039abdc7c07cb88 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 14:36:17 +0800 Subject: [PATCH 11/88] update --- tests/system-test/1-insert/update_data.py | 46 +++++++++++++---------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 12b61b4458..35b082f726 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -85,12 +85,7 @@ class TDTestCase: tdSql.execute(f'insert into {tbname} values({self.ts},"{self.binary}")') elif 'nchar' in col_type.lower(): tdSql.execute(f'insert into {tbname} values({self.ts},"{self.nchar}")') - - def update_and_check_data(self,tbname,col_name,col_type,value): - if 'binary' in col_type.lower() or 'nchar' in col_type.lower(): - tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")') - else: - tdSql.execute(f'insert into {tbname} values({self.ts},{value})') + def data_check(self,tbname,col_name,col_type,value): tdSql.query(f'select {col_name} from {tbname}') if col_type.lower() == 'float' or col_type.lower() == 'double': if abs(tdSql.queryResult[0][0] - value) / value <= 0.0001: @@ -99,7 +94,15 @@ class TDTestCase: tdLog.exit(f'{col_name} data check failure') else: tdSql.checkEqual(tdSql.queryResult[0][0],value) - + pass + def update_and_check_data(self,tbname,col_name,col_type,value,dbname): + if 'binary' in col_type.lower() or 'nchar' in col_type.lower(): + tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{value})') + self.data_check(tbname,col_name,col_type,value) + tdSql.execute(f'flush database {dbname}') + self.data_check(tbname,col_name,col_type,value) def update_check_ntb(self): up_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) up_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) @@ -124,31 +127,31 @@ class TDTestCase: tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') self.insert_base_data(col_name,self.ntbname) if col_type.lower() == 'tinyint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_tinyint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_tinyint,self.dbname) elif col_type.lower() == 'smallint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_smallint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_smallint,self.dbname) elif col_type.lower() == 'int': - self.update_and_check_data(self.ntbname,col_name,col_type,up_int) + self.update_and_check_data(self.ntbname,col_name,col_type,up_int,self.dbname) elif col_type.lower() == 'bigint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_bigint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_bigint,self.dbname) elif col_type.lower() == 'tinyint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_untinyint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_untinyint,self.dbname) elif col_type.lower() == 'smallint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unsmallint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_unsmallint,self.dbname) elif col_type.lower() == 'int unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_unint,self.dbname) elif col_type.lower() == 'bigint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unbigint) + self.update_and_check_data(self.ntbname,col_name,col_type,up_unbigint,self.dbname) elif col_type.lower() == 'bool': - self.update_and_check_data(self.ntbname,col_name,col_type,up_bool) + self.update_and_check_data(self.ntbname,col_name,col_type,up_bool,self.dbname) elif col_type.lower() == 'float': - self.update_and_check_data(self.ntbname,col_name,col_type,up_float) + self.update_and_check_data(self.ntbname,col_name,col_type,up_float,self.dbname) elif col_type.lower() == 'double': - self.update_and_check_data(self.ntbname,col_name,col_type,up_double) + self.update_and_check_data(self.ntbname,col_name,col_type,up_double,self.dbname) elif 'binary' in col_type.lower(): - self.update_and_check_data(self.ntbname,col_name,col_type,up_binary) + self.update_and_check_data(self.ntbname,col_name,col_type,up_binary,self.dbname) elif 'nchar' in col_type.lower(): - self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar) + self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname) tdSql.execute(f'drop table {self.ntbname}') for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') @@ -156,6 +159,9 @@ class TDTestCase: tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)') tdSql.query(f'select {col_name} from {self.ntbname}') tdSql.checkEqual(tdSql.queryResult[0][0],None) + tdSql.execute(f'flush database {self.dbname}') + tdSql.query(f'select {col_name} from {self.ntbname}') + tdSql.checkEqual(tdSql.queryResult[0][0],None) tdSql.execute(f'drop table {self.ntbname}') From 418696316aa4080f4fab0465f8a8b0cff010fa8c Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Sat, 16 Jul 2022 15:55:49 +0800 Subject: [PATCH 12/88] update case --- tests/system-test/2-query/last_row.py | 32 +++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index de8ed20fb3..577be8716f 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -26,7 +26,7 @@ class TDTestCase: tdSql.execute("drop database if exists test ") tdLog.info(" prepare datas for auto check abs function ") - tdSql.execute(f" create database test cachelast {cache_value} ") + tdSql.execute(f" create database test cachemodel {cache_value} ") tdSql.execute(" use test ") tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)") @@ -65,7 +65,7 @@ class TDTestCase: def prepare_datas(self ,cache_value): tdSql.execute("drop database if exists db ") - create_db_sql = f"create database if not exists db keep 3650 duration 1000 cachelast {cache_value}" + create_db_sql = f"create database if not exists db keep 3650 duration 1000 cachemodel {cache_value}" tdSql.execute(create_db_sql) tdSql.execute("use db") tdSql.execute( @@ -129,7 +129,7 @@ class TDTestCase: tdSql.execute("drop database if exists testdb ") # prepare datas tdSql.execute( - f"create database if not exists testdb keep 3650 duration 1000 cachelast {cache_value}") + f"create database if not exists testdb keep 3650 duration 1000 cachemodel {cache_value}") tdSql.execute(" use testdb ") tdSql.execute(f" create stable stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp , uc1 int unsigned,\ @@ -540,7 +540,7 @@ class TDTestCase: def check_boundary_values(self): tdSql.execute("drop database if exists bound_test") - tdSql.execute("create database if not exists bound_test cachelast 2") + tdSql.execute("create database if not exists bound_test cachemodel 'LAST_ROW' ") time.sleep(3) tdSql.execute("use bound_test") tdSql.execute( @@ -841,27 +841,27 @@ class TDTestCase: tdLog.printNoPrefix("==========step1:create table ==============") # cache_last 0 - self.prepare_datas(0) - self.prepare_tag_datas(0) - self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,0) + self.prepare_datas("'NONE' ") + self.prepare_tag_datas("'NONE'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'NONE'") self.basic_query() # cache_last 1 - self.prepare_datas(1) - self.prepare_tag_datas(1) - self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,1) + self.prepare_datas("'LAST_ROW'") + self.prepare_tag_datas("'LAST_ROW'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_ROW'") self.basic_query() # cache_last 2 - self.prepare_datas(2) - self.prepare_tag_datas(2) - self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,2) + self.prepare_datas("'LAST_VALUE'") + self.prepare_tag_datas("'LAST_VALUE'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_VALUE'") self.basic_query() # cache_last 3 - self.prepare_datas(3) - self.prepare_tag_datas(3) - self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,3) + self.prepare_datas("'BOTH'") + self.prepare_tag_datas("'BOTH'") + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'BOTH'") self.basic_query() From 7615341dffedf532efe1c124a953b1eaec22694d Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 16:08:46 +0800 Subject: [PATCH 13/88] update --- tests/system-test/1-insert/update_data.py | 218 +++++++++++++--------- 1 file changed, 131 insertions(+), 87 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 35b082f726..f6ede9c2f9 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -11,6 +11,7 @@ # -*- coding: utf-8 -*- +from optparse import Values import random import string from util import constant @@ -18,16 +19,18 @@ from util.log import * from util.cases import * from util.sql import * from util.common import * +from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) + self.setsql = TDSetSql() self.dbname = 'db_test' self.ntbname = 'ntb' self.stbname = 'stb' + self.ctbname = 'ctb' self.ts = 1537146000000 self.str_length = 20 - self.column_dict = { 'col1': 'tinyint', 'col2': 'smallint', @@ -44,19 +47,19 @@ class TDTestCase: 'col13': f'nchar({self.str_length})', 'col_ts' : 'timestamp' } - self.tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) - self.smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) - self.int = random.randint(constant.INT_MIN,constant.INT_MAX) - self.bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) - self.untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) - self.unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) - self.unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX) - self.unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) - self.bool = random.randint(0,100)%2 - self.float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) - self.double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) - self.binary = tdCom.getLongName(self.str_length) - self.tnchar = tdCom.getLongName(self.str_length) + self.tinyint = None + self.smallint = None + self.int = None + self.bigint = None + self.untinyint = None + self.unsmallint = None + self.unint = None + self.unbigint = None + self.bool = None + self.float = None + self.double = None + self.binary = None + self.tnchar = None def insert_base_data(self,col_type,tbname,value=None): if value == None: if col_type.lower() == 'tinyint': @@ -94,7 +97,6 @@ class TDTestCase: tdLog.exit(f'{col_name} data check failure') else: tdSql.checkEqual(tdSql.queryResult[0][0],value) - pass def update_and_check_data(self,tbname,col_name,col_type,value,dbname): if 'binary' in col_type.lower() or 'nchar' in col_type.lower(): tdSql.execute(f'insert into {tbname} values({self.ts},"{value}")') @@ -102,8 +104,57 @@ class TDTestCase: tdSql.execute(f'insert into {tbname} values({self.ts},{value})') self.data_check(tbname,col_name,col_type,value) tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') self.data_check(tbname,col_name,col_type,value) - def update_check_ntb(self): + def error_check(self,tbname,column_dict,tb_type=None,stbname=None): + str_length = self.str_length+1 + for col_name,col_type in column_dict.items(): + if tb_type == 'ntb': + tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})') + elif tb_type == 'ctb': + tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') + tdSql.execute(f'create table {tbname} using {stbname} tags(1)') + self.insert_base_data(col_name,tbname) + if col_type.lower() == 'double': + for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'float': + for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): + for error_value in [tdCom.getLongName(str_length)]: + tdSql.error(f'insert into {tbname} values({self.ts},"{error_value}")') + elif col_type.lower() == 'bool': + for error_value in [tdCom.getLongName(self.str_length)]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'tinyint': + for error_value in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'smallint': + for error_value in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'int': + for error_value in [constant.INT_MIN-1,constant.INT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bigint': + for error_value in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'tinyint unsigned': + for error_value in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'smallint unsigned': + for error_value in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'int unsigned': + for error_value in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + elif col_type.lower() == 'bigint unsigned': + for error_value in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + tdSql.execute(f'drop table {tbname}') + if tb_type == 'ctb': + tdSql.execute(f'drop table {stbname}') + def update_data_check(self,tbname,column_dict,dbname,tb_type=None,stbname=None): up_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) up_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) up_int = random.randint(constant.INT_MIN,constant.INT_MAX) @@ -119,102 +170,95 @@ class TDTestCase: nchar_length = random.randint(0,self.str_length) up_binary = tdCom.getLongName(binary_length) up_nchar = tdCom.getLongName(nchar_length) - tdSql.execute(f'drop database if exists {self.dbname}') - tdSql.execute(f'create database {self.dbname}') - tdSql.execute(f'use {self.dbname}') - - for col_name,col_type in self.column_dict.items(): - tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') - self.insert_base_data(col_name,self.ntbname) + + for col_name,col_type in column_dict.items(): + if tb_type == 'ntb': + tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})') + elif tb_type == 'ctb': + tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') + tdSql.execute(f'create table {tbname} using {stbname} tags(1)') + self.insert_base_data(col_name,tbname) if col_type.lower() == 'tinyint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_tinyint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_tinyint,dbname) elif col_type.lower() == 'smallint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_smallint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_smallint,dbname) elif col_type.lower() == 'int': - self.update_and_check_data(self.ntbname,col_name,col_type,up_int,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_int,dbname) elif col_type.lower() == 'bigint': - self.update_and_check_data(self.ntbname,col_name,col_type,up_bigint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_bigint,dbname) elif col_type.lower() == 'tinyint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_untinyint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_untinyint,dbname) elif col_type.lower() == 'smallint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unsmallint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_unsmallint,dbname) elif col_type.lower() == 'int unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_unint,dbname) elif col_type.lower() == 'bigint unsigned': - self.update_and_check_data(self.ntbname,col_name,col_type,up_unbigint,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_unbigint,dbname) elif col_type.lower() == 'bool': - self.update_and_check_data(self.ntbname,col_name,col_type,up_bool,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_bool,dbname) elif col_type.lower() == 'float': - self.update_and_check_data(self.ntbname,col_name,col_type,up_float,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_float,dbname) elif col_type.lower() == 'double': - self.update_and_check_data(self.ntbname,col_name,col_type,up_double,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_double,dbname) elif 'binary' in col_type.lower(): - self.update_and_check_data(self.ntbname,col_name,col_type,up_binary,self.dbname) + self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname) elif 'nchar' in col_type.lower(): - self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname) - tdSql.execute(f'drop table {self.ntbname}') - for col_name,col_type in self.column_dict.items(): - tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') - self.insert_base_data(col_name,self.ntbname) - tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)') - tdSql.query(f'select {col_name} from {self.ntbname}') + self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname) + tdSql.execute(f'insert into {tbname} values({self.ts},null)') + tdSql.query(f'select {col_name} from {tbname}') tdSql.checkEqual(tdSql.queryResult[0][0],None) tdSql.execute(f'flush database {self.dbname}') - tdSql.query(f'select {col_name} from {self.ntbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') tdSql.checkEqual(tdSql.queryResult[0][0],None) - tdSql.execute(f'drop table {self.ntbname}') - - - def update_check_error_ntb(self): + tdSql.execute(f'drop table {tbname}') + if tb_type == 'ctb': + tdSql.execute(f'drop table {stbname}') + def update_check(self): tdSql.execute(f'drop database if exists {self.dbname}') tdSql.execute(f'create database {self.dbname}') tdSql.execute(f'use {self.dbname}') - str_length = self.str_length+1 + self.update_data_check(self.ntbname,self.column_dict,self.dbname,'ntb') for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') self.insert_base_data(col_name,self.ntbname) - if col_type.lower() == 'double': - for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'float': - for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): - for error_value in [tdCom.getLongName(str_length)]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},"{error_value}")') - elif col_type.lower() == 'bool': - for error_value in [tdCom.getLongName(self.str_length)]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'tinyint': - for error_value in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'smallint': - for error_value in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'int': - for error_value in [constant.INT_MIN-1,constant.INT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'bigint': - for error_value in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'tinyint unsigned': - for error_value in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'smallint unsigned': - for error_value in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'int unsigned': - for error_value in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') - elif col_type.lower() == 'bigint unsigned': - for error_value in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {self.ntbname} values({self.ts},{error_value})') + if 'binary' in col_type.lower(): + up_binary = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.ntbname} modify column {col_name} binary({self.str_length+1})') + self.update_and_check_data(self.ntbname,col_name,col_type,up_binary,self.dbname) + elif 'nchar' in col_type.lower(): + up_nchar = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.ntbname} modify column {col_name} nchar({self.str_length+1})') + self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname) tdSql.execute(f'drop table {self.ntbname}') + + self.update_data_check(self.ctbname,self.column_dict,self.dbname,'ctb',self.stbname) + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') + tdSql.execute(f'create table {self.ctbname} using {self.stbname} tags(1)') + self.insert_base_data(col_name,self.ctbname) + if 'binary' in col_type.lower(): + up_binary = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.stbname} modify column {col_name} binary({self.str_length+1})') + self.update_and_check_data(self.ctbname,col_name,col_type,up_binary,self.dbname) + elif 'nchar' in col_type.lower(): + up_nchar = tdCom.getLongName(self.str_length+1) + tdSql.execute(f'alter table {self.stbname} modify column {col_name} nchar({self.str_length+1})') + self.update_and_check_data(self.ctbname,col_name,col_type,up_nchar,self.dbname) + tdSql.execute(f'drop table {self.stbname}') + + + def update_check_error(self): + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(f'use {self.dbname}') + self.error_check(self.ntbname,self.column_dict,'ntb') + self.error_check(self.ctbname,self.column_dict,'ctb',self.stbname) def run(self): - self.update_check_ntb() - self.update_check_error_ntb() - + self.update_check() + self.update_check_error() + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 887cca3aa86296339cb1e341e42bcb46a5ef11e4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 16 Jul 2022 16:11:16 +0800 Subject: [PATCH 14/88] fix:error in schemaless --- source/client/src/clientSml.c | 2 +- source/client/test/smlTest.cpp | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5f26db67fc..e1230abbce 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -609,7 +609,7 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { } kvVal->type = TSDB_DATA_TYPE_BIGINT; kvVal->i = (int64_t)result; - } else if ((left == 3 && strncasecmp(endptr, "u64", left) == 0)) { + } else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) { if (result >= (double)UINT64_MAX || result < 0) { errno = 0; uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 17025db730..59eb841ab3 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -499,6 +499,32 @@ TEST(testCase, smlGetTimestampLen_Test) { ASSERT_EQ(len, 3); } +TEST(testCase, smlParseNumber_Test) { + SSmlKv kv = {0}; + char buf[64] = {0}; + SSmlMsgBuf msg = {0}; + msg.buf = buf; + msg.len = 64; + kv.value = "3.2e-900"; + kv.length = 8; + bool res = smlParseNumber(&kv, &msg); + printf("res:%d,v:%f, %f\n", res,kv.d, HUGE_VAL); +} + +//#include +//TEST(testCase, number_Test) { +// char *str[] = { +//// "-000 0999", +// "- abc", +// }; +// for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ +// errno = 0; +// char *end = NULL; +// long result = strtol(str[i], &end, 10); +// printf("errno:%d,len:%d,result:%ld\n", errno, end - str[i], result); +// } +// +//} /* TEST(testCase, smlProcess_influx_Test) { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); From bc3f7303ef6968280c081856d9f3e9fbdae2df88 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 16 Jul 2022 16:16:18 +0800 Subject: [PATCH 15/88] update --- tests/system-test/1-insert/update_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index f6ede9c2f9..387fc18774 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -95,6 +95,8 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) else: tdLog.exit(f'{col_name} data check failure') + elif col_type.lower() == 'timestamp': + tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f"))) else: tdSql.checkEqual(tdSql.queryResult[0][0],value) def update_and_check_data(self,tbname,col_name,col_type,value,dbname): @@ -204,6 +206,8 @@ class TDTestCase: self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname) elif 'nchar' in col_type.lower(): self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname) + elif col_type.lower() == 'timestamp': + self.update_and_check_data(tbname,col_name,col_type,self.ts+1,dbname) tdSql.execute(f'insert into {tbname} values({self.ts},null)') tdSql.query(f'select {col_name} from {tbname}') tdSql.checkEqual(tdSql.queryResult[0][0],None) From 22903b3dffc91b39ff31f026ef39c27ffb59da8a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 16 Jul 2022 16:34:56 +0800 Subject: [PATCH 16/88] fix:nchar is too long --- source/client/src/clientSml.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index e1230abbce..83f2cf9307 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1047,6 +1047,10 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab continue; } + if(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + // add kv to SSmlKv SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if (!kv) return TSDB_CODE_OUT_OF_MEMORY; From 38e5bb1a11c04fb6e8ee32133175d01c241267f1 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Sat, 16 Jul 2022 16:39:49 +0800 Subject: [PATCH 17/88] update case --- tests/system-test/2-query/last_row.py | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 577be8716f..d4d4f06563 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -605,28 +605,28 @@ class TDTestCase: tdSql.checkRows(0) # nest query for last_row - # tdSql.query("select last_row(t1) from (select ts , c1 ,t1 from stb1)") - # tdSql.checkData(0,0,4) - # tdSql.query("select distinct(c1) ,t1 from stb1") - # tdSql.checkRows(20) - # tdSql.query("select last_row(c1) from (select _rowts , c1 ,t1 from stb1)") - # tdSql.checkData(0,0,None) + tdSql.query("select last_row(t1) from (select ts , c1 ,t1 from stb1)") + tdSql.checkData(0,0,4) + tdSql.query("select distinct(c1) ,t1 from stb1") + tdSql.checkRows(20) + tdSql.query("select last_row(c1) from (select _rowts , c1 ,t1 from stb1)") + tdSql.checkData(0,0,None) - # tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") - # tdSql.checkData(0,0,None) + tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") + tdSql.checkData(0,0,None) - # tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from stb1)") - # tdSql.checkData(0,1,None,None) + tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from stb1)") + tdSql.checkData(0,1,None,None) - # tdSql.query("select ts , last_row(c1) ,c1 from (select ts , max(c1) c1 ,t1 from stb1 where ts >now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts Date: Sat, 16 Jul 2022 17:45:55 +0800 Subject: [PATCH 18/88] test: extent tmqSim --- tests/test/c/tmqSim.c | 525 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 521 insertions(+), 4 deletions(-) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index b4f86d52b5..5459e3f159 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "taos.h" #include "taosdef.h" @@ -35,6 +36,8 @@ #define MAX_ROW_STR_LEN (16 * 1024) #define MAX_CONSUMER_THREAD_CNT (16) #define MAX_VGROUP_CNT (32) +#define SEND_TIME_UNIT 10 // ms +#define MAX_SQL_LEN 1048576 typedef enum { NOTIFY_CMD_START_CONSUM, @@ -42,6 +45,12 @@ typedef enum { NOTIFY_CMD_ID_BUTT, } NOTIFY_CMD_ID; +typedef enum enumQUERY_TYPE { + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT +} QUERY_TYPE; + typedef struct { TdThread thread; int32_t consumerId; @@ -58,6 +67,7 @@ typedef struct { int64_t consumeMsgCnt; int64_t consumeRowCnt; + int64_t consumeLen; int32_t checkresult; char topicString[1024]; @@ -77,14 +87,19 @@ typedef struct { int32_t rowsOfPerVgroups[MAX_VGROUP_CNT][2]; // [i][0]: vgroup id, [i][1]: rows of consume int64_t ts; - TAOS* taos; + TAOS* taos; + + // below parameters is used by omb test + int32_t producerRate; // unit: msgs/s + int64_t totalProduceMsgs; + int64_t totalMsgsLen; } SThreadInfo; typedef struct { // input from argvs char cdbName[32]; - char dbName[32]; + char dbName[64]; int32_t showMsgFlag; int32_t showRowFlag; int32_t saveRowFlag; @@ -93,11 +108,22 @@ typedef struct { int32_t useSnapshot; int64_t nowTime; SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; + + SThreadInfo stProdThreads[MAX_CONSUMER_THREAD_CNT]; + + // below parameters is used by omb test + char topic[64]; + int32_t producers; + int32_t producerRate; + int32_t runDurationMinutes; + int32_t batchSize; + int32_t payloadLen; } SConfInfo; static SConfInfo g_stConfInfo; TdFilePtr g_fp = NULL; static int running = 1; +char* g_payload = NULL; // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -117,7 +143,29 @@ static void printHelp() { printf("%s%s\n", indent, "-s"); printf("%s%s%s%d\n", indent, indent, "saveRowFlag, default is ", g_stConfInfo.saveRowFlag); printf("%s%s\n", indent, "-y"); - printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay); + printf("%s%s%s%ds\n", indent, indent, "consume delay, default is ", g_stConfInfo.consumeDelay); + printf("%s%s\n", indent, "-e"); + printf("%s%s%s%d\n", indent, indent, "snapshot, default is ", g_stConfInfo.useSnapshot); + + printf("%s%s\n", indent, "-t"); + printf("%s%s%s\n", indent, indent, "topic name, default is null"); + + printf("%s%s\n", indent, "-x"); + printf("%s%s%s\n", indent, indent, "consume thread number, default is 1"); + + + printf("%s%s\n", indent, "-l"); + printf("%s%s%s\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); + printf("%s%s\n", indent, "-p"); + printf("%s%s%s\n", indent, indent, "producer thread number, default is 0"); + printf("%s%s\n", indent, "-b"); + printf("%s%s%s\n", indent, indent, "batch size, default is 1"); + printf("%s%s\n", indent, "-i"); + printf("%s%s%s\n", indent, indent, "produce rate unit is msgs /s, default is 100000"); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s\n", indent, indent, "payload len unit is byte, default is 1000"); + + exit(EXIT_SUCCESS); } @@ -144,7 +192,11 @@ void initLogFile() { pid_t process_id = getpid(); - sprintf(filename, "%s/../log/tmqlog-%d-%s.txt", configDir, process_id, getCurrentTimeString(tmpString)); + if (0 != strlen(g_stConfInfo.topic)) { + sprintf(filename, "/tmp/tmqlog-%d-%s.txt", process_id, getCurrentTimeString(tmpString)); + } else { + sprintf(filename, "%s/../log/tmqlog-%d-%s.txt", configDir, process_id, getCurrentTimeString(tmpString)); + } #ifdef WINDOWS for (int i = 2; i < sizeof(filename); i++) { if (filename[i] == ':') filename[i] = '-'; @@ -199,6 +251,9 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.showRowFlag = 0; g_stConfInfo.saveRowFlag = 0; g_stConfInfo.consumeDelay = 5; + g_stConfInfo.numOfThread = 1; + g_stConfInfo.batchSize = 1; + g_stConfInfo.producers = 0; g_stConfInfo.nowTime = taosGetTimestampMs(); @@ -222,12 +277,38 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.consumeDelay = atol(argv[++i]); } else if (strcmp(argv[i], "-e") == 0) { g_stConfInfo.useSnapshot = atol(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + char tmpBuf[56]; + strcpy(tmpBuf, argv[++i]); + sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); + } else if (strcmp(argv[i], "-x") == 0) { + g_stConfInfo.numOfThread = atol(argv[++i]); + } else if (strcmp(argv[i], "-l") == 0) { + g_stConfInfo.runDurationMinutes = atol(argv[++i]); + } else if (strcmp(argv[i], "-p") == 0) { + g_stConfInfo.producers = atol(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + g_stConfInfo.batchSize = atol(argv[++i]); + } else if (strcmp(argv[i], "-i") == 0) { + g_stConfInfo.producerRate = atol(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + g_stConfInfo.payloadLen = atol(argv[++i]); } else { pError("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); } } + g_payload = taosMemoryCalloc(g_stConfInfo.payloadLen + 1, 1); + if (NULL == g_payload) { + pPrint("%s failed to malloc for payload %s", GREEN, NC); + exit(-1); + } + + for (int32_t i = 0; i < g_stConfInfo.payloadLen; i++) { + strcpy(&g_payload[i], "a"); + } + initLogFile(); taosFprintfFile(g_fp, "====parseArgument() success\n"); @@ -240,6 +321,11 @@ void parseArgument(int32_t argc, char* argv[]) { pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC); pPrint("%s saveRowFlag:%d %s", GREEN, g_stConfInfo.saveRowFlag, NC); + + pPrint("%s snapshot:%d %s", GREEN, g_stConfInfo.useSnapshot, NC); + + pPrint("%s omb topic:%s %s", GREEN, g_stConfInfo.topic, NC); + pPrint("%s numOfThread:%d %s", GREEN, g_stConfInfo.numOfThread, NC); #endif } @@ -909,8 +995,439 @@ int32_t getConsumeInfo() { return 0; } + +static int32_t omb_data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex, int64_t* lenOfRows) { + char buf[16*1024]; + int32_t totalRows = 0; + int32_t totalLen = 0; + + // printf("topic: %s\n", tmq_get_topic_name(msg)); + //int32_t vgroupId = tmq_get_vgroup_id(msg); + //const char* dbName = tmq_get_db_name(msg); + + //taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + //taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", + // tmq_get_topic_name(msg), vgroupId); + + while (1) { + TAOS_ROW row = taos_fetch_row(msg); + + if (row == NULL) break; + + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + //int32_t* length = taos_fetch_lengths(msg); + //int32_t precision = taos_result_precision(msg); + //const char* tbName = tmq_get_table_name(msg); + + taos_print_row(buf, row, fields, numOfFields); + totalLen += strlen(buf); + totalRows++; + } + + *lenOfRows = totalLen; + return totalRows; +} + +void omb_loop_consume(SThreadInfo* pInfo) { + int32_t code; + + int32_t once_flag = 0; + + int64_t totalMsgs = 0; + int64_t totalRows = 0; + + char tmpString[128]; + taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), + pInfo->consumerId); + printf("%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), + pInfo->consumerId); + + pInfo->ts = taosGetTimestampMs(); + + int64_t lastTotalMsgs = 0; + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + + int64_t totalLenOfMsg = 0; + int64_t lastTotalLenOfMsg = 0; + int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); + while (running) { + TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay); + if (tmqMsg) { + int64_t lenOfMsg = 0; + totalRows += omb_data_msg_process(tmqMsg, pInfo, totalMsgs, &lenOfMsg); + totalLenOfMsg += lenOfMsg; + taos_free_result(tmqMsg); + totalMsgs++; + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 10 * 1000) { + int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg; + int64_t deltaTime = currentPrintTime - lastPrintTime; + printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalRows, totalMsgs, + (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, + currentLenOfMsg*1000.0/(1024*1024)/deltaTime); + + taosFprintfFile( + g_fp, "consumer id %d has currently poll total msgs: %" PRId64 ", period cons rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, currentLenOfMsg*1000.0/deltaTime); + lastPrintTime = currentPrintTime; + lastTotalMsgs = totalMsgs; + lastTotalLenOfMsg = totalLenOfMsg; + } + } else { + char tmpString[128]; + taosFprintfFile(g_fp, "%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); + printf("%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); + int64_t currentPrintTime = taosGetTimestampMs(); + int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg; + int64_t deltaTime = currentPrintTime - lastPrintTime; + printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalRows, totalMsgs, + (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, + currentLenOfMsg*1000.0/(1024*1024)/deltaTime); + break; + } + } + + pInfo->consumeMsgCnt = totalMsgs; + pInfo->consumeRowCnt = totalRows; + pInfo->consumeLen = totalLenOfMsg; + +} + + +void* ombConsumeThreadFunc(void* param) { + SThreadInfo* pInfo = (SThreadInfo*)param; + + //################### set key ######################## + tmq_conf_t* conf = tmq_conf_new(); + // tmq_conf_set(conf, "td.connect.ip", "localhost"); + // tmq_conf_set(conf, "td.connect.port", "6030"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + // tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, pInfo); + tmq_conf_set(conf, "group.id", "ombCgrp"); + // tmq_conf_set(conf, "msg.with.table.name", "true"); + // tmq_conf_set(conf, "client.id", "c-001"); + // tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "enable.auto.commit", "false"); + // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + // tmq_conf_set(conf, "auto.offset.reset", "none"); + // tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + // + if (g_stConfInfo.useSnapshot) { + tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + } + + pInfo->tmq = tmq_consumer_new(conf, NULL, 0); + + tmq_conf_destroy(conf); + + //################### set topic ########################## + pInfo->topicList = tmq_list_new(); + tmq_list_append(pInfo->topicList, g_stConfInfo.topic); + + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { + taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); + assert(0); + return NULL; + } + + int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); + if (err != 0) { + pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); + assert(0); + return NULL; + } + + tmq_list_destroy(pInfo->topicList); + pInfo->topicList = NULL; + + omb_loop_consume(pInfo); + + err = tmq_unsubscribe(pInfo->tmq); + if (err != 0) { + pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_unsubscribe()! reason: %s\n", tmq_err2str(err)); + } + + err = tmq_consumer_close(pInfo->tmq); + if (err != 0) { + pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_consumer_close()! reason: %s\n", tmq_err2str(err)); + } + pInfo->tmq = NULL; + + return NULL; +} + + + +static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type) { + TAOS_RES *res = taos_query(taos, command); + int32_t code = taos_errno(res); + + if (code != 0) { + pPrint("%s Failed to execute <%s>, reason: %s %s", GREEN, command, taos_errstr(res), NC); + taos_free_result(res); + return -1; + } + + if (INSERT_TYPE == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; +} + +void* ombProduceThreadFunc(void* param) { + SThreadInfo* pInfo = (SThreadInfo*)param; + + pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (pInfo->taos == NULL) { + printf("taos_connect() fail\n"); + return NULL; + } + + int64_t affectedRowsTotal = 0; + int64_t sendMsgs = 0; + + uint32_t totalSendLoopTimes = g_stConfInfo.runDurationMinutes * 60 * 1000 / SEND_TIME_UNIT; // send some msgs per 10ms + uint32_t batchPerTblTimes = pInfo->producerRate / 100 / g_stConfInfo.batchSize; + uint32_t remainder = (pInfo->producerRate / 100) % g_stConfInfo.batchSize; + if (remainder) { + batchPerTblTimes += 1; + } + + char* sqlBuf = taosMemoryMalloc(MAX_SQL_LEN); + if (NULL == sqlBuf) { + printf("malloc fail for sqlBuf\n"); + return NULL; + } + + printf("Produce Info: totalSendLoopTimes: %d, batchPerTblTimes: %d, producerRate: %d\n", totalSendLoopTimes, batchPerTblTimes, pInfo->producerRate); + + char ctbName[64] = {0}; + sprintf(ctbName, "%s.ctb%d", g_stConfInfo.dbName, pInfo->consumerId); + + int64_t lastPrintTime = taosGetTimestampUs(); + int64_t totalMsgLen = 0; + //int64_t timeStamp = taosGetTimestampUs(); + while (totalSendLoopTimes) { + int64_t startTs = taosGetTimestampUs(); + for (int i = 0; i < batchPerTblTimes; ++i) { + uint32_t msgsOfSql = g_stConfInfo.batchSize; + if ((i == batchPerTblTimes - 1) && (0 != remainder)) { + msgsOfSql = remainder; + } + int len = 0; + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "insert into %s values ", ctbName); + for (int j = 0; j < msgsOfSql; j++) { + int64_t timeStamp = taosGetTimestampNs(); + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); + sendMsgs++; + pInfo->totalProduceMsgs++; + } + + totalMsgLen += len; + pInfo->totalMsgsLen += len; + + int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); + if (affectedRows < 0) { + return NULL; + } + + affectedRowsTotal += affectedRows; + + //printf("Produce Info: affectedRows: %" PRId64 "\n", affectedRows); + } + totalSendLoopTimes -= 1; + + // calc spent time + int64_t currentTs = taosGetTimestampUs(); + int64_t delta = currentTs - startTs; + if (delta < SEND_TIME_UNIT * 1000) { + int64_t sleepLen = (int32_t)(SEND_TIME_UNIT * 1000 - delta); + //printf("sleep %" PRId64 " us, use time: %" PRId64 " us\n", sleepLen, delta); + taosUsleep((int32_t)sleepLen); + } + + currentTs = taosGetTimestampUs(); + delta = currentTs - lastPrintTime; + if (delta > 10 * 1000 * 1000) { + printf("producer[%d] info: %" PRId64 " msgs, %" PRId64 " Byte, %" PRId64 " us, totalSendLoopTimes: %d\n", + pInfo->consumerId, sendMsgs, totalMsgLen, delta, totalSendLoopTimes); + printf("producer[%d] rate: %1.f msgs/s, %1.f KB/s\n", + pInfo->consumerId, + sendMsgs * 1000.0 * 1000 / delta, + (totalMsgLen / 1024.0) / (delta / (1000*1000))); + lastPrintTime = currentTs; + sendMsgs = 0; + totalMsgLen = 0; + } + } + + printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); + return NULL; +} + + +void printProduceInfo(int64_t start) { + int64_t totalMsgs = 0; + int64_t totalLenOfMsgs = 0; + for (int i = 0; i < g_stConfInfo.producers; i++) { + totalMsgs += g_stConfInfo.stProdThreads[i].totalProduceMsgs; + totalLenOfMsgs += g_stConfInfo.stProdThreads[i].totalMsgsLen; + } + + int64_t end = taosGetTimestampUs(); + + int64_t t = end - start; + if (0 == t) t = 1; + + double tInMs = (double)t / 1000000.0; + printf("Spent %.3f seconds to prod %" PRIu64 " msgs, %" PRIu64 " Byte\n\n", tInMs, totalMsgs, totalLenOfMsgs); + + + printf("Spent %.3f seconds to prod %" PRIu64 " msgs with %d producer(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalMsgs, g_stConfInfo.producers, + (double)totalMsgs / tInMs, + (double)totalLenOfMsgs/(1024.0*1024)/tInMs); + return; +} + + +void startOmbConsume() { + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + + if (0 != g_stConfInfo.producers) { + TAOS* taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (taos == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); + ASSERT(0); + return ; + } + + char stbName[16] = "stb"; + char ctbPrefix[16] = "ctb"; + + char sql[256] = {0}; + sprintf(sql, "drop database if exists %s", g_stConfInfo.dbName); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + sprintf(sql, "create database if not exists %s precision 'ns' vgroups %d", g_stConfInfo.dbName, g_stConfInfo.producers); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + sprintf(sql, "create stable %s.%s (ts timestamp, payload binary(%d)) tags (t bigint) ", g_stConfInfo.dbName, stbName, g_stConfInfo.payloadLen); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + for (int i = 0; i < g_stConfInfo.producers; i++) { + sprintf(sql, "create table %s.%s%d using %s.stb tags(%d) ", g_stConfInfo.dbName, ctbPrefix, i, g_stConfInfo.dbName, i); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + } + + // create topic + sprintf(sql, "create topic %s as stable %s.%s", g_stConfInfo.topic, g_stConfInfo.dbName, stbName); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + + int32_t producerRate = ceil(g_stConfInfo.producerRate / g_stConfInfo.producers); + + printf("==== create %d produce thread ====\n", g_stConfInfo.producers); + for (int32_t i = 0; i < g_stConfInfo.producers; ++i) { + g_stConfInfo.stProdThreads[i].consumerId = i; + g_stConfInfo.stProdThreads[i].producerRate = producerRate; + taosThreadCreate(&(g_stConfInfo.stProdThreads[i].thread), &thattr, ombProduceThreadFunc, + (void*)(&(g_stConfInfo.stProdThreads[i]))); + } + + if (0 == g_stConfInfo.numOfThread) { + int64_t start = taosGetTimestampUs(); + for (int32_t i = 0; i < g_stConfInfo.producers; i++) { + taosThreadJoin(g_stConfInfo.stProdThreads[i].thread, NULL); + taosThreadClear(&g_stConfInfo.stProdThreads[i].thread); + } + + printProduceInfo(start); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + return; + } + + } + + // pthread_create one thread to consume + taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); + for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { + g_stConfInfo.stThreads[i].consumerId = i; + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, ombConsumeThreadFunc, + (void*)(&(g_stConfInfo.stThreads[i]))); + } + + int64_t start = taosGetTimestampUs(); + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); + taosThreadClear(&g_stConfInfo.stThreads[i].thread); + } + + int64_t end = taosGetTimestampUs(); + + int64_t totalRows = 0; + int64_t totalMsgs = 0; + int64_t totalLenOfMsgs = 0; + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + totalMsgs += g_stConfInfo.stThreads[i].consumeMsgCnt; + totalLenOfMsgs += g_stConfInfo.stThreads[i].consumeLen; + totalRows += g_stConfInfo.stThreads[i].consumeRowCnt; + } + + int64_t t = end - start; + if (0 == t) t = 1; + + double tInMs = (double)t / 1000000.0; + taosFprintfFile(g_fp, + "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalMsgs, g_stConfInfo.numOfThread, + (double)(totalMsgs / tInMs), + (double)totalLenOfMsgs/(1024*1024)/tInMs); + + printf("Spent %.3f seconds to cons rows: %" PRIu64 " msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalRows, totalMsgs, g_stConfInfo.numOfThread, + (double)(totalMsgs / tInMs), + (double)totalLenOfMsgs/(1024*1024)/tInMs); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + + return; +} + + int main(int32_t argc, char* argv[]) { parseArgument(argc, argv); + + if (0 != strlen(g_stConfInfo.topic)) { + startOmbConsume(); + return 0; + } + getConsumeInfo(); saveConfigToLogFile(); From df1cd5e6e4ea3ca405d441f1b8a88b11d032c067 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Sat, 16 Jul 2022 17:49:29 +0800 Subject: [PATCH 19/88] update --- tests/system-test/2-query/last_row.py | 362 +++++++++++++------------- 1 file changed, 180 insertions(+), 182 deletions(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index d4d4f06563..97d072e72b 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -28,10 +28,10 @@ class TDTestCase: tdSql.execute(f" create database test cachemodel {cache_value} ") tdSql.execute(" use test ") - tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ + tdSql.execute(" create stable test.stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)") for tbnum in range(tbnums): - tbname = "sub_tb_%d"%tbnum + tbname = "test.sub_tb_%d"%tbnum tdSql.execute(" create table %s using stb tags(%d) "%(tbname , tbnum)) ts = self.ts @@ -57,8 +57,8 @@ class TDTestCase: coltypes = tdSql.queryResult for coltype in coltypes: colname = coltype[0] - abs_sql = "select abs({}) from {} order by tbname ".format(colname, tbname) - origin_sql = "select {} from {} order by tbname".format(colname, tbname) + abs_sql = "select abs({}) from {} order by tbname ".format(colname, 'test.'+tbname) + origin_sql = "select {} from {} order by tbname".format(colname, 'test.'+tbname) if coltype[1] in support_types: self.check_result_auto(origin_sql , abs_sql) @@ -69,7 +69,7 @@ class TDTestCase: tdSql.execute(create_db_sql) tdSql.execute("use db") tdSql.execute( - '''create table stb1 + '''create table db.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int) ''' @@ -77,38 +77,38 @@ class TDTestCase: tdSql.execute( ''' - create table t1 + create table db.t1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) ''' ) for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + tdSql.execute(f'create table db.ct{i+1} using stb1 tags ( {i+1} )') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into db.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into db.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + "insert into db.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") tdSql.execute( - "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + "insert into db.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") tdSql.execute( - "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + "insert into db.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") tdSql.execute( - "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + "insert into db.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") tdSql.execute( - "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + "insert into db.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( - "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + "insert into db.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( - "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + "insert into db.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( - f'''insert into t1 values + f'''insert into db.t1 values ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) @@ -132,45 +132,45 @@ class TDTestCase: f"create database if not exists testdb keep 3650 duration 1000 cachemodel {cache_value}") tdSql.execute(" use testdb ") - tdSql.execute(f" create stable stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp , uc1 int unsigned,\ + tdSql.execute(f" create stable testdb.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp , uc1 int unsigned,\ uc2 bigint unsigned ,uc3 smallint unsigned , uc4 tinyint unsigned ) tags( t1 int , t2 bigint , t3 smallint , t4 tinyint , t5 float , t6 double , t7 bool , t8 binary(36)\ , t9 nchar(36) , t10 int unsigned , t11 bigint unsigned ,t12 smallint unsigned , t13 tinyint unsigned ,t14 timestamp ) ") tdSql.execute( ''' - create table t1 + create table testdb.t1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) ''' ) for i in range(4): tdSql.execute( - f'create table ct{i+1} using stb1 tags ( {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" ,{111*i}, {1*i},{1*i},{1*i},now())') + f'create table testdb.ct{i+1} using stb1 tags ( {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" ,{111*i}, {1*i},{1*i},{1*i},now())') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i} )" + f"insert into testdb.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i} )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i})" + f"insert into testdb.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a ,{111*i},{1111*i},{i},{i})" ) tdSql.execute( - "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a ,0,0,0,0)") + "insert into testdb.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a ,0,0,0,0)") tdSql.execute( - "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a , 999 , 9999 , 9 , 9)") + "insert into testdb.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a , 999 , 9999 , 9 , 9)") tdSql.execute( - "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a , 999 , 99999 , 9 , 9)") + "insert into testdb.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a , 999 , 99999 , 9 , 9)") tdSql.execute( - "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a ,999 , 99999 , 9 , 9)") + "insert into testdb.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a ,999 , 99999 , 9 , 9)") tdSql.execute( - "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") + "insert into testdb.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") tdSql.execute( - "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") + "insert into testdb.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL) ") tdSql.execute( - "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL ) ") + "insert into testdb.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL, NULL, NULL ) ") tdSql.execute( - f'''insert into t1 values + f'''insert into testdb.t1 values ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) @@ -221,24 +221,24 @@ class TDTestCase: tdSql.execute("use testdb") # bug need fix - tdSql.error("select last_row(c1 ,NULL) from t1") + tdSql.error("select last_row(c1 ,NULL) from testdb.t1") error_sql_lists = [ - "select last_row from t1", - "select last_row(-+--+c1) from t1", - "select last_row(123--123)==1 from t1", - "select last_row(c1) as 'd1' from t1", - "select last_row(c1 ,NULL) from t1", - "select last_row(,) from t1;", - "select last_row(abs(c1) ab from t1)", - "select last_row(c1) as int from t1", - "select last_row from stb1", - "select last_row(123--123)==1 from stb1", - "select last_row(c1) as 'd1' from stb1", - "select last_row(c1 ,NULL) from stb1", - "select last_row(,) from stb1;", - "select last_row(abs(c1) ab from stb1)", - "select last_row(c1) as int from stb1" + "select last_row from testdb.t1", + "select last_row(-+--+c1) from testdb.t1", + "select last_row(123--123)==1 from testdb.t1", + "select last_row(c1) as 'd1' from testdb.t1", + "select last_row(c1 ,NULL) from testdb.t1", + "select last_row(,) from testdb.t1;", + "select last_row(abs(c1) ab from testdb.t1)", + "select last_row(c1) as int from testdb.t1", + "select last_row from testdb.stb1", + "select last_row(123--123)==1 from testdb.stb1", + "select last_row(c1) as 'd1' from testdb.stb1", + "select last_row(c1 ,NULL) from testdb.stb1", + "select last_row(,) from testdb.stb1;", + "select last_row(abs(c1) ab from testdb.stb1)", + "select last_row(c1) as int from testdb.stb1" ] for error_sql in error_sql_lists: tdSql.error(error_sql) @@ -254,51 +254,51 @@ class TDTestCase: colname = coltype[0] col_note = coltype[-1] if col_note != "TAG": - abs_sql = "select last_row({}) from {}".format(colname, tbname) + abs_sql = "select last_row({}) from {}".format(colname, "testdb."+tbname) tdSql.query(abs_sql) def basic_abs_function(self): # basic query - tdSql.query("select c1 from ct3") + tdSql.query("select c1 from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select c1 from t1") + tdSql.query("select c1 from testdb.t1") tdSql.checkRows(12) - tdSql.query("select c1 from stb1") + tdSql.query("select c1 from testdb.stb1") tdSql.checkRows(25) # used for empty table , ct3 is empty - tdSql.query("select last_row(c1) from ct3") + tdSql.query("select last_row(c1) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c2) from ct3") + tdSql.query("select last_row(c2) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c3) from ct3") + tdSql.query("select last_row(c3) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c4) from ct3") + tdSql.query("select last_row(c4) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c5) from ct3") + tdSql.query("select last_row(c5) from testdb.ct3") tdSql.checkRows(0) - tdSql.query("select last_row(c6) from ct3") + tdSql.query("select last_row(c6) from testdb.ct3") # used for regular table # bug need fix - tdSql.query("select last_row(c1) from t1") + tdSql.query("select last_row(c1) from testdb.t1") tdSql.checkData(0, 0, None) - tdSql.query("select last_row(c1) from ct4") + tdSql.query("select last_row(c1) from testdb.ct4") tdSql.checkData(0, 0, None) - tdSql.query("select last_row(c1) from stb1") + tdSql.query("select last_row(c1) from testdb.stb1") tdSql.checkData(0, 0, None) # # bug need fix - tdSql.query("select last_row(c1), c2, c3 , c4, c5 from t1") + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.t1") tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) tdSql.checkData(0, 2, None) # # bug need fix - tdSql.query("select last_row(c1), c2, c3 , c4, c5 from ct1") + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.ct1") tdSql.checkData(0, 0, 9) tdSql.checkData(0, 1, -99999) tdSql.checkData(0, 2, -999) @@ -306,7 +306,7 @@ class TDTestCase: tdSql.checkData(0, 4,-9.99000) # bug need fix - tdSql.query("select last_row(c1), c2, c3 , c4, c5 from stb1 where tbname='ct1'") + tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.stb1 where tbname='ct1'") tdSql.checkData(0, 0, 9) tdSql.checkData(0, 1, -99999) tdSql.checkData(0, 2, -999) @@ -314,23 +314,23 @@ class TDTestCase: tdSql.checkData(0, 4,-9.99000) # bug fix - tdSql.query("select last_row(abs(c1)) from ct1") + tdSql.query("select last_row(abs(c1)) from testdb.ct1") tdSql.checkData(0,0,9) # # bug fix - tdSql.query("select last_row(c1+1) from ct1") - tdSql.query("select last_row(c1+1) from stb1") - tdSql.query("select last_row(c1+1) from t1") + tdSql.query("select last_row(c1+1) from testdb.ct1") + tdSql.query("select last_row(c1+1) from testdb.stb1") + tdSql.query("select last_row(c1+1) from testdb.t1") # used for stable table - tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from ct1") + tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from testdb.ct1") tdSql.checkData(0,0,9) tdSql.checkData(0,1,-99999) tdSql.checkData(0,2,-999) tdSql.checkData(0,3,None) # bug need fix - tdSql.query("select last_row(c1 ,c2 ,c3) from stb1 ") + tdSql.query("select last_row(c1 ,c2 ,c3) from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,None) @@ -349,97 +349,97 @@ class TDTestCase: # tdSql.checkData(0,4,-99) # bug need fix - tdSql.query("select abs(last_row(c1)-2)+max(c1),ceil(last_row(c4)-2) from stb1 where c4 is not null") + tdSql.query("select abs(last_row(c1)-2)+max(c1),ceil(last_row(c4)-2) from testdb.stb1 where c4 is not null") tdSql.checkData(0,0,16.000000000) tdSql.checkData(0,1,-101.000000000) - tdSql.query("select abs(last_row(c1)-2)+max(c1),ceil(last_row(c4)-2) from ct1 where c4<0") + tdSql.query("select abs(last_row(c1)-2)+max(c1),ceil(last_row(c4)-2) from testdb.ct1 where c4<0") tdSql.checkData(0,0,16.000000000) tdSql.checkData(0,1,-101.000000000) - tdSql.query("select last_row(ceil(c1+2)+floor(c1)-10) from stb1") + tdSql.query("select last_row(ceil(c1+2)+floor(c1)-10) from testdb.stb1") tdSql.checkData(0,0,None) - tdSql.query("select last_row(ceil(c1+2)+floor(c1)-10) from ct1") + tdSql.query("select last_row(ceil(c1+2)+floor(c1)-10) from testdb.ct1") tdSql.checkData(0,0,10.000000000) # filter for last_row # bug need fix - # tdSql.query("select last_row(ts ,c1 ) from stb1 where ts < now-1d ") + # tdSql.query("select last_row(ts ,c1 ) from testdb.stb1 where ts < now-1d ") # tdSql.checkData(0,1,9) - # tdSql.query("select last_row(ts ,c1 ) from ct4 where ts < now-1d ") + # tdSql.query("select last_row(ts ,c1 ) from testdb.ct4 where ts < now-1d ") # tdSql.checkData(0,1,9) # bug need fix for all function - tdSql.query("select last_row(ts ,c1 ) from ct4 where t1 = 1 ") + tdSql.query("select last_row(ts ,c1 ) from testdb.ct4 where t1 = 1 ") tdSql.checkRows(0) - tdSql.query("select count(c1) from ct4 where t1 = 1 ") + tdSql.query("select count(c1) from testdb.ct4 where t1 = 1 ") tdSql.checkRows(0) - tdSql.query("select last_row(c1) ,last(c1) from stb1 where c1 is null") + tdSql.query("select last_row(c1) ,last(c1) from testdb.stb1 where c1 is null") tdSql.checkRows(1) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.query("select last_row(c1) ,count(*) from stb1 where c1 is null") + tdSql.query("select last_row(c1) ,count(*) from testdb.stb1 where c1 is null") tdSql.checkData(0,0,None) tdSql.checkData(0,1,3) - tdSql.query("select last_row(c1) ,count(c1) from stb1 where c1 is null") + tdSql.query("select last_row(c1) ,count(c1) from testdb.stb1 where c1 is null") tdSql.checkData(0,0,None) tdSql.checkData(0,1,0) # bug need fix - tdSql.query("select tbname ,last_row(c1) from stb1") + tdSql.query("select tbname ,last_row(c1) from testdb.stb1") tdSql.checkData(0,0,'ct4') tdSql.checkData(0,1,None) - tdSql.query(" select tbname ,last_row(c1) from stb1 partition by tbname order by tbname ") + tdSql.query(" select tbname ,last_row(c1) from testdb.stb1 partition by tbname order by tbname ") tdSql.checkData(0,0,'ct1') tdSql.checkData(0,1,9) tdSql.checkData(1,0,'ct4') tdSql.checkData(1,1,None) - tdSql.query(" select tbname ,last_row(c1) from stb1 group by tbname order by tbname ") + tdSql.query(" select tbname ,last_row(c1) from testdb.stb1 group by tbname order by tbname ") tdSql.checkData(0,0,'ct1') tdSql.checkData(0,1,9) tdSql.checkData(1,0,'ct4') tdSql.checkData(1,1,None) - tdSql.query(" select t1 ,count(c1) from stb1 partition by t1 ") + tdSql.query(" select t1 ,count(c1) from testdb.stb1 partition by t1 ") tdSql.checkRows(2) # filter by tbname - tdSql.query("select last_row(c1) from stb1 where tbname = 'ct1' ") + tdSql.query("select last_row(c1) from testdb.stb1 where tbname = 'ct1' ") tdSql.checkData(0,0,9) # bug need fix - tdSql.query("select tbname ,last_row(c1) from stb1 where tbname = 'ct1' ") + tdSql.query("select tbname ,last_row(c1) from testdb.stb1 where tbname = 'ct1' ") tdSql.checkData(0,1,9) - tdSql.query("select tbname ,last_row(c1) from stb1 partition by tbname order by tbname") + tdSql.query("select tbname ,last_row(c1) from testdb.stb1 partition by tbname order by tbname") tdSql.checkData(0, 0, 'ct1') tdSql.checkData(0, 1, 9) tdSql.checkData(1, 0, 'ct4') tdSql.checkData(1, 1, None) - tdSql.query("select tbname ,last_row(c1) from stb1 group by tbname order by tbname") + tdSql.query("select tbname ,last_row(c1) from testdb.stb1 group by tbname order by tbname") tdSql.checkData(0, 0, 'ct1') tdSql.checkData(0, 1, 9) tdSql.checkData(1, 0, 'ct4') tdSql.checkData(1, 1, None) # last_row for only tag - tdSql.query("select last_row(t1 ,t2 ,t3 , t4 ) from stb1") + tdSql.query("select last_row(t1 ,t2 ,t3 , t4 ) from testdb.stb1") tdSql.checkData(0,0,3) tdSql.checkData(0,1,33333) tdSql.checkData(0,2,333) tdSql.checkData(0,3,3) - tdSql.query("select last_row(abs(floor(t1)) ,t2 ,ceil(abs(t3)) , abs(ceil(t4)) ) from stb1") + tdSql.query("select last_row(abs(floor(t1)) ,t2 ,ceil(abs(t3)) , abs(ceil(t4)) ) from testdb.stb1") tdSql.checkData(0,0,3) tdSql.checkData(0,1,33333) tdSql.checkData(0,2,333) @@ -448,7 +448,7 @@ class TDTestCase: # filter by tag tdSql.query("select tbname ,last_row(c1) from testdb.stb1 where t1 =0 ") tdSql.checkData(0,1,9) - tdSql.query("select tbname ,last_row(c1) ,t1 from stb1 partition by t1 order by t1") + tdSql.query("select tbname ,last_row(c1) ,t1 from testdb.stb1 partition by t1 order by t1") tdSql.checkData(0, 0, 'ct1') tdSql.checkData(0, 1, 9) tdSql.checkData(1, 0, 'ct4') @@ -456,48 +456,48 @@ class TDTestCase: # filter by col - tdSql.query("select tbname ,last_row(c1),abs(c1)from stb1 where c1 =1;") + tdSql.query("select tbname ,last_row(c1),abs(c1)from testdb.stb1 where c1 =1;") tdSql.checkData(0, 0, 'ct1') tdSql.checkData(0, 1, 1) tdSql.checkData(0, 2, 1) - tdSql.query("select last_row(c1) from stb1 where abs(ceil(c1))*c1==1") + tdSql.query("select last_row(c1) from testdb.stb1 where abs(ceil(c1))*c1==1") tdSql.checkData(0,0,1) # mix with common functions - tdSql.query("select last_row(*) ,last(*) from stb1 ") + tdSql.query("select last_row(*) ,last(*) from testdb.stb1 ") tdSql.checkRows(1) - tdSql.query("select last_row(*) ,last(*) from stb1 ") + tdSql.query("select last_row(*) ,last(*) from testdb.stb1 ") tdSql.checkRows(1) - tdSql.query("select last_row(c1+abs(c1)) from stb1 partition by tbname order by tbname") - tdSql.query("select last(c1), max(c1+abs(c1)),last_row(c1+abs(c1)) from stb1 partition by tbname order by tbname") + tdSql.query("select last_row(c1+abs(c1)) from testdb.stb1 partition by tbname order by tbname") + tdSql.query("select last(c1), max(c1+abs(c1)),last_row(c1+abs(c1)) from testdb.stb1 partition by tbname order by tbname") # # bug need fix ,taosd crash - tdSql.error("select last_row(*) ,last(*) from stb1 partition by tbname order by last(*)") - tdSql.error("select last_row(*) ,last(*) from stb1 partition by tbname order by last_row(*)") + tdSql.error("select last_row(*) ,last(*) from testdb.stb1 partition by tbname order by last(*)") + tdSql.error("select last_row(*) ,last(*) from testdb.stb1 partition by tbname order by last_row(*)") # mix with agg functions - tdSql.query("select last(*), last_row(*),last(c1), last_row(c1) from stb1 ") - tdSql.query("select last(*), last_row(*),last(c1), last_row(c1) from ct1 ") - tdSql.query("select last(*), last_row(*),last(c1+1)*max(c1), last_row(c1+2)/2 from t1 ") - tdSql.query("select last_row(*) ,abs(c1/2)+100 from stb1 where tbname =\"ct1\" ") - tdSql.query("select c1, last_row(c5) from ct1 ") - tdSql.error("select c1, last_row(c5) ,last(c1) from stb1 ") + tdSql.query("select last(*), last_row(*),last(c1), last_row(c1) from testdb.stb1 ") + tdSql.query("select last(*), last_row(*),last(c1), last_row(c1) from testdb.ct1 ") + tdSql.query("select last(*), last_row(*),last(c1+1)*max(c1), last_row(c1+2)/2 from testdb.t1 ") + tdSql.query("select last_row(*) ,abs(c1/2)+100 from testdb.stb1 where tbname =\"ct1\" ") + tdSql.query("select c1, last_row(c5) from testdb.ct1 ") + tdSql.error("select c1, last_row(c5) ,last(c1) from testdb.stb1 ") # agg functions mix with agg functions - tdSql.query("select last(c1) , max(c5), count(c5) from stb1") - tdSql.query("select last_row(c1) , max(c5), count(c5) from ct1") + tdSql.query("select last(c1) , max(c5), count(c5) from testdb.stb1") + tdSql.query("select last_row(c1) , max(c5), count(c5) from testdb.ct1") # bug fix for compute - tdSql.query("select last_row(c1) -0 ,last(c1)-0 ,last(c1)+last_row(c1) from ct4 ") + tdSql.query("select last_row(c1) -0 ,last(c1)-0 ,last(c1)+last_row(c1) from testdb.ct4 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,0.000000000) tdSql.checkData(0,2,None) - tdSql.query(" select c1, abs(c1) -0 ,last_row(c1-0.1)-0.1 from ct1") + tdSql.query(" select c1, abs(c1) -0 ,last_row(c1-0.1)-0.1 from testdb.ct1") tdSql.checkData(0,0,9) tdSql.checkData(0,1,9.000000000) tdSql.checkData(0,2,8.800000000) @@ -505,7 +505,7 @@ class TDTestCase: def abs_func_filter(self): tdSql.execute("use db") tdSql.query( - "select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,last_row(log(c1,2)-0.5) from ct4 where c1>5 ") + "select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,last_row(log(c1,2)-0.5) from db.ct4 where c1>5 ") tdSql.checkData(0, 0, 6) tdSql.checkData(0, 1, 6.000000000) tdSql.checkData(0, 2, 6.000000000) @@ -513,19 +513,19 @@ class TDTestCase: tdSql.checkData(0, 4, 2.084962501) tdSql.query( - "select last_row(c1,c2,c1+5) from ct4 where c1=5 ") + "select last_row(c1,c2,c1+5) from db.ct4 where c1=5 ") tdSql.checkData(0, 0, 5) tdSql.checkData(0, 1, 55555) tdSql.checkData(0, 2, 10.000000000) tdSql.query( - "select last(c1,c2,c1+5) from ct4 where c1=5 ") + "select last(c1,c2,c1+5) from db.ct4 where c1=5 ") tdSql.checkData(0, 0, 5) tdSql.checkData(0, 1, 55555) tdSql.checkData(0, 2, 10.000000000) tdSql.query( - "select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + "select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from db.ct4 where c1>log(c1,2) limit 1 ") tdSql.checkRows(1) tdSql.checkData(0, 0, 8) tdSql.checkData(0, 1, 88888) @@ -544,35 +544,35 @@ class TDTestCase: time.sleep(3) tdSql.execute("use bound_test") tdSql.execute( - "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + "create table bound_test.stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" ) - tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute(f'create table bound_test.sub1_bound using stb_bound tags ( 1 )') tdSql.execute( - f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.error( - f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into bound_test.sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) # check basic elem for table per row tdSql.query( - "select last(c1) ,last_row(c2), last_row(c3)+1 , last(c4)+1 from sub1_bound ") + "select last(c1) ,last_row(c2), last_row(c3)+1 , last(c4)+1 from bound_test.sub1_bound ") tdSql.checkData(0, 0, -2147483646) tdSql.checkData(0, 1, -9223372036854775806) tdSql.checkData(0, 2, -32765.000000000) tdSql.checkData(0, 3, -125.000000000) # check + - * / in functions tdSql.query( - "select last_row(c1+1) ,last_row(c2) , last(c3*1) , last(c4/2) from sub1_bound ") + "select last_row(c1+1) ,last_row(c2) , last(c3*1) , last(c4/2) from bound_test.sub1_bound ") def test_tag_compute_for_scalar_function(self): @@ -580,53 +580,51 @@ class TDTestCase: # bug need fix - tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ") - tdSql.query("select c1 ,t1 from stb1 where t1 =0 ") + tdSql.query(" select sum(c1) from testdb.stb1 where t1+10 >1; ") + tdSql.query("select c1 ,t1 from testdb.stb1 where t1 =0 ") tdSql.checkRows(13) - tdSql.query("select last_row(c1,t1) from stb1 ") + tdSql.query("select last_row(c1,t1) from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,3) - tdSql.query("select last_row(c1),t1 from stb1 ") + tdSql.query("select last_row(c1),t1 from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,3) - tdSql.query("select last_row(c1,t1),last(t1) from stb1 ") + tdSql.query("select last_row(c1,t1),last(t1) from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,3) tdSql.checkData(0,2,3) - tdSql.query("select last_row(t1) from stb1 where t1 >0 ") + tdSql.query("select last_row(t1) from testdb.stb1 where t1 >0 ") tdSql.checkRows(1) tdSql.checkData(0,0,3) - tdSql.query("select last_row(t1) from stb1 where t1 =3 ") + tdSql.query("select last_row(t1) from testdb.stb1 where t1 =3 ") tdSql.checkRows(1) tdSql.checkData(0,0,3) - tdSql.query("select last_row(t1) from stb1 where t1 =2") + tdSql.query("select last_row(t1) from testdb.stb1 where t1 =2") tdSql.checkRows(0) # nest query for last_row - tdSql.query("select last_row(t1) from (select ts , c1 ,t1 from stb1)") - tdSql.checkData(0,0,4) - tdSql.query("select distinct(c1) ,t1 from stb1") + tdSql.query("select last_row(t1) from (select ts , c1 ,t1 from testdb.stb1)") + tdSql.checkData(0,0,3) + tdSql.query("select distinct(c1) ,t1 from testdb.stb1") tdSql.checkRows(20) - tdSql.query("select last_row(c1) from (select _rowts , c1 ,t1 from stb1)") + tdSql.query("select last_row(c1) from (select _rowts , c1 ,t1 from testdb.stb1)") tdSql.checkData(0,0,None) - tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") + tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from testdb.stb1)") tdSql.checkData(0,0,None) - tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from stb1)") - tdSql.checkData(0,1,None,None) + tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from testdb.stb1)") + tdSql.checkData(0,1,None) - tdSql.query("select ts , last_row(c1) ,c1 from (select ts , max(c1) c1 ,t1 from stb1 where ts >now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts ="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" interval(50d) sliding(30d) fill(NULL)') + tdSql.query('select max(c1) from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" interval(50d) sliding(30d) fill(NULL)') tdSql.checkRows(8) tdSql.checkData(7,0,None) - tdSql.query('select last_row(c1) from t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" interval(50d) sliding(30d) fill(value ,2 )') + tdSql.query('select last_row(c1) from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" interval(50d) sliding(30d) fill(value ,2 )') tdSql.checkRows(8) tdSql.checkData(7,0,2) - tdSql.query('select last_row(c1) from stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') - tdSql.query('select last_row(c1) from (select ts , c1 from t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" ) interval(10s) sliding(5s)') + tdSql.query('select last_row(c1) from testdb.stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') + tdSql.query('select last_row(c1) from (select ts , c1 from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" ) interval(10s) sliding(5s)') # join tdSql.query("use test") @@ -791,15 +789,15 @@ class TDTestCase: def support_super_table_test(self): tdSql.execute(" use testdb ") - self.check_result_auto( " select c1 from stb1 order by ts " , "select abs(c1) from stb1 order by ts" ) - self.check_result_auto( " select c1 from stb1 order by tbname " , "select abs(c1) from stb1 order by tbname" ) - self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) - self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname " , "select abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c1 from testdb.stb1 order by ts " , "select abs(c1) from testdb.stb1 order by ts" ) + self.check_result_auto( " select c1 from testdb.stb1 order by tbname " , "select abs(c1) from testdb.stb1 order by tbname" ) + self.check_result_auto( " select c1 from testdb.stb1 where c1 > 0 order by tbname " , "select abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select c1 from testdb.stb1 where c1 > 0 order by tbname " , "select abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) - self.check_result_auto( " select t1,c1 from stb1 order by ts " , "select t1, abs(c1) from stb1 order by ts" ) - self.check_result_auto( " select t2,c1 from stb1 order by tbname " , "select t2 ,abs(c1) from stb1 order by tbname" ) - self.check_result_auto( " select t3,c1 from stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from stb1 where c1 > 0 order by tbname" ) - self.check_result_auto( " select t4,c1 from stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t1,c1 from testdb.stb1 order by ts " , "select t1, abs(c1) from testdb.stb1 order by ts" ) + self.check_result_auto( " select t2,c1 from testdb.stb1 order by tbname " , "select t2 ,abs(c1) from testdb.stb1 order by tbname" ) + self.check_result_auto( " select t3,c1 from testdb.stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) + self.check_result_auto( " select t4,c1 from testdb.stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) pass def basic_query(self): From 853e6b50b217fbb6e4c70eccb2cd226ca6ae564c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 16 Jul 2022 09:50:18 +0000 Subject: [PATCH 20/88] fix vnode snapshot state --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 26 +++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index c42c080fb8..420cf3c473 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -175,6 +175,7 @@ _err: int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) { int32_t code = 0; + SVnode *pVnode = pWriter->pVnode; if (pWriter->pMetaSnapWriter) { code = metaSnapWriterClose(&pWriter->pMetaSnapWriter, rollback); @@ -186,8 +187,31 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) { if (code) goto _err; } + if (!rollback) { + SVnodeInfo info = {0}; + char dir[TSDB_FILENAME_LEN]; + + pVnode->state.committed = pWriter->ever; + pVnode->state.applied = pWriter->ever; + // pVnode->state.applyTerm = ; + // pVnode->state.commitTerm = ; + + info.config = pVnode->config; + info.state.committed = pVnode->state.applied; + info.state.commitTerm = pVnode->state.applyTerm; + info.state.commitID = pVnode->state.commitID; + snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); + code = vnodeSaveInfo(dir, &info); + if (code) goto _err; + + code = vnodeCommitInfo(dir, &info); + if (code) goto _err; + } else { + ASSERT(0); + } + _exit: - vInfo("vgId:%d vnode snapshot writer closed, rollback:%d", TD_VID(pWriter->pVnode), rollback); + vInfo("vgId:%d vnode snapshot writer closed, rollback:%d", TD_VID(pVnode), rollback); taosMemoryFree(pWriter); return code; From fdf7930702c599b0ce3c669815f7890ebaf340b4 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 16 Jul 2022 18:09:36 +0800 Subject: [PATCH 21/88] refactor(sync): add index/term in snapshot write --- include/libs/sync/sync.h | 2 +- source/dnode/mnode/impl/src/mndSync.c | 2 +- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- source/libs/sync/src/syncSnapshot.c | 25 +++++++++++++------------ 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index c226d7c8cc..06a58af0e3 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -134,7 +134,7 @@ typedef struct SSyncFSM { int32_t (*FpSnapshotDoRead)(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len); int32_t (*FpSnapshotStartWrite)(struct SSyncFSM* pFsm, void* pWriterParam, void** ppWriter); - int32_t (*FpSnapshotStopWrite)(struct SSyncFSM* pFsm, void* pWriter, bool isApply); + int32_t (*FpSnapshotStopWrite)(struct SSyncFSM* pFsm, void* pWriter, bool isApply, SSnapshot* pSnapshot); int32_t (*FpSnapshotDoWrite)(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len); } SSyncFSM; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 2811aeb43a..36362bed87 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -143,7 +143,7 @@ int32_t mndSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void **ppWrit return sdbStartWrite(pMnode->pSdb, (SSdbIter **)ppWriter); } -int32_t mndSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { +int32_t mndSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) { mInfo("stop to apply snapshot to sdb, apply:%d", isApply); SMnode *pMnode = pFsm->data; return sdbStopWrite(pMnode->pSdb, pWriter, isApply); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 81177fd5d8..a799965a9d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -515,7 +515,7 @@ static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void #endif } -static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { +static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) { #ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; int32_t code = vnodeSnapWriterClose(pWriter, !isApply); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 87cc5685f3..924a4df90d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -374,14 +374,14 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) { int32_t len = 256; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; char host[64]; @@ -434,8 +434,8 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { if (pReceiver != NULL) { // close writer if (pReceiver->pWriter != NULL) { - int32_t ret = - pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, + false, &(pReceiver->snapshot)); ASSERT(ret == 0); pReceiver->pWriter = NULL; } @@ -483,8 +483,8 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapsh static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { // force close, abandon incomplete data if (pReceiver->pWriter != NULL) { - int32_t ret = - pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false, + &(pReceiver->snapshot)); ASSERT(ret == 0); pReceiver->pWriter = NULL; } @@ -524,8 +524,8 @@ int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend // FpSnapshotStopWrite should not be called, assert writer == NULL int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { if (pReceiver->pWriter != NULL) { - int32_t ret = - pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false, + &(pReceiver->snapshot)); ASSERT(ret == 0); pReceiver->pWriter = NULL; } @@ -574,7 +574,8 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap } // stop writer, apply data - code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true); + code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true, + &(pReceiver->snapshot)); if (code != 0) { syncNodeErrorLog(pReceiver->pSyncNode, "snapshot stop writer true error"); ASSERT(0); @@ -646,7 +647,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { cJSON_AddStringToObject(pFromId, "addr", u64buf); { uint64_t u64 = pReceiver->fromId.addr; - cJSON * pTmp = pFromId; + cJSON *pTmp = pFromId; char host[128] = {0}; uint16_t port; syncUtilU642Addr(u64, host, sizeof(host), &port); @@ -679,14 +680,14 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event) { int32_t len = 256; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); SRaftId fromId = pReceiver->fromId; char host[128]; From 604e5b750c5672bd21b4793249c26383c88ec9bc Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 16 Jul 2022 18:19:54 +0800 Subject: [PATCH 22/88] refactor(sync): add index/term in vnodeSnapWriterClose --- source/dnode/vnode/inc/vnode.h | 12 ++++++------ source/dnode/vnode/src/vnd/vnodeSnapshot.c | 2 +- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6b5f795eb0..57141bbbb5 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -116,11 +116,11 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur); // typedef struct STsdb STsdb; typedef struct STsdbReader STsdbReader; -#define BLOCK_LOAD_OFFSET_ORDER 1 +#define BLOCK_LOAD_OFFSET_ORDER 1 #define BLOCK_LOAD_TABLESEQ_ORDER 2 -#define BLOCK_LOAD_EXTERN_ORDER 3 +#define BLOCK_LOAD_EXTERN_ORDER 3 -#define LASTROW_RETRIEVE_TYPE_ALL 0x1 +#define LASTROW_RETRIEVE_TYPE_ALL 0x1 #define LASTROW_RETRIEVE_TYPE_SINGLE 0x2 int32_t tsdbSetTableId(STsdbReader *pReader, int64_t uid); @@ -191,7 +191,7 @@ int32_t vnodeSnapReaderClose(SVSnapReader *pReader); int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData); // SVSnapWriter int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter); -int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback); +int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot); int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData); // structs @@ -237,8 +237,8 @@ typedef struct { uint64_t groupId; } STableKeyInfo; -#define TABLE_ROLLUP_ON ((int8_t)0x1) -#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) +#define TABLE_ROLLUP_ON ((int8_t)0x1) +#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) #define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON) struct SMetaEntry { int64_t version; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index c42c080fb8..2466dd5538 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -173,7 +173,7 @@ _err: return code; } -int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) { +int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot) { int32_t code = 0; if (pWriter->pMetaSnapWriter) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index a799965a9d..712cee9fd0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -518,7 +518,7 @@ static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) { #ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; - int32_t code = vnodeSnapWriterClose(pWriter, !isApply); + int32_t code = vnodeSnapWriterClose(pWriter, !isApply, pSnapshot); return code; #else taosMemoryFree(pWriter); From 9ec14ece62f08c3343e728e84434b8b8af52f7b1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 16 Jul 2022 18:58:29 +0800 Subject: [PATCH 23/88] fix: fix taosd mem leak --- include/libs/function/function.h | 7 +--- source/dnode/vnode/src/tsdb/tsdbRead.c | 1 + source/libs/executor/src/dataInserter.c | 3 ++ source/libs/executor/src/executorimpl.c | 4 +- source/libs/executor/src/sortoperator.c | 2 + source/libs/function/src/tudf.c | 1 + source/libs/nodes/src/nodesUtilFuncs.c | 1 + source/libs/scalar/src/scalar.c | 40 +++++++++---------- .../libs/scalar/test/filter/filterTests.cpp | 2 +- tests/script/tsim/valgrind/basic1.sim | 5 +-- 10 files changed, 34 insertions(+), 32 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 4d27325d75..8cb48cc9f0 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -172,13 +172,8 @@ typedef struct tExprNode { void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); -typedef enum { - SHOULD_FREE_COLDATA = 0x1, // the newly created column data needs to be destroyed. - DELEGATED_MGMT_COLDATA = 0x2, // input column data should not be released. -} ECOLDATA_MGMT_TYPE_E; - struct SScalarParam { - ECOLDATA_MGMT_TYPE_E type; + bool colAlloced; SColumnInfoData *columnData; SHashObj *pHashFilter; int32_t hashValueType; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f0aea0cefb..1efacbcebb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3186,6 +3186,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 *suid = 0; if (mr.me.type == TSDB_CHILD_TABLE) { + tDecoderClear(&mr.coder); *suid = mr.me.ctbEntry.suid; code = metaGetTableEntryByUid(&mr, *suid); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index e53c9fae6f..a575e355f1 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -90,6 +90,7 @@ _return: tsem_post(&pInserter->ready); + taosMemoryFree(pMsg->pData); taosMemoryFree(param); return TSDB_CODE_SUCCESS; @@ -283,6 +284,8 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize); taosArrayDestroy(pInserter->pDataBlocks); taosMemoryFree(pInserter->pSchema); + taosMemoryFree(pInserter->pParam); + taosHashCleanup(pInserter->pCols); taosThreadMutexDestroy(&pInserter->mutex); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c8f2083456..9ce0f73ec8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -624,7 +624,8 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; ASSERT(pResult->info.capacity > 0); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); - + colDataDestroy(&idata); + numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) { @@ -679,6 +680,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; ASSERT(pResult->info.capacity > 0); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); + colDataDestroy(&idata); numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 9795907404..748ceb6a62 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -234,6 +234,7 @@ void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); + tsortDestroySortHandle(pInfo->pSortHandle); taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pColMatchInfo); @@ -674,6 +675,7 @@ void destroyMultiwayMergeOperatorInfo(void* param, int32_t numOfOutput) { pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock); + tsortDestroySortHandle(pInfo->pSortHandle); taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pColMatchInfo); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index c16c3e3937..6ad552576c 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -855,6 +855,7 @@ int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { memcpy(output->columnData, taosArrayGet(input->pDataBlock, 0), sizeof(SColumnInfoData)); + output->colAlloced = true; return 0; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 7265e7ee78..e0a6ce67ee 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -952,6 +952,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { SQueryInserterNode* pSink = (SQueryInserterNode*)pNode; destroyDataSinkNode((SDataSinkNode*)pSink); + nodesDestroyList(pSink->pCols); break; } case QUERY_NODE_PHYSICAL_PLAN_DELETE: { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 484d95cb5a..eb03ee1678 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -55,7 +55,7 @@ int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarPara } pParam->columnData = pColumnData; - pParam->type = SHOULD_FREE_COLDATA; + pParam->colAlloced = true; return TSDB_CODE_SUCCESS; } @@ -166,7 +166,7 @@ void sclFreeRes(SHashObj *res) { } void sclFreeParam(SScalarParam *param) { - if (param->columnData != NULL) { + if (param->columnData != NULL && param->colAlloced) { colDataDestroy(param->columnData); taosMemoryFreeClear(param->columnData); } @@ -191,6 +191,19 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { return TSDB_CODE_SUCCESS; } +void sclFreeParamList(SScalarParam *param, int32_t paramNum) { + if (NULL == param) { + return; + } + + for (int32_t i = 0; i < paramNum; ++i) { + SScalarParam* p = param + i; + sclFreeParam(p); + } + + taosMemoryFree(param); +} + int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { switch (nodeType(node)) { case QUERY_NODE_LEFT_VALUE: { @@ -274,6 +287,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } *param = *res; + param->colAlloced = false; break; } default: @@ -455,11 +469,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp _return: - for (int32_t i = 0; i < paramNum; ++i) { -// sclFreeParamNoData(params + i); - } - - taosMemoryFreeClear(params); + sclFreeParamList(params, paramNum); SCL_RET(code); } @@ -533,11 +543,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o _return: - for (int32_t i = 0; i < paramNum; ++i) { -// sclFreeParamNoData(params + i); - } - - taosMemoryFreeClear(params); + sclFreeParamList(params, paramNum); SCL_RET(code); } @@ -573,14 +579,8 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp code = terrno; _return: - for (int32_t i = 0; i < paramNum; ++i) { - if (params[i].type == SHOULD_FREE_COLDATA) { - colDataDestroy(params[i].columnData); - taosMemoryFreeClear(params[i].columnData); - } - } - taosMemoryFreeClear(params); + sclFreeParamList(params, paramNum); SCL_RET(code); } @@ -871,7 +871,6 @@ EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } - output.type = DELEGATED_MGMT_COLDATA; if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; @@ -906,7 +905,6 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } - output.type = DELEGATED_MGMT_COLDATA; if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index d8c64eef55..4c4d03fb37 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -207,7 +207,7 @@ void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { void initScalarParam(SScalarParam* pParam) { memset(pParam, 0, sizeof(SScalarParam)); - pParam->type = SHOULD_FREE_COLDATA; + pParam->colAlloced = true; } } diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 3e39f35fa7..3c47cae5cc 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -21,12 +21,11 @@ sql create table tb4 using st1 tags(4); sql insert into tb4 select * from tb1; -goto _OVER - sql select * from tb4; if $rows != 2 then return -1 endi + sql insert into tb4 select ts,f1,f2 from st1; sql select * from tb4; if $rows != 6 then @@ -59,4 +58,4 @@ endi if $system_content == $null then return -1 -endi \ No newline at end of file +endi From f0d1f393fd06937e316a6e2ba3008a2c8f302dd0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 16 Jul 2022 23:24:59 +0800 Subject: [PATCH 24/88] fix(query): add limit/offset implementation in exchange operator. --- source/common/src/tdatablock.c | 4 +- source/libs/executor/inc/executorimpl.h | 26 +++-- source/libs/executor/src/executil.c | 19 ++++ source/libs/executor/src/executorimpl.c | 131 ++++++++++++++---------- source/libs/executor/src/sortoperator.c | 14 +-- 5 files changed, 124 insertions(+), 70 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index cd21de52ed..3da12d97f0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1479,10 +1479,12 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s } beigin++; } + if (dataOffset > 0) { memmove(pColInfoData->pData, pColInfoData->pData + dataOffset, dataLen); - memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[start], (end - start) * sizeof(int32_t)); } + + memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[start], (end - start) * sizeof(int32_t)); return dataLen; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index cd50a13edd..2ffb1be260 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -247,6 +247,16 @@ typedef struct SLoadRemoteDataInfo { uint64_t totalElapsed; // total elapsed time } SLoadRemoteDataInfo; +typedef struct SLimitInfo { + SLimit limit; + SLimit slimit; + uint64_t currentGroupId; + int64_t remainGroupOffset; + int64_t numOfOutputGroups; + int64_t remainOffset; + int64_t numOfOutputRows; +} SLimitInfo; + typedef struct SExchangeInfo { SArray* pSources; SArray* pSourceDataInfo; @@ -257,6 +267,7 @@ typedef struct SExchangeInfo { int32_t current; SLoadRemoteDataInfo loadInfo; uint64_t self; + SLimitInfo limitInfo; } SExchangeInfo; typedef struct SColMatchInfo { @@ -542,15 +553,7 @@ typedef struct SProjectOperatorInfo { SNode* pFilterNode; // filter info, which is push down by optimizer SSDataBlock* existDataBlock; SArray* pPseudoColInfo; - SLimit limit; - SLimit slimit; - - uint64_t groupId; - int64_t curSOffset; - int64_t curGroupOutput; - - int64_t curOffset; - int64_t curOutput; + SLimitInfo limitInfo; } SProjectOperatorInfo; typedef struct SIndefOperatorInfo { @@ -791,6 +794,9 @@ int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInf const char* pkey); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); +int32_t handleLimitOffset(SOperatorInfo *pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); +bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); +void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo); void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); @@ -837,7 +843,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 061ac23905..17484ec84b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -898,4 +898,23 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI } return w; +} + +bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo) { + return (pLimitInfo->limit.limit != -1 || pLimitInfo->limit.offset != -1 || pLimitInfo->slimit.limit != -1 || + pLimitInfo->slimit.offset != -1); +} + + +static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } +static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } + +void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo) { + SLimit limit = {.limit = getLimit(pLimit), .offset = getOffset(pLimit)}; + SLimit slimit = {.limit = getLimit(pSLimit), .offset = getOffset(pSLimit)}; + + pLimitInfo->limit = limit; + pLimitInfo->slimit= slimit; + pLimitInfo->remainOffset = limit.offset; + pLimitInfo->remainGroupOffset = slimit.offset; } \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b8612960e4..933beb9479 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -43,6 +43,11 @@ #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) +enum { + PROJECT_RETRIEVE_CONTINUE = 0x1, + PROJECT_RETRIEVE_DONE = 0x2, +}; + #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = taosRand(); @@ -2340,7 +2345,7 @@ static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { +static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2366,6 +2371,44 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { } } +static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + while(1) { + SSDataBlock* pBlock = doLoadRemoteDataImpl(pOperator); + if (pBlock == NULL) { + return NULL; + } + + ASSERT(pBlock == pExchangeInfo->pResult); + + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; + if (hasLimitOffsetInfo(pLimitInfo)) { + int32_t status = handleLimitOffset(pOperator, pLimitInfo, pExchangeInfo->pResult, false); + if (status == PROJECT_RETRIEVE_CONTINUE) { + continue; + } else if (status == PROJECT_RETRIEVE_DONE) { + size_t rows = pExchangeInfo->pResult->info.rows; + pExchangeInfo->limitInfo.numOfOutputRows += rows; + + if (rows == 0) { + doSetOperatorCompleted(pOperator); + return NULL; + } else { + return pExchangeInfo->pResult; + } + } + } else { + return pExchangeInfo->pResult; + } + } +} + static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const char* id) { pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); if (pInfo->pSourceDataInfo == NULL) { @@ -2405,6 +2448,7 @@ static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* taosArrayPush(pInfo->pSources, pNode); } + initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo); pInfo->self = taosAddRef(exchangeObjRefPool, pInfo); return initDataSource(numOfSources, pInfo, id); @@ -3148,68 +3192,60 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) { return TDB_CODE_SUCCESS; } -enum { - PROJECT_RETRIEVE_CONTINUE = 0x1, - PROJECT_RETRIEVE_DONE = 0x2, -}; - -static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) { - SProjectOperatorInfo* pProjectInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pProjectInfo->binfo; - SSDataBlock* pRes = pInfo->pRes; - - if (pProjectInfo->curSOffset > 0) { - if (pProjectInfo->groupId == 0) { // it is the first group - pProjectInfo->groupId = pBlock->info.groupId; - blockDataCleanup(pInfo->pRes); +int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) { + if (pLimitInfo->remainGroupOffset > 0) { + if (pLimitInfo->currentGroupId == 0) { // it is the first group + pLimitInfo->currentGroupId = pBlock->info.groupId; + blockDataCleanup(pBlock); return PROJECT_RETRIEVE_CONTINUE; - } else if (pProjectInfo->groupId != pBlock->info.groupId) { - pProjectInfo->curSOffset -= 1; + } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { + // now it is the data from a new group + pLimitInfo->remainGroupOffset -= 1; // ignore data block in current group - if (pProjectInfo->curSOffset > 0) { - blockDataCleanup(pInfo->pRes); + if (pLimitInfo->remainGroupOffset > 0) { + blockDataCleanup(pBlock); return PROJECT_RETRIEVE_CONTINUE; } } // set current group id of the project operator - pProjectInfo->groupId = pBlock->info.groupId; + pLimitInfo->currentGroupId = pBlock->info.groupId; } - if (pProjectInfo->groupId != 0 && pProjectInfo->groupId != pBlock->info.groupId) { - pProjectInfo->curGroupOutput += 1; - if ((pProjectInfo->slimit.limit > 0) && (pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput)) { + if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { + pLimitInfo->numOfOutputGroups += 1; + if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { pOperator->status = OP_EXEC_DONE; - blockDataCleanup(pRes); + blockDataCleanup(pBlock); return PROJECT_RETRIEVE_DONE; } // reset the value for a new group data - pProjectInfo->curOffset = 0; - pProjectInfo->curOutput = 0; + pLimitInfo->numOfOutputRows = 0; + pLimitInfo->remainOffset = pLimitInfo->limit.offset; } // here we reach the start position, according to the limit/offset requirements. // set current group id - pProjectInfo->groupId = pBlock->info.groupId; + pLimitInfo->currentGroupId = pBlock->info.groupId; - if (pProjectInfo->curOffset >= pRes->info.rows) { - pProjectInfo->curOffset -= pRes->info.rows; - blockDataCleanup(pRes); + if (pLimitInfo->remainOffset >= pBlock->info.rows) { + pLimitInfo->remainOffset -= pBlock->info.rows; + blockDataCleanup(pBlock); return PROJECT_RETRIEVE_CONTINUE; - } else if (pProjectInfo->curOffset < pRes->info.rows && pProjectInfo->curOffset > 0) { - blockDataTrimFirstNRows(pRes, pProjectInfo->curOffset); - pProjectInfo->curOffset = 0; + } else if (pLimitInfo->remainOffset < pBlock->info.rows && pLimitInfo->remainOffset > 0) { + blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); + pLimitInfo->remainOffset = 0; } // check for the limitation in each group - if (pProjectInfo->limit.limit >= 0 && pProjectInfo->curOutput + pRes->info.rows >= pProjectInfo->limit.limit) { - int32_t keepRows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput); - blockDataKeepFirstNRows(pRes, keepRows); - if (pProjectInfo->slimit.limit > 0 && pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput) { + if (pLimitInfo->limit.limit >= 0 && pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) { + int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); + blockDataKeepFirstNRows(pBlock, keepRows); + if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { pOperator->status = OP_EXEC_DONE; } @@ -3219,8 +3255,8 @@ static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) // todo optimize performance // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the // they may not belong to the same group the limit/offset value is not valid in this case. - if (pRes->info.rows >= pOperator->resultInfo.threshold || pProjectInfo->slimit.offset != -1 || - pProjectInfo->slimit.limit != -1) { + if ((!holdDataInBuf) || (pBlock->info.rows >= pOperator->resultInfo.threshold) || pLimitInfo->slimit.offset != -1 || + pLimitInfo->slimit.limit != -1) { return PROJECT_RETRIEVE_DONE; } else { // not full enough, continue to accumulate the output data in the buffer. return PROJECT_RETRIEVE_CONTINUE; @@ -3306,7 +3342,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, code); } - int32_t status = handleLimitOffset(pOperator, pBlock); + int32_t status = handleLimitOffset(pOperator, &pProjectInfo->limitInfo, pInfo->pRes, true); // filter shall be applied after apply functions and limit/offset on the result doFilter(pProjectInfo->pFilterNode, pInfo->pRes); @@ -3318,9 +3354,9 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } } - pProjectInfo->curOutput += pInfo->pRes->info.rows; - size_t rows = pInfo->pRes->info.rows; + pProjectInfo->limitInfo.numOfOutputRows += rows; + pOperator->resultInfo.totalRows += rows; if (pOperator->cost.openCost == 0) { @@ -3764,10 +3800,6 @@ static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) return pList; } -static int64_t getLimit(SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } - -static int64_t getOffset(SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } - SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo) { SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); @@ -3780,13 +3812,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys SExprInfo* pExprInfo = createExprInfo(pProjPhyNode->pProjections, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pProjPhyNode->node.pOutputDataBlockDesc); - SLimit limit = {.limit = getLimit(pProjPhyNode->node.pLimit), .offset = getOffset(pProjPhyNode->node.pLimit)}; - SLimit slimit = {.limit = getLimit(pProjPhyNode->node.pSlimit), .offset = getOffset(pProjPhyNode->node.pSlimit)}; + initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo); - pInfo->limit = limit; - pInfo->slimit = slimit; - pInfo->curOffset = limit.offset; - pInfo->curSOffset = slimit.offset; pInfo->binfo.pRes = pResBlock; pInfo->pFilterNode = pProjPhyNode->node.pConditions; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 229d7dd670..a3b79d9597 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -22,31 +22,31 @@ static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, - SExecTaskInfo* pTaskInfo) { +// todo add limit/offset impl +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) { goto _error; } - SDataBlockDescNode* pDescNode = pSortPhyNode->node.pOutputDataBlockDesc; + SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc; int32_t numOfCols = 0; SSDataBlock* pResBlock = createResDataBlock(pDescNode); - SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); + SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); int32_t numOfOutputCols = 0; SArray* pColMatchColInfo = - extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); + extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); pInfo->binfo.pRes = pResBlock; initResultSizeInfo(pOperator, 1024); - pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); - pInfo->pCondition = pSortPhyNode->node.pConditions; + pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys); + pInfo->pCondition = pSortNode->node.pConditions; pInfo->pColMatchInfo = pColMatchColInfo; pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; From cc4332d5ac5268885a8e529e787393ed13d35538 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 17 Jul 2022 11:29:20 +0800 Subject: [PATCH 25/88] test:update the error threshold. --- tests/script/tsim/valgrind/checkError1.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index 2b60d7a890..a13cb3a1ca 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -152,7 +152,7 @@ endi system_content sh/checkValgrind.sh -n dnode2 print cmd return result ----> [ $system_content ] -if $system_content > 2 then +if $system_content > 6 then return -1 endi From fc5544d42a13abb8feb0085fc7c7fda61dd567e1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 17 Jul 2022 12:28:55 +0800 Subject: [PATCH 26/88] fix(query):fix the invalid returned code in the compare function --- source/libs/executor/src/groupoperator.c | 22 ++++++++++++++-------- source/libs/executor/src/tsort.c | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index c266e8be23..2964948e70 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -440,19 +440,19 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); - SDataGroupInfo* pGInfo = NULL; - void *pPage = getCurrentDataGroupInfo(pInfo, &pGInfo, len); + SDataGroupInfo* pGroupInfo = NULL; + void *pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len); - pGInfo->numOfRows += 1; - if (pGInfo->groupId == 0) { - pGInfo->groupId = calcGroupId(pInfo->keyBuf, len); + pGroupInfo->numOfRows += 1; + + // group id + if (pGroupInfo->groupId == 0) { + pGroupInfo->groupId = calcGroupId(pInfo->keyBuf, len); } // number of rows int32_t* rows = (int32_t*) pPage; - // group id - size_t numOfCols = pOperator->exprSupp.numOfExprs; for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; @@ -603,7 +603,13 @@ static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { static int compareDataGroupInfo(const void* group1, const void* group2) { const SDataGroupInfo* pGroupInfo1 = group1; const SDataGroupInfo* pGroupInfo2 = group2; - return pGroupInfo1->groupId - pGroupInfo2->groupId; + + if (pGroupInfo1->groupId == pGroupInfo2->groupId) { + ASSERT(0); + return 0; + } + + return (pGroupInfo1->groupId < pGroupInfo2->groupId)? -1:1; } static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index cd8470883b..4f525441b9 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -91,7 +91,7 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page tsortSetComparFp(pSortHandle, msortComparFn); if (idstr != NULL) { - pSortHandle->idStr = strdup(idstr); + pSortHandle->idStr = strdup(idstr); } return pSortHandle; From 107cb15dbd3b5aebd3c3cf2fe7e358f1b44b2119 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sun, 17 Jul 2022 22:52:40 +0800 Subject: [PATCH 27/88] feat(query): add stateduration function scalar version --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 24 ++++++++++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 9bc6aa8351..ab53c37279 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -115,6 +115,7 @@ int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 992fdbd137..aba1df216e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2430,6 +2430,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, .processFunc = stateDurationFunction, + .sprocessFunc = stateDurationScalarFunction, .finalizeFunc = NULL }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 2e32d903aa..7dcba0e1b4 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2577,3 +2577,27 @@ int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; } + +int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int8_t op = getStateOpType(varDataVal(pInput[1].columnData->pData)); + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); + continue; + } + + bool ret = checkStateOp(op, pInputData, i, &pInput[2]); + int64_t out = -1; + if (ret) { + out = 0; + } + colDataAppend(pOutputData, i, (char*)&out, false); + } + + pOutput->numOfRows = pInput->numOfRows; + return TSDB_CODE_SUCCESS; +} From 17610f98a02f3b70200d3744e60293263ac759be Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 18 Jul 2022 08:43:30 +0800 Subject: [PATCH 28/88] update --- tests/system-test/1-insert/update_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 387fc18774..d2151976c8 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -108,6 +108,8 @@ class TDTestCase: tdSql.execute(f'flush database {dbname}') tdSql.execute('reset query cache') self.data_check(tbname,col_name,col_type,value) + for func in ['first','last']: + tdSql.execute(f'select {func}({col_name}) from {tbname}') def error_check(self,tbname,column_dict,tb_type=None,stbname=None): str_length = self.str_length+1 for col_name,col_type in column_dict.items(): From 3101633bbffed76461a66d607abe11a15c02ba9e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 18 Jul 2022 08:47:05 +0800 Subject: [PATCH 29/88] fix: fix scalar crash issue --- source/libs/scalar/src/scalar.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index eb03ee1678..14406a26ed 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -166,13 +166,18 @@ void sclFreeRes(SHashObj *res) { } void sclFreeParam(SScalarParam *param) { - if (param->columnData != NULL && param->colAlloced) { + if (!param->colAlloced) { + return; + } + + if (param->columnData != NULL) { colDataDestroy(param->columnData); taosMemoryFreeClear(param->columnData); } if (param->pHashFilter != NULL) { taosHashCleanup(param->pHashFilter); + param->pHashFilter = NULL; } } @@ -238,11 +243,14 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type)); param->hashValueType = type; + param->colAlloced = true; if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pHashFilter); + param->pHashFilter = NULL; sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); return TSDB_CODE_QRY_OUT_OF_MEMORY; } + param->colAlloced = false; break; } case QUERY_NODE_COLUMN: { From da5e5c52e0c8a709cc8b08430eaf7ab3b68c2778 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 18 Jul 2022 08:53:42 +0800 Subject: [PATCH 30/88] add delete_data.py --- tests/system-test/1-insert/delete_data.py | 88 +++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/system-test/1-insert/delete_data.py diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py new file mode 100644 index 0000000000..6c0c2d24b7 --- /dev/null +++ b/tests/system-test/1-insert/delete_data.py @@ -0,0 +1,88 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db_test' + self.setsql = TDSetSql() + self.ntbname = 'ntb' + self.rowNum = 10 + self.tbnum = 20 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + # first column must be timestamp + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + + } + self.value = [ + f'1' + ] + + self.param_list = [1,100] + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) + for i in range(row_num): + insert_list = [] + + def delete_all_data(self,tbname): + tdSql.execute(f'delete from {tbname}') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(0) + + def delete_one_row(self,tbname,column_dict): + for column_name,column_type in column_dict.items(): + if column_type.lower() == 'timestamp': + + tdSql.execute(f'delete from {tbname} where {column_name}={self.ts}') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(self.rowNum-1) + tdSql.query(f'select * from {tbname} where {column_name}={self.ts}') + tdSql.checkRows(0) + tdSql.execute(f'insert into {tbname} values({self.ts},{self.value_list[0]}') + tdSql.query(f'select * from {tbname} where {column_name}={self.ts}') + tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f"))) + + def delete_data_ntb(self): + tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'use {self.dbname}') + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + for i in range(self.rowNum): + tdSql.execute(f'insert into {self.ntbname} values({self.ts+i},{self.value_list[0]})') + + + # self.delete_all_data(self.ntbname) + + def run(self): + self.delete_data_ntb() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 3f8d9f347eb94d1e770c227f41a4f7422a3c497b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jul 2022 09:29:21 +0800 Subject: [PATCH 31/88] test: valgrind case --- tests/script/tsim/valgrind/checkError2.sim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/script/tsim/valgrind/checkError2.sim b/tests/script/tsim/valgrind/checkError2.sim index 2f83ea4bc6..e81d702d82 100644 --- a/tests/script/tsim/valgrind/checkError2.sim +++ b/tests/script/tsim/valgrind/checkError2.sim @@ -78,18 +78,18 @@ sql select max(c1), max(c2), max(c3) from ct1 sql select sum(c1), sum(c2), sum(c3) from ct1 print =============== step9: insert select -#sql create table ct4 using stb tags(4000); -#sql insert into ct4 select * from ct1; -#sql select * from ct4; -#sql insert into ct4 select ts,c1,c2,c3 from stb; +sql create table ct4 using stb tags(4000); +sql insert into ct4 select * from ct1; +sql select * from ct4; +sql insert into ct4 select ts,c1,c2,c3 from stb; -#sql create table tb1 (ts timestamp, c1 int, c2 float, c3 double); -#sql insert into tb1 (ts, c1, c2, c3) select * from ct1; -#sql select * from tb1; +sql create table tb1 (ts timestamp, c1 int, c2 float, c3 double); +sql insert into tb1 (ts, c1, c2, c3) select * from ct1; +sql select * from tb1; -#sql create table tb2 (ts timestamp, f1 binary(10), c1 int, c2 double); -#sql insert into tb2 (c2, c1, ts) select c2+1, c1, ts+3 from ct2; -#sql select * from tb2; +sql create table tb2 (ts timestamp, f1 binary(10), c1 int, c2 double); +sql insert into tb2 (c2, c1, ts) select c2+1, c1, ts+3 from ct2; +sql select * from tb2; _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT From 677fd4764fd7307e79514e418a74ec6f01327ab1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 18 Jul 2022 09:42:40 +0800 Subject: [PATCH 32/88] fix: dismiss memory issues --- source/common/src/tmsg.c | 4 ++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index aeb83d3425..a95d58d353 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4919,8 +4919,8 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1; if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1; if (tDecodeI8(pCoder, &pReq->rollup) < 0) return -1; - if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1; - if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; + if (tDecodeSSchemaWrapperEx(pCoder, &pReq->schemaRow) < 0) return -1; + if (tDecodeSSchemaWrapperEx(pCoder, &pReq->schemaTag) < 0) return -1; if (pReq->rollup) { if (tDecodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index abe1f37a8c..b49e12fc08 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -422,8 +422,8 @@ static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *p goto _err; } - taosMemoryFree(req.schemaRow.pSchema); - taosMemoryFree(req.schemaTag.pSchema); + // taosMemoryFree(req.schemaRow.pSchema); + // taosMemoryFree(req.schemaTag.pSchema); tDecoderClear(&coder); return 0; From 9f94dcb38496954f27c427f8fa33ef9cae87b60f Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 18 Jul 2022 09:45:33 +0800 Subject: [PATCH 33/88] refactor(sync): add index/term in vnodeSnapWriterClose --- source/libs/sync/test/syncConfigChangeSnapshotTest.cpp | 2 +- source/libs/sync/test/syncSnapshotReceiverTest.cpp | 2 +- source/libs/sync/test/syncTestTool.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index 339ebe90e7..8c6c68bbf8 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -125,7 +125,7 @@ int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void* pParam, void** ppWriter) return 0; } -int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply, SSnapshot *pSnapshot) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d", pFsm, pWriter, isApply); diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index e5d93ddff4..b744843b1e 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -30,7 +30,7 @@ int32_t SnapshotStopRead(struct SSyncFSM* pFsm, void* pReader) { return 0; } int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len) { return 0; } int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void* pParam, void** ppWriter) { return 0; } -int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { return 0; } +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply, SSnapshot *pSnapshot) { return 0; } int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len) { return 0; } SSyncSnapshotReceiver* createReceiver() { diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index f35c6f8a2f..b0a561cb89 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -126,7 +126,7 @@ int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void* pParam, void** ppWriter) return 0; } -int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply, SSnapshot *pSnapshot) { if (isApply) { gSnapshotLastApplyIndex = gFinishLastApplyIndex; gSnapshotLastApplyTerm = gFinishLastApplyTerm; From 2a3d603089d5ce7a2233d7deb0aa37988c57f692 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 10:03:42 +0800 Subject: [PATCH 34/88] update --- tests/system-test/2-query/last_row.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index ce314e22b2..22f165342a 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -335,19 +335,9 @@ class TDTestCase: tdSql.checkData(0,1,None) tdSql.checkData(0,2,None) - # tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from stb1 where ts Date: Mon, 18 Jul 2022 10:34:26 +0800 Subject: [PATCH 35/88] doc: refine limits and keywords --- docs/en/12-taos-sql/14-limit.md | 34 +++--------------- docs/zh/12-taos-sql/14-limit.md | 57 ++++++++++++++++-------------- docs/zh/12-taos-sql/20-keywords.md | 42 ++-------------------- 3 files changed, 37 insertions(+), 96 deletions(-) diff --git a/docs/en/12-taos-sql/14-limit.md b/docs/en/12-taos-sql/14-limit.md index db55cdd69e..e8bb77fc27 100644 --- a/docs/en/12-taos-sql/14-limit.md +++ b/docs/en/12-taos-sql/14-limit.md @@ -1,5 +1,5 @@ --- -title: Limits & Restrictions +title: Naming & Restrictions --- ## Naming Rules @@ -16,41 +16,21 @@ The legal character set is `[a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/]`. ## General Limits -- Maximum length of database name is 32 bytes. +- Maximum length of database name is 32 bytes, and it can't include "." or special characters. - Maximum length of table name is 192 bytes, excluding the database name prefix and the separator. -- Maximum length of each data row is 48K bytes since version 2.1.7.0 , before which the limit was 16K bytes. Please note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type. +- Maximum length of each data row is 48K bytes. Please note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type. - Maximum length of column name is 64. - Maximum number of columns is 4096. There must be at least 2 columns, and the first column must be timestamp. - Maximum length of tag name is 64. - Maximum number of tags is 128. There must be at least 1 tag. The total length of tag values should not exceed 16K bytes. - Maximum length of singe SQL statement is 1048576, i.e. 1 MB. It can be configured in the parameter `maxSQLLength` in the client side, the applicable range is [65480, 1048576]. -- At most 4096 columns (or 1024 prior to 2.1.7.0) can be returned by `SELECT`. Functions in the query statement constitute columns. An error is returned if the limit is exceeded. +- At most 4096 columns can be returned by `SELECT`. Functions in the query statement constitute columns. An error is returned if the limit is exceeded. - Maximum numbers of databases, STables, tables are dependent only on the system resources. -- Maximum of database name is 32 bytes, and it can't include "." or special characters. - Maximum number of replicas for a database is 3. - Maximum length of user name is 23 bytes. - Maximum length of password is 15 bytes. - Maximum number of rows depends only on the storage space. -- Maximum number of tables depends only on the number of nodes. -- Maximum number of databases depends only on the number of nodes. -- Maximum number of vnodes for a single database is 64. - -## Restrictions of `GROUP BY` - -`GROUP BY` can be performed on tags and `TBNAME`. It can be performed on data columns too, with the only restriction being it can only be performed on one data column and the number of unique values in that column is lower than 100,000. Please note that `GROUP BY` cannot be performed on float or double types. - -## Restrictions of `IS NOT NULL` - -`IS NOT NULL` can be used on any data type of columns. The non-empty string evaluation expression, i.e. `< > ""` can only be used on non-numeric data types. - -## Restrictions of `ORDER BY` - -- Only one `order by` is allowed for normal table and subtable. -- At most two `order by` are allowed for STable, and the second one must be `ts`. -- `order by tag` must be used with `group by tag` on same tag. This rule is also applicable to `tbname`. -- `order by column` must be used with `group by column` or `top/bottom` on same column. This rule is applicable to table and STable. -- `order by ts` is applicable to table and STable. -- If `order by ts` is used with `group by`, the result set is sorted using `ts` in each group. +- Maximum number of vnodes for a single database is 1024. ## Restrictions of Table/Column Names @@ -71,7 +51,3 @@ For example: The characters inside escape characters must be printable characters. ::: - -### Applicable Versions - -Escape character "\`" is available from version 2.3.0.1. diff --git a/docs/zh/12-taos-sql/14-limit.md b/docs/zh/12-taos-sql/14-limit.md index 7673e24a83..95d1837ac6 100644 --- a/docs/zh/12-taos-sql/14-limit.md +++ b/docs/zh/12-taos-sql/14-limit.md @@ -1,37 +1,43 @@ --- -sidebar_label: 边界限制 -title: 边界限制 +sidebar_label: 命名与边界限制 +title: 命名与边界限制 --- +## 名称命名规则 + +1. 合法字符：英文字符、数字和下划线 +2. 允许英文字符或下划线开头，不允许以数字开头 +3. 不区分大小写 +4. 转义后表（列）名规则： + 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。可用让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。 + 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 + + 例如：\`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 + 需要注意的是转义字符中的内容必须是可打印字符。 + +## 密码合法字符集 + +`[a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/]` + +去掉了 `` ‘“`\ `` (单双引号、撇号、反斜杠、空格) + ## 一般限制 - 数据库名最大长度为 32。 - 表名最大长度为 192，不包括数据库名前缀和分隔符 - 每行数据最大长度 48KB （注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）。 -- 列名最大长度为 64，最多允许 4096 列，最少需要 2 列，第一列必须是时间戳。注：从 2.1.7.0 版本（不含)以前最多允许 4096 列 -- 标签名最大长度为 64，最多允许 128 个，至少要有 1 个标签，一个表中标签值的总长度不超过 16KB 。 +- 列名最大长度为 64 +- 最多允许 4096 列，最少需要 2 列，第一列必须是时间戳。 +- 标签名最大长度为 64 +- 最多允许 128 个，至少要有 1 个标签，一个表中标签值的总长度不超过 16KB 。 - SQL 语句最大长度 1048576 个字符，也可通过客户端配置参数 maxSQLLength 修改，取值范围 65480 ~ 1048576。 -- SELECT 语句的查询结果，最多允许返回 4096 列（语句中的函数调用可能也会占用一些列空间），超限时需要显式指定较少的返回数据列，以避免语句执行报错。注： 2.1.7.0 版本（不含）之前为最多允许 1024 列 +- SELECT 语句的查询结果，最多允许返回 4096 列（语句中的函数调用可能也会占用一些列空间），超限时需要显式指定较少的返回数据列，以避免语句执行报错。 - 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制。 - -## GROUP BY 的限制 - -TAOS SQL 支持对标签、TBNAME 进行 GROUP BY 操作，也支持普通列进行 GROUP BY，前提是：仅限一列且该列的唯一值小于 10 万个。注意：group by 不支持 float,double 类型。 - -## IS NOT NULL 的限制 - -IS NOT NULL 与不为空的表达式适用范围。 - -IS NOT NULL 支持所有类型的列。不为空的表达式为 <\>""，仅对非数值类型的列适用。 - -## ORDER BY 的限制 - -- 非超级表只能有一个 order by. -- 超级表最多两个 order by， 并且第二个必须为 ts. -- order by tag，必须和 group by tag 一起，并且是同一个 tag。 tbname 和 tag 一样逻辑。 只适用于超级表 -- order by 普通列，必须和 group by 一起或者和 top/bottom 一起，并且是同一个普通列。 适用于超级表和普通表。如果同时存在 group by 和 top/bottom 一起，order by 优先必须和 group by 同一列。 -- order by ts. 适用于超级表和普通表。 -- order by ts 同时含有 group by 时 针对 group 内部用 ts 排序 +- 数据库的副本数只能设置为 1 或 3 +- 用户名的最大长度是 23 个字节 +- 用户密码的最大长度是 15 个字节 +- 总数据行数取决于可用资源 +- 单个数据库的虚拟结点数上限为 1024 ## 表(列)名合法性说明 @@ -49,6 +55,3 @@ IS NOT NULL 支持所有类型的列。不为空的表达式为 <\>""，仅对 转义字符中的内容必须是可打印字符。 ::: - -### 支持版本 -支持转义符的功能从 2.3.0.1 版本开始。 \ No newline at end of file diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index 202f223b45..d91c9be2ac 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -1,46 +1,8 @@ --- -sidebar_label: 参数限制与保留关键字 -title: TDengine 参数限制与保留关键字 +sidebar_label: 保留关键字 +title: TDengine 保留关键字 --- -## 名称命名规则 - -1. 合法字符：英文字符、数字和下划线 -2. 允许英文字符或下划线开头，不允许以数字开头 -3. 不区分大小写 -4. 转义后表（列）名规则： - 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。可用让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。 - 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 - - 例如：\`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 - 需要注意的是转义字符中的内容必须是可打印字符。 - 支持转义符的功能从 2.3.0.1 版本开始。 - -## 密码合法字符集 - -`[a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/]` - -去掉了 `` ‘“`\ `` (单双引号、撇号、反斜杠、空格) - -- 数据库名：不能包含“.”以及特殊字符，不能超过 32 个字符 -- 表名：不能包含“.”以及特殊字符，与所属数据库名一起，不能超过 192 个字节 ，每行数据最大长度 48KB -- 表的列名：不能包含特殊字符，不能超过 64 个字节 -- 数据库名、表名、列名，都不能以数字开头，合法的可用字符集是“英文字符、数字和下划线” -- 表的列数：不能超过 1024 列，最少需要 2 列，第一列必须是时间戳（从 2.1.7.0 版本开始，改为最多支持 4096 列） -- 记录的最大长度：包括时间戳 8 字节，不能超过 48KB（每个 BINARY/NCHAR 类型的列还会额外占用 2 个 字节 的存储位置） -- 单条 SQL 语句默认最大字符串长度：1048576 字节，但可通过系统配置参数 maxSQLLength 修改，取值范围 65480 ~ 1048576 字节 -- 数据库副本数：不能超过 3 -- 用户名：不能超过 23 个 字节 -- 用户密码：不能超过 15 个 字节 -- 标签(Tags)数量：不能超过 128 个，可以 0 个 -- 标签的总长度：不能超过 16KB -- 记录条数：仅受存储空间限制 -- 表的个数：仅受节点个数限制 -- 库的个数：仅受节点个数限制 -- 单个库上虚拟节点个数：不能超过 64 个 -- 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 -- SELECT 语句的查询结果，最多允许返回 1024 列（语句中的函数调用可能也会占用一些列空间），超限时需要显式指定较少的返回数据列，以避免语句执行报错。（从 2.1.7.0 版本开始，改为最多允许 4096 列） - ## 保留关键字 目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable 名、数据列名及标签列名等。这些关键字列表如下： From 41eefb3d71c1b08a6b49d74462dab4b71fe4d47e Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Mon, 18 Jul 2022 10:46:40 +0800 Subject: [PATCH 36/88] ci: skip docs pr --- Jenkinsfile2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 32e9b11520..1b04e40f2a 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -333,6 +333,11 @@ pipeline { } stages { stage('run test') { + when { + allOf { + not { expression { env.CHANGE_BRANCH =~ /docs\// }} + } + } parallel { stage('windows test') { agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "} From 648dc7267b302ee59fa50f018b996115e8634efc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jul 2022 10:53:17 +0800 Subject: [PATCH 37/88] test: valgrind case --- tests/script/jenkins/basic.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 7347446556..0bdbd644a0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -311,6 +311,7 @@ ./test.sh -f tsim/valgrind/checkError2.sim ./test.sh -f tsim/valgrind/checkError3.sim ./test.sh -f tsim/valgrind/checkError4.sim +./test.sh -f tsim/valgrind/checkError5.sim # --- vnode # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim From 0fa9ce71d86f50f64c3cb80052653adb27310ee6 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 18 Jul 2022 11:03:03 +0800 Subject: [PATCH 38/88] refactor(sync): add index/term in vnodeSnapWriterClose --- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index c6b423f712..59242cbd8a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -193,8 +193,8 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * pVnode->state.committed = pWriter->ever; pVnode->state.applied = pWriter->ever; - // pVnode->state.applyTerm = ; - // pVnode->state.commitTerm = ; + pVnode->state.applyTerm = pSnapshot->lastApplyTerm; + pVnode->state.commitTerm = pSnapshot->lastApplyTerm; info.config = pVnode->config; info.state.committed = pVnode->state.applied; From bd4e24cc80b477e1ad8a280d577482648287e103 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jul 2022 10:57:56 +0800 Subject: [PATCH 39/88] test: valgrind case --- tests/script/tsim/stable/alter_count.sim | 2 - tests/script/tsim/stable/disk.sim | 146 +++++++++++------------ tests/script/tsim/stable/dnode3.sim | 142 +++++++++++----------- tests/script/tsim/stable/metrics.sim | 67 +++++------ tests/script/tsim/stable/refcount.sim | 48 ++++---- tests/script/tsim/stable/tag_filter.sim | 1 - tests/script/tsim/stable/values.sim | 94 +++++++-------- tests/script/tsim/stable/vnode3.sim | 136 +++++++++++---------- tests/script/tsim/valgrind/basic1.sim | 128 ++++++++++++++------ 9 files changed, 397 insertions(+), 367 deletions(-) diff --git a/tests/script/tsim/stable/alter_count.sim b/tests/script/tsim/stable/alter_count.sim index decad53f64..83ea4b14fa 100644 --- a/tests/script/tsim/stable/alter_count.sim +++ b/tests/script/tsim/stable/alter_count.sim @@ -249,9 +249,7 @@ endi print ============== step18 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 3000 #sql select count(g) from tb #if $data00 != 12 then diff --git a/tests/script/tsim/stable/disk.sim b/tests/script/tsim/stable/disk.sim index ff734b4234..aeb1f1d91f 100644 --- a/tests/script/tsim/stable/disk.sim +++ b/tests/script/tsim/stable/disk.sim @@ -77,12 +77,11 @@ if $data00 != $rowNum then endi print =============== step3 -# TODO : where condition -# sql select count(tbcol) from $tb where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 5 then -# return -1 -# endi +sql select count(tbcol) from $tb where ts <= 1519833840000 +print ===> $data00 +if $data00 != 5 then + return -1 +endi print =============== step4 sql select count(tbcol) as b from $tb @@ -105,89 +104,82 @@ if $data00 != $rowNum then endi print =============== step6 -# TODO -# sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) -# print ===> $data00 -# if $data00 != 1 then -# return -1 -# endi -# if $rows != 5 then -# return -1 -# endi +sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi +if $rows != 5 then + return -1 +endi print =============== step7 -# TODO -# sql select count(*) from $mt -# print select count(*) from $mt ===> $data00 -# if $data00 != $totalNum then -# return -1 -# endi +sql select count(*) from $mt +print select count(*) from $mt ===> $data00 +if $data00 != $totalNum then + return -1 +endi -# TODO -# print ==========> block opt will cause this crash, table scan need to fix this during plan gen ===============> -#sql select count(tbcol) from $mt -#print ===> $data00 -#if $data00 != $totalNum then -# return -1 -#endi +print ==========> block opt will cause this crash, table scan need to fix this during plan gen ===============> +sql select count(tbcol) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi print =============== step8 -# TODO -# sql select count(tbcol) as c from $mt where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 50 then -# return -1 -# endi -# -# sql select count(tbcol) as c from $mt where tgcol < 5 -# print ===> $data00 -# if $data00 != 100 then -# return -1 -# endi -# -# sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 25 then -# return -1 -# endi +sql select count(tbcol) as c from $mt where ts <= 1519833840000 +print ===> $data00 +if $data00 != 50 then + return -1 +endi + +sql select count(tbcol) as c from $mt where tgcol < 5 +print ===> $data00 +if $data00 != 100 then + return -1 +endi + +sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 +print ===> $data00 +if $data00 != 25 then + return -1 +endi print =============== step9 -# TODO : count from stable -# sql select count(tbcol) as b from $mt interval(1m) -# print select count(tbcol) as b from $mt interval(1m) ===> $data01 -# if $data01 != 10 then -# return -1 -# endi +sql select count(tbcol) as b from $mt interval(1m) +print select count(tbcol) as b from $mt interval(1m) ===> $data01 +if $data00 != 10 then + return -1 +endi -# sql select count(tbcol) as b from $mt interval(1d) -# print ===> $data02 -# if $data01 != 200 then -# return -1 -# endi +sql select count(tbcol) as b from $mt interval(1d) +print ===> $data02 +if $data00 != 200 then + return -1 +endi print =============== step10 -# TODO -# print select count(tbcol) as b from $mt group by tgcol -# sql select count(tbcol) as b from $mt group by tgcol -# print ===> $data00 -# if $data00 != $rowNum then -# return -1 -# endi +print select count(tbcol) as b from $mt group by tgcol +sql select count(tbcol) as b from $mt group by tgcol +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +if $rows != $tbNum then + return -1 +endi -# if $rows != $tbNum then -# return -1 -# endi -# print =============== step11 -# TODO : where condition -# sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol -# print ===> $data01 -# if $data01 != 1 then -# return -1 -# endi -# if $rows != 50 then -# return -1 -# endi +sql select count(tbcol) as b from $mt where ts <= 1519833840000 partition by tgcol interval(1m) +print ===> $data01 +if $data00 != 1 then + return -1 +endi +if $rows != 50 then + return -1 +endi print =============== clear sql drop database $db diff --git a/tests/script/tsim/stable/dnode3.sim b/tests/script/tsim/stable/dnode3.sim index 4a2d148738..9e728a12ab 100644 --- a/tests/script/tsim/stable/dnode3.sim +++ b/tests/script/tsim/stable/dnode3.sim @@ -86,12 +86,11 @@ if $data00 != $rowNum then endi print =============== step3 -# TODO : where condition -# sql select count(tbcol) from $tb where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 5 then -# return -1 -# endi +sql select count(tbcol) from $tb where ts <= 1519833840000 +print ===> $data00 +if $data00 != 5 then + return -1 +endi print =============== step4 sql select count(tbcol) as b from $tb @@ -114,82 +113,81 @@ if $data00 != $rowNum then endi print =============== step6 -# sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) -# print ===> $data00 -# if $data00 != 1 then -# return -1 -# endi -# if $rows != 5 then -# return -1 -# endi +sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi +if $rows != 5 then + return -1 +endi print =============== step7 -# print select count(*) from $mt -# sql select count(*) from $mt -# print ===> $data00 -# if $data00 != $totalNum then -# return -1 -# endi -# -# sql select count(tbcol) from $mt -# print ===> $data00 -# if $data00 != $totalNum then -# return -1 -# endi +print select count(*) from $mt +sql select count(*) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi + +sql select count(tbcol) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi print =============== step8 -# sql select count(tbcol) as c from $mt where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 50 then -# return -1 -# endi -# -# sql select count(tbcol) as c from $mt where tgcol < 5 -# print ===> $data00 -# if $data00 != 100 then -# return -1 -# endi -# -# sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 25 then -# return -1 -# endi +sql select count(tbcol) as c from $mt where ts <= 1519833840000 +print ===> $data00 +if $data00 != 50 then + return -1 +endi + +sql select count(tbcol) as c from $mt where tgcol < 5 +print ===> $data00 +if $data00 != 100 then + return -1 +endi + +sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 +print ===> $data00 +if $data00 != 25 then + return -1 +endi print =============== step9 -# TODO : group by in stable -# sql select count(tbcol) as b from $mt interval(1m) -# print ===> $data00 -# if $data00 != 10 then -# return -1 -# endi -# -# sql select count(tbcol) as b from $mt interval(1d) -# print ===> $data00 -# if $data00 != 200 then -# return -1 -# endi +sql select count(tbcol) as b from $mt interval(1m) +print ===> $data00 +if $data00 != 10 then + return -1 +endi + +sql select count(tbcol) as b from $mt interval(1d) +print ===> $data00 +if $data00 != 200 then + return -1 +endi print =============== step10 -# sql select count(tbcol) as b from $mt group by tgcol -# print ===> $data00 -# if $data00 != $rowNum then -# return -1 -# endi -# -# if $rows != $tbNum then -# return -1 -# endi +sql select count(tbcol) as b from $mt group by tgcol +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +if $rows != $tbNum then + return -1 +endi print =============== step11 -# sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol -# print ===> $data00 -# if $data00 != 1 then -# return -1 -# endi -# if $rows != 50 then -# return -1 -# endi +sql select count(tbcol) as b from $mt where ts <= 1519833840000 partition by tgcol interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi +if $rows != 50 then + return -1 +endi print =============== clear sql drop database $db diff --git a/tests/script/tsim/stable/metrics.sim b/tests/script/tsim/stable/metrics.sim index c652670d7f..a1c370d40a 100644 --- a/tests/script/tsim/stable/metrics.sim +++ b/tests/script/tsim/stable/metrics.sim @@ -24,17 +24,15 @@ if $rows != 1 then return -1 endi -#TODO OPEN THIS WHEN STABLE DELETE WORKS -#print =============== step2 -#sql drop table $mt -#sql show stables -#if $rows != 0 then -# return -1 -#endi +print =============== step2 +sql drop table $mt +sql show stables +if $rows != 0 then + return -1 +endi -#print =============== step3 -#sql create table $mt (ts timestamp, speed int) TAGS(sp int) -#TODO OPEN THIS WHEN STABLE DELETE WORKS +print =============== step3 +sql create table $mt (ts timestamp, speed int) TAGS(sp int) sql show stables if $rows != 1 then @@ -46,11 +44,10 @@ endi if $data04 != 1 then return -1 endi -# TODO : select * from stable -# sql select * from $mt -# if $rows != 0 then -# return -1 -# endi +sql select * from $mt +if $rows != 0 then + return -1 +endi print =============== step4 $i = 0 @@ -67,9 +64,9 @@ sql show tables if $rows != 3 then return -1 endi -# if $data03 != $mt then -# return -1 -# endi +if $data04 != $mt then + return -1 +endi sql show stables if $rows != 1 then @@ -78,9 +75,9 @@ endi if $data00 != $mt then return -1 endi -# if $data04 != 3 then -# return -1 -# endi +if $data03 != 2 then + return -1 +endi print =============== step5 $i = 0 @@ -95,28 +92,28 @@ sql insert into $tb values (now + 1m , 1 ) print =============== step6 -# sql select * from $mt -# print select * from $mt ==> $rows $data00 -# if $rows != 3 then -# return -1 -# endi +sql select * from $mt +print select * from $mt ==> $rows $data00 +if $rows != 3 then + return -1 +endi print =============== step7 -# sql select * from $mt where sp = 1 -# print select * from $mt where sp = 1 ==> $rows $data00 -# if $rows != 1 then -# return -1 -# endi +sql select * from $mt where sp = 1 +print select * from $mt where sp = 1 ==> $rows $data00 +if $rows != 1 then + return -1 +endi print =============== step8 sql drop table $mt print =============== step9 -#sql show tables -#if $rows != 0 then -# return -1 -#endi +sql show tables +if $rows != 0 then + return -1 +endi sql show stables if $rows != 0 then diff --git a/tests/script/tsim/stable/refcount.sim b/tests/script/tsim/stable/refcount.sim index d77c8e0890..8f4f09cbb3 100644 --- a/tests/script/tsim/stable/refcount.sim +++ b/tests/script/tsim/stable/refcount.sim @@ -11,7 +11,7 @@ sql create table d1.t2 (ts timestamp, i int); sql create table d1.t3 (ts timestamp, i int); sql insert into d1.t1 values(now, 1); sql insert into d1.t2 values(now, 1); -# sql drop table d1.t1; +sql drop table d1.t1; sql drop database d1; sql show databases; @@ -27,14 +27,14 @@ sql create table d2.t2 (ts timestamp, i int); sql create table d2.t3 (ts timestamp, i int); sql insert into d2.t1 values(now, 1); sql insert into d2.t2 values(now, 1); -# sql drop table d2.t1; -# sql drop table d2.t2; -# sql drop table d2.t3; -# -# sql show d2.tables; -# if $rows != 0 then -# return -1 -# endi +sql drop table d2.t1; +sql drop table d2.t2; +sql drop table d2.t3; + +sql show d2.tables; +if $rows != 0 then + return -1 +endi sql show d2.vgroups; if $rows != 2 then @@ -56,14 +56,14 @@ sql create table d3.t1 using d3.st tags(1); sql create table d3.t2 using d3.st tags(1); sql create table d3.t3 using d3.st tags(1); sql insert into d3.t1 values(now, 1); -# sql drop table d3.t1; -# sql drop table d3.t2; -# sql drop table d3.t3; -# -# sql show d3.tables; -# if $rows != 0 then -# return -1 -# endi +sql drop table d3.t1; +sql drop table d3.t2; +sql drop table d3.t3; + +sql show d3.tables; +if $rows != 0 then + return -1 +endi sql show d3.vgroups; if $rows != 2 then @@ -85,13 +85,13 @@ sql create table d4.t1 using d4.st tags(1); sql create table d4.t2 using d4.st tags(1); sql create table d4.t3 using d4.st tags(1); sql insert into d4.t1 values(now, 1); -# sql drop table d4.t1; +sql drop table d4.t1; sql drop table d4.st; -# -# sql show d4.tables; -# if $rows != 0 then -# return -1 -# endi + +sql show d4.tables; +if $rows != 0 then + return -1 +endi sql show d4.stables; if $rows != 0 then @@ -113,7 +113,7 @@ sql create table d5.t1 using d5.st tags(1); sql create table d5.t2 using d5.st tags(1); sql create table d5.t3 using d5.st tags(1); sql insert into d5.t1 values(now, 1); -# sql drop table d5.t1; +sql drop table d5.t1; sql drop database d5; diff --git a/tests/script/tsim/stable/tag_filter.sim b/tests/script/tsim/stable/tag_filter.sim index 1f400eb803..f44142fbbf 100644 --- a/tests/script/tsim/stable/tag_filter.sim +++ b/tests/script/tsim/stable/tag_filter.sim @@ -31,7 +31,6 @@ if $rows != 1 then return -1 endi - sql select * from db.stb where t1 < 1 if $rows != 0 then return -=1 diff --git a/tests/script/tsim/stable/values.sim b/tests/script/tsim/stable/values.sim index 88eca28a12..d3da101e27 100644 --- a/tests/script/tsim/stable/values.sim +++ b/tests/script/tsim/stable/values.sim @@ -12,22 +12,22 @@ sql create table vdb0.vtb01 using vdb0.mt tags( 0 ) sql create database vdb1 sql create table vdb1.mt (ts timestamp, tbcol int) TAGS(tgcol int) -# sql_error create table vdb1.vtb10 using vdb0.mt tags( 1 ) -# sql_error create table vdb1.vtb11 using vdb0.mt tags( 1 ) +sql_error create table vdb1.vtb10 using vdb0.mt tags( 1 ) +sql_error create table vdb1.vtb11 using vdb0.mt tags( 1 ) sql create table vdb1.vtb10 using vdb1.mt tags( 1 ) sql create table vdb1.vtb11 using vdb1.mt tags( 1 ) sql create database vdb2 sql create table vdb2.mt (ts timestamp, tbcol int) TAGS(tgcol int) -# sql_error create table vdb2.vtb20 using vdb0.mt tags( 2 ) -# sql_error create table vdb2.vtb21 using vdb0.mt tags( 2 ) +sql_error create table vdb2.vtb20 using vdb0.mt tags( 2 ) +sql_error create table vdb2.vtb21 using vdb0.mt tags( 2 ) sql create table vdb2.vtb20 using vdb2.mt tags( 2 ) sql create table vdb2.vtb21 using vdb2.mt tags( 2 ) sql create database vdb3 sql create table vdb3.mt (ts timestamp, tbcol int) TAGS(tgcol int) -# sql_error create table vdb3.vtb20 using vdb0.mt tags( 2 ) -# sql_error create table vdb3.vtb21 using vdb0.mt tags( 2 ) +sql_error create table vdb3.vtb20 using vdb0.mt tags( 2 ) +sql_error create table vdb3.vtb21 using vdb0.mt tags( 2 ) sql create table vdb3.vtb30 using vdb3.mt tags( 3 ) sql create table vdb3.vtb31 using vdb3.mt tags( 3 ) @@ -40,7 +40,7 @@ sql insert into vdb2.vtb20 values (1519833600000 , 12) (1519833600001, 22) (1519 sql insert into vdb2.vtb21 values (1519833600000 , 12) (1519833600001, 22) (1519833600002, 32) sql insert into vdb3.vtb30 values (1519833600000 , 13) (1519833600001, 23) (1519833600002, 33) sql insert into vdb3.vtb31 values (1519833600000 , 13) (1519833600001, 23) (1519833600002, 33) -# sql select * from vdb0.mt +sql select * from vdb0.mt sql select ts from vdb0.mt if $rows != 6 then @@ -56,8 +56,7 @@ sql insert into vdb2.vtb20 values (1519833600003 , 42) (1519833600005, 52) (1519 sql insert into vdb2.vtb21 values (1519833600003 , 42) (1519833600005, 52) (1519833600004, 62) sql insert into vdb3.vtb30 values (1519833600003 , 43) (1519833600005, 53) (1519833600004, 63) sql insert into vdb3.vtb31 values (1519833600003 , 43) (1519833600005, 53) (1519833600004, 63) -# TODO : select * from stable -# sql select * from vdb0.mt +sql select * from vdb0.mt sql select ts from vdb0.mt if $rows != 12 then @@ -65,50 +64,49 @@ if $rows != 12 then endi print =============== step4 -# TODO : insert into diffrent table -# sql insert into vdb0.vtb00 values(1519833600006, 60) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 60) (1519833600007, 70) -# sql insert into vdb1.vtb10 values(1519833600006, 61) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 61) (1519833600007, 71) -# sql insert into vdb2.vtb20 values(1519833600006, 62) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 62) (1519833600007, 72) -# sql insert into vdb3.vtb30 values(1519833600006, 63) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 63) (1519833600007, 73) -# # sql select * from vdb0.mt -# sql select ts from vdb0.mt -# -# if $rows != 16 then -# return -1 -# endi +sql insert into vdb0.vtb00 values(1519833600006, 60) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 60) (1519833600007, 70) +sql insert into vdb1.vtb10 values(1519833600006, 61) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 61) (1519833600007, 71) +sql insert into vdb2.vtb20 values(1519833600006, 62) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 62) (1519833600007, 72) +sql insert into vdb3.vtb30 values(1519833600006, 63) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 63) (1519833600007, 73) +sql select * from vdb0.mt +sql select ts from vdb0.mt + +if $rows != 16 then + return -1 +endi print =============== step5 -# sql insert into vdb0.vtb00 values(1519833600008, 80) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 80) (1519833600007, 70) -# sql insert into vdb1.vtb10 values(1519833600008, 81) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 81) (1519833600007, 71) -# sql insert into vdb2.vtb20 values(1519833600008, 82) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 82) (1519833600007, 72) -# sql insert into vdb3.vtb30 values(1519833600008, 83) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 83) (1519833600007, 73) -# # sql select * from vdb0.mt -# sql select ts from vdb0.mt -# -# if $rows != 17 then -# return -1 -# endi +sql insert into vdb0.vtb00 values(1519833600008, 80) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 80) (1519833600007, 70) +sql insert into vdb1.vtb10 values(1519833600008, 81) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 81) (1519833600007, 71) +sql insert into vdb2.vtb20 values(1519833600008, 82) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 82) (1519833600007, 72) +sql insert into vdb3.vtb30 values(1519833600008, 83) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 83) (1519833600007, 73) +sql select * from vdb0.mt +sql select ts from vdb0.mt + +if $rows != 17 then + return -1 +endi print =============== step6 -# sql insert into vdb0.vtb00 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb10 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb20 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb30 values(1519833600009, 90) (1519833600010, 100) -# sql insert into vdb0.vtb01 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb11 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb21 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb31 values(1519833600009, 90) (1519833600010, 100) -# -# # sql select * from vdb0.mt -# sql select ts from vdb0.mt -# -# if $rows != 21 then -# return -1 -# endi +sql insert into vdb0.vtb00 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb10 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb20 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb30 values(1519833600009, 90) (1519833600010, 100) +sql insert into vdb0.vtb01 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb11 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb21 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb31 values(1519833600009, 90) (1519833600010, 100) + +sql select * from vdb0.mt +sql select ts from vdb0.mt + +if $rows != 21 then + return -1 +endi print =============== step7 -# sql insert into vdb0.vtb00 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb10 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb20 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb30 values(1519833600012, 120) (1519833600011, 110) -# sql insert into vdb0.vtb01 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb11 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb21 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb31 values(1519833600012, 120) (1519833600011, 110) -# -# # sql select * from vdb0.mt -# sql select ts from vdb0.mt -# -# if $rows != 25 then -# return -1 -# endi +sql insert into vdb0.vtb00 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb10 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb20 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb30 values(1519833600012, 120) (1519833600011, 110) +sql insert into vdb0.vtb01 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb11 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb21 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb31 values(1519833600012, 120) (1519833600011, 110) + +sql select * from vdb0.mt +sql select ts from vdb0.mt + +if $rows != 25 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stable/vnode3.sim b/tests/script/tsim/stable/vnode3.sim index 186d0f5eea..584578b211 100644 --- a/tests/script/tsim/stable/vnode3.sim +++ b/tests/script/tsim/stable/vnode3.sim @@ -60,12 +60,11 @@ if $data00 != $rowNum then endi print =============== step3 -# TODO : where condition -# sql select count(tbcol) from $tb where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 5 then -# return -1 -# endi +sql select count(tbcol) from $tb where ts <= 1519833840000 +print ===> $data00 +if $data00 != 5 then + return -1 +endi print =============== step4 sql select count(tbcol) as b from $tb @@ -88,81 +87,80 @@ if $data00 != $rowNum then endi print =============== step6 -# sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) -# print ===> $data00 -# if $data00 != 1 then -# return -1 -# endi -# if $rows != 5 then -# return -1 -#endi +sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi +if $rows != 5 then + return -1 +endi print =============== step7 -# TODO : count(*) err -# sql select count(*) from $mt -# print ===> $data00 -# if $data00 != $totalNum then -# return -1 -# endi -# -# sql select count(tbcol) from $mt -# print ===> $data00 -# if $data00 != $totalNum then -# return -1 -# endi +sql select count(*) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi + +sql select count(tbcol) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi print =============== step8 -# sql select count(tbcol) as c from $mt where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 50 then -# return -1 -# endi +sql select count(tbcol) as c from $mt where ts <= 1519833840000 +print ===> $data00 +if $data00 != 50 then + return -1 +endi -# sql select count(tbcol) as c from $mt where tgcol < 5 -# print ===> $data00 -# if $data00 != 100 then -# return -1 -# endi +sql select count(tbcol) as c from $mt where tgcol < 5 +print ===> $data00 +if $data00 != 100 then + return -1 +endi -# sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 25 then -# return -1 -# endi +sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 +print ===> $data00 +if $data00 != 25 then + return -1 +endi print =============== step9 -# sql select count(tbcol) as b from $mt interval(1m) -# print ===> $data00 -# if $data00 != 10 then -# return -1 -# endi -# -# sql select count(tbcol) as b from $mt interval(1d) -# print ===> $data00 -# if $data00 != 200 then -# return -1 -# endi +sql select count(tbcol) as b from $mt interval(1m) +print ===> $data00 +if $data00 != 10 then + return -1 +endi + +sql select count(tbcol) as b from $mt interval(1d) +print ===> $data00 +if $data00 != 200 then + return -1 +endi print =============== step10 -# sql select count(tbcol) as b from $mt group by tgcol -# print ===> $data00 -# if $data00 != $rowNum then -# return -1 -# endi -# -# if $rows != $tbNum then -# return -1 -# endi +sql select count(tbcol) as b from $mt group by tgcol +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +if $rows != $tbNum then + return -1 +endi print =============== step11 -# sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol -# print ===> $data01 -# if $data01 != 1 then -# return -1 -# endi -# if $rows != 50 then -# return -1 -# endi +sql select count(tbcol) as b from $mt where ts <= 1519833840000 partition by tgcol interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi +if $rows != 50 then + return -1 +endi print =============== clear sql drop database $db diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 3c47cae5cc..f3d418cfd1 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -1,49 +1,99 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start -v sql connect -print ======== step1 -sql drop database if exists db1; -sql create database db1 vgroups 3; -sql use db1; -sql create stable st1 (ts timestamp, f1 int, f2 binary(200)) tags(t1 int); -sql create table tb1 using st1 tags(1); -sql insert into tb1 values ('2022-07-07 10:01:01', 11, "aaa"); -sql insert into tb1 values ('2022-07-07 11:01:02', 12, "bbb"); -sql create table tb2 using st1 tags(2); -sql insert into tb2 values ('2022-07-07 10:02:01', 21, "aaa"); -sql insert into tb2 values ('2022-07-07 11:02:02', 22, "bbb"); -sql create table tb3 using st1 tags(3); -sql insert into tb3 values ('2022-07-07 10:03:01', 31, "aaa"); -sql insert into tb3 values ('2022-07-07 11:03:02', 32, "bbb"); -sql create table tb4 using st1 tags(4); - -sql insert into tb4 select * from tb1; - -sql select * from tb4; -if $rows != 2 then +print =============== step1: create drop show dnodes +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not ready! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +if $rows != 1 then return -1 endi +if $data(1)[4] != ready then + goto step1 +endi -sql insert into tb4 select ts,f1,f2 from st1; -sql select * from tb4; -if $rows != 6 then - return -1 -endi -sql create table tba (ts timestamp, f1 binary(10), f2 bigint, f3 double); -sql_error insert into tba select * from tb1; -sql insert into tba (ts,f2,f1) select * from tb1; -sql select * from tba; -if $rows != 2 then - return -1 -endi -sql create table tbb (ts timestamp, f1 binary(10), f2 bigint, f3 double); -sql insert into tbb (f2,f1,ts) select f1+1,f2,ts+3 from tb2; -sql select * from tbb; -if $rows != 2 then - return -1 -endi +print =============== step2: create db +sql create database db +sql use db +sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" +sql create table db.c1 using db.stb tags(101, 102, "103") + +print =============== step3: alter stb +sql_error alter table db.stb add column ts int +sql alter table db.stb add column c3 int +sql alter table db.stb add column c4 bigint +sql alter table db.stb add column c5 binary(12) +sql alter table db.stb drop column c1 +sql alter table db.stb drop column c4 +sql alter table db.stb MODIFY column c2 binary(32) +sql alter table db.stb add tag t4 bigint +sql alter table db.stb add tag c1 int +sql alter table db.stb add tag t5 binary(12) +sql alter table db.stb drop tag c1 +sql alter table db.stb drop tag t5 +sql alter table db.stb MODIFY tag t3 binary(32) +sql alter table db.stb rename tag t1 tx +sql alter table db.stb comment 'abcde' ; +sql drop table db.stb + +print =============== step4: alter tb +sql create table tb (ts timestamp, a int) +sql insert into tb values(now-28d, -28) +sql select count(a) from tb +sql alter table tb add column b smallint +sql insert into tb values(now-25d, -25, 0) +sql select count(b) from tb +sql alter table tb add column c tinyint +sql insert into tb values(now-22d, -22, 3, 0) +sql select count(c) from tb +sql alter table tb add column d int +sql insert into tb values(now-19d, -19, 6, 0, 0) +sql select count(d) from tb +sql alter table tb add column e bigint +sql alter table tb add column f float +sql alter table tb add column g double +sql alter table tb add column h binary(10) +sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb +sql select * from tb order by ts desc + +print =============== step5: alter stb and insert data +sql create table stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" +sql show db.stables +sql describe stb +sql_error alter table stb add column ts int + +sql create table db.ctb using db.stb tags(101, 102, "103") +sql insert into db.ctb values(now, 1, "2") +sql show db.tables +sql select * from db.stb +sql select * from tb + +sql alter table stb add column c3 int +sql describe stb +sql select * from db.stb +sql select * from tb +sql insert into db.ctb values(now+1s, 1, 2, 3) +sql select * from db.stb + +sql alter table db.stb add column c4 bigint +sql select * from db.stb +sql insert into db.ctb values(now+2s, 1, 2, 3, 4) + +sql alter table db.stb drop column c1 +sql reset query cache +sql select * from tb +sql insert into db.ctb values(now+3s, 2, 3, 4) +sql select * from db.stb _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -52,7 +102,7 @@ $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 1 then +if $system_content > 0 then return -1 endi From 22ea676a5a288839842f8fed1bc53ff21b379ff1 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 18 Jul 2022 11:29:15 +0800 Subject: [PATCH 40/88] shell: limit taos_history file size --- tools/shell/src/shellEngine.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d6b7f18fb9..d661667412 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -748,6 +748,13 @@ int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool ver void shellReadHistory() { SShellHistory *pHistory = &shell.history; + int64_t file_size; + if (taosStatFile(pHistory->file, &file_size, NULL) != 0) { + return; + } else if (file_size > SHELL_MAX_COMMAND_SIZE) { + taosRemoveFile(pHistory->file); + return; + } TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) return; @@ -771,6 +778,12 @@ void shellReadHistory() { void shellWriteHistory() { SShellHistory *pHistory = &shell.history; + int64_t file_size; + if (taosStatFile(pHistory->file, &file_size, NULL) != 0) { + return; + } else if (file_size > SHELL_MAX_COMMAND_SIZE) { + taosRemoveFile(pHistory->file); + } TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND); if (pFile == NULL) return; From 3ba69ca5dbca786d2d835dc2540afa87f2965c81 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 18 Jul 2022 11:31:13 +0800 Subject: [PATCH 41/88] shell: limit taos_history file size --- tools/shell/src/shellEngine.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d661667412..4cfa46bd3c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -778,12 +778,6 @@ void shellReadHistory() { void shellWriteHistory() { SShellHistory *pHistory = &shell.history; - int64_t file_size; - if (taosStatFile(pHistory->file, &file_size, NULL) != 0) { - return; - } else if (file_size > SHELL_MAX_COMMAND_SIZE) { - taosRemoveFile(pHistory->file); - } TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND); if (pFile == NULL) return; From 977d6d5242bc762075d5180b4ad48661c540b8f8 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 18 Jul 2022 11:34:24 +0800 Subject: [PATCH 42/88] update test case --- tests/system-test/1-insert/update_data.py | 41 ++++------------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index d2151976c8..c2b8e7ab08 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -11,7 +11,6 @@ # -*- coding: utf-8 -*- -from optparse import Values import random import string from util import constant @@ -23,7 +22,7 @@ from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(),logSql) self.setsql = TDSetSql() self.dbname = 'db_test' self.ntbname = 'ntb' @@ -60,34 +59,8 @@ class TDTestCase: self.double = None self.binary = None self.tnchar = None - def insert_base_data(self,col_type,tbname,value=None): - if value == None: - if col_type.lower() == 'tinyint': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.tinyint})') - elif col_type.lower() == 'smallint': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.smallint})') - elif col_type.lower() == 'int': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.int})') - elif col_type.lower() == 'bigint': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.bigint})') - elif col_type.lower() == 'tinyint unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.untinyint})') - elif col_type.lower() == 'smallint unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.unsmallint})') - elif col_type.lower() == 'int unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.unint})') - elif col_type.lower() == 'bigint unsigned': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.unbigint})') - elif col_type.lower() == 'bool': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.bool})') - elif col_type.lower() == 'float': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.float})') - elif col_type.lower() == 'double': - tdSql.execute(f'insert into {tbname} values({self.ts},{self.double})') - elif 'binary' in col_type.lower(): - tdSql.execute(f'insert into {tbname} values({self.ts},"{self.binary}")') - elif 'nchar' in col_type.lower(): - tdSql.execute(f'insert into {tbname} values({self.ts},"{self.nchar}")') + + def data_check(self,tbname,col_name,col_type,value): tdSql.query(f'select {col_name} from {tbname}') if col_type.lower() == 'float' or col_type.lower() == 'double': @@ -118,7 +91,7 @@ class TDTestCase: elif tb_type == 'ctb': tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') tdSql.execute(f'create table {tbname} using {stbname} tags(1)') - self.insert_base_data(col_name,tbname) + tdSql.execute(f'insert into {tbname} values({self.ts},null)') if col_type.lower() == 'double': for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') @@ -181,7 +154,7 @@ class TDTestCase: elif tb_type == 'ctb': tdSql.execute(f'create table {stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') tdSql.execute(f'create table {tbname} using {stbname} tags(1)') - self.insert_base_data(col_name,tbname) + tdSql.execute(f'insert into {tbname} values({self.ts},null)') if col_type.lower() == 'tinyint': self.update_and_check_data(tbname,col_name,col_type,up_tinyint,dbname) elif col_type.lower() == 'smallint': @@ -227,7 +200,7 @@ class TDTestCase: self.update_data_check(self.ntbname,self.column_dict,self.dbname,'ntb') for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') - self.insert_base_data(col_name,self.ntbname) + tdSql.execute(f'insert into {self.ntbname} values({self.ts},null)') if 'binary' in col_type.lower(): up_binary = tdCom.getLongName(self.str_length+1) tdSql.execute(f'alter table {self.ntbname} modify column {col_name} binary({self.str_length+1})') @@ -242,7 +215,7 @@ class TDTestCase: for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') tdSql.execute(f'create table {self.ctbname} using {self.stbname} tags(1)') - self.insert_base_data(col_name,self.ctbname) + tdSql.execute(f'insert into {self.ctbname} values({self.ts},null)') if 'binary' in col_type.lower(): up_binary = tdCom.getLongName(self.str_length+1) tdSql.execute(f'alter table {self.stbname} modify column {col_name} binary({self.str_length+1})') From 500553d581609447b42e78b6912bf9a8da902711 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 18 Jul 2022 11:41:56 +0800 Subject: [PATCH 43/88] refactor: add debug log --- examples/c/tmq.c | 30 +++++++++++++++++------ include/common/tmsg.h | 1 - include/libs/stream/tstream.h | 4 +++ source/client/src/tmq.c | 12 +-------- source/common/src/tmsg.c | 2 -- source/dnode/mnode/impl/src/mndConsumer.c | 9 ++++--- source/dnode/mnode/impl/src/mndDef.c | 2 ++ source/dnode/vnode/src/inc/tq.h | 1 - source/dnode/vnode/src/tq/tq.c | 6 +++++ source/dnode/vnode/src/tq/tqExec.c | 1 - source/dnode/vnode/src/tq/tqPush.c | 2 ++ source/libs/stream/src/stream.c | 3 +++ source/libs/stream/src/streamData.c | 2 ++ source/libs/stream/src/streamDispatch.c | 6 +++++ source/libs/stream/src/streamExec.c | 3 +++ tests/test/c/tmqDemo.c | 4 +-- 16 files changed, 58 insertions(+), 30 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 94e0b86821..38c3eba647 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -15,10 +15,10 @@ #include #include +#include #include #include #include "taos.h" -#include static int running = 1; static void msg_process(TAOS_RES* msg) { @@ -28,8 +28,8 @@ static void msg_process(TAOS_RES* msg) { printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { - tmq_raw_data *raw = tmq_get_raw_meta(msg); - if(raw){ + tmq_raw_data* raw = tmq_get_raw_meta(msg); + if (raw) { TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", NULL, 0); if (pConn == NULL) { return; @@ -55,7 +55,7 @@ static void msg_process(TAOS_RES* msg) { } tmq_free_raw_meta(raw); char* result = tmq_get_json_meta(msg); - if(result){ + if (result) { printf("meta result: %s\n", result); } tmq_free_json_meta(result); @@ -96,7 +96,9 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 nchar(8), t4 bool)"); + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); if (taos_errno(pRes) != 0) { printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); return -1; @@ -152,6 +154,7 @@ int32_t init_env() { } taos_free_result(pRes); +#if 0 pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); if (taos_errno(pRes) != 0) { printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); @@ -264,7 +267,9 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 nchar(8), t4 bool)"); + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); if (taos_errno(pRes) != 0) { printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); return -1; @@ -277,6 +282,7 @@ int32_t init_env() { return -1; } taos_free_result(pRes); +#endif return 0; } @@ -296,8 +302,15 @@ int32_t create_topic() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ + /*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/ + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic topic2 as select ts, c1, c2, c3 from st1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; @@ -353,6 +366,7 @@ tmq_t* build_consumer() { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "client.id", "my app 1"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "msg.with.table.name", "true"); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 61ae7541a0..9bdfacbf36 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2923,7 +2923,6 @@ typedef struct { SMqRspHead head; STqOffsetVal reqOffset; STqOffsetVal rspOffset; - int32_t skipLogNum; int32_t blockNum; int8_t withTbName; int8_t withSchema; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 5c4d8ce250..8c69c0f2de 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -83,6 +83,7 @@ typedef struct { int32_t srcVgId; int32_t childId; int64_t sourceVer; + int64_t reqId; SArray* blocks; // SArray } SStreamDataBlock; @@ -324,6 +325,8 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem); if (pSubmitClone == NULL) { + qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); + terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); return -1; } @@ -412,6 +415,7 @@ typedef struct { typedef struct { int64_t streamId; + int64_t reqId; int32_t srcTaskId; int32_t srcNodeId; int32_t dstTaskId; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ed0ec516b2..5b845fd455 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1052,6 +1052,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int32_t code = -1; req.consumerId = tmq->consumerId; + tstrncpy(req.clientId, tmq->clientId, 256); tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) goto FAIL; @@ -1146,14 +1147,6 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para conf->commitCbUserParam = param; } -#if 0 -int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { - if (tmq_message == NULL) return 0; - SMqPollRsp* pRsp = &tmq_message->msg; - return pRsp->skipLogNum; -} -#endif - int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; SMqClientVg* pVg = pParam->pVg; @@ -1296,9 +1289,6 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { offsetNew = *pOffset; } - /*tscDebug("consumer:%" PRId64 ", (epoch %d) offset of vgId:%d updated to %" PRId64 ", vgKey is %s", - * tmq->consumerId, epoch,*/ - /*pVgEp->vgId, offset, vgKey);*/ SMqClientVg clientVg = { .pollCnt = 0, .currentOffsetNew = offsetNew, diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index aeb83d3425..8c2438eef1 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5648,7 +5648,6 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) { int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->skipLogNum) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; if (pRsp->blockNum != 0) { if (tEncodeI8(pEncoder, pRsp->withTbName) < 0) return -1; @@ -5674,7 +5673,6 @@ int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; - if (tDecodeI32(pDecoder, &pRsp->skipLogNum) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; if (pRsp->blockNum != 0) { pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *)); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 5509ee88bf..a60db8a8c2 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -92,7 +92,9 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SMqConsumerLostMsg *pLostMsg = pMsg->pCont; SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId); - ASSERT(pConsumer); + if (pConsumer == NULL) { + return 0; + } mInfo("receive consumer lost msg, consumer id %" PRId64 ", status %s", pLostMsg->consumerId, mndConsumerStatusName(pConsumer->status)); @@ -450,6 +452,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { int32_t code = -1; SArray *newSub = subscribe.topicNames; taosArraySortString(newSub, taosArrayCompareString); + taosArrayRemoveDuplicate(newSub, taosArrayCompareString, taosMemoryFree); int32_t newTopicNum = taosArrayGetSize(newSub); // check topic existance @@ -907,8 +910,8 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false); // client id - char clientId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; - tstrncpy(varDataVal(clientId), pConsumer->clientId, TSDB_CGROUP_LEN); + char clientId[256 + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(varDataVal(clientId), pConsumer->clientId, 256); varDataSetLen(clientId, strlen(varDataVal(clientId))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)clientId, false); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index c26424e049..abac0573da 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -199,6 +199,7 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { int32_t tlen = 0; int32_t sz; tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); + tlen += taosEncodeString(buf, pConsumer->clientId); tlen += taosEncodeString(buf, pConsumer->cgroup); tlen += taosEncodeFixedI8(buf, pConsumer->updateType); tlen += taosEncodeFixedI32(buf, pConsumer->epoch); @@ -264,6 +265,7 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { int32_t sz; buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); + buf = taosDecodeStringTo(buf, pConsumer->clientId); buf = taosDecodeStringTo(buf, pConsumer->cgroup); buf = taosDecodeFixedI8(buf, &pConsumer->updateType); buf = taosDecodeFixedI32(buf, &pConsumer->epoch); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index c62b7e95bf..757749a9b6 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -52,7 +52,6 @@ typedef struct { int64_t reqOffset; int64_t processedVer; int32_t epoch; - int32_t skipLogNum; // rpc info int64_t reqId; SRpcHandleInfo rpcInfo; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f36e6f874f..fb05aeecd9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -635,6 +635,8 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { pSubmit = streamDataSubmitNew(pReq); if (pSubmit == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + qError("failed to create data submit for stream since out of memory"); failed = true; } @@ -644,12 +646,16 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { SStreamTask* pTask = *(SStreamTask**)pIter; if (!pTask->isDataScan) continue; + qDebug("data submit enqueue stream task: %d", pTask->taskId); + if (!failed) { if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { + qError("stream task input failed, task id %d", pTask->taskId); continue; } if (streamLaunchByWrite(pTask, TD_VID(pTq->pVnode)) < 0) { + qError("stream task launch failed, task id %d", pTask->taskId); continue; } } else { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 0bdbe82b77..eb17f06cc9 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -217,7 +217,6 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR } if (pRsp->blockNum == 0) { - pRsp->skipLogNum++; return -1; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index e9e5f6cd8b..4b2160434f 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -242,6 +242,8 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) void* data = taosMemoryMalloc(msgLen); if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + qError("failed to copy data for stream since out of memory"); return -1; } memcpy(data, msg, msgLen); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 2a96db3bfc..99a06575a9 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -143,6 +143,9 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, // enqueue if (pData != NULL) { + qDebug("task %d(child %d) recv retrieve req from task %d, reqId %ld", pTask->taskId, pTask->selfChildId, + pReq->srcTaskId, pReq->reqId); + pData->type = STREAM_INPUT__DATA_RETRIEVE; pData->srcVgId = 0; // decode diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index dbf6350c93..6be15222db 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -57,7 +57,9 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock pDataBlock->info.type = pRetrieve->streamBlockType; + pData->reqId = pReq->reqId; pData->blocks = pArray; + return 0; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index e2faf28abe..5dec33d0fb 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -65,6 +65,7 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->reqId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->dstNodeId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->dstTaskId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->srcNodeId) < 0) return -1; @@ -77,6 +78,7 @@ int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* p int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->reqId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->dstNodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->dstTaskId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->srcNodeId) < 0) return -1; @@ -121,6 +123,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t sz = taosArrayGetSize(pTask->childEpInfo); ASSERT(sz > 0); for (int32_t i = 0; i < sz; i++) { + req.reqId = tGenIdPI64(); SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i); req.dstNodeId = pEpInfo->nodeId; req.dstTaskId = pEpInfo->taskId; @@ -154,6 +157,9 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) ASSERT(0); return -1; } + + qDebug("task %d(child %d) send retrieve req to task %d at node %d, reqId %ld", pTask->taskId, pTask->selfChildId, + pEpInfo->taskId, pEpInfo->nodeId, req.reqId); } return 0; FAIL: diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 9d22da8662..f6eb9e32f2 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -54,6 +54,9 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) block.info.type = STREAM_PULL_OVER; block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); + + qDebug("task %d(child %d) processed retrieve, reqId %ld", pTask->taskId, pTask->selfChildId, + pRetrieveBlock->reqId); } break; } diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 02fd3c1396..61c50fb0e8 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -388,13 +388,11 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog } /*taosSsleep(3);*/ int32_t batchCnt = 0; - int32_t skipLogNum = 0; int64_t startTime = taosGetTimestampUs(); while (running) { TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 3000); if (tmqmessage) { batchCnt++; - /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/ if (0 != g_stConfInfo.showMsgFlag) { /*msg_process(tmqmessage);*/ } @@ -412,7 +410,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog } if (0 == g_stConfInfo.simCase) { - printf("consume result: msgs: %d, skip log cnt: %d, time used:%.3f second\n", batchCnt, skipLogNum, consumeTime); + printf("consume result: msgs: %d, time used:%.3f second\n", batchCnt, consumeTime); } else { printf("{consume success: %d}", totalMsgs); } From d0a83e7bf52998c7730aaa4f3ec7d28a140918d6 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 18 Jul 2022 12:30:20 +0800 Subject: [PATCH 44/88] doc: add time series db specific query features --- docs/zh/12-taos-sql/12-interval.md | 131 +++++++++++++++++------------ 1 file changed, 79 insertions(+), 52 deletions(-) diff --git a/docs/zh/12-taos-sql/12-interval.md b/docs/zh/12-taos-sql/12-interval.md index b0619ea5ce..0fece8a7d4 100644 --- a/docs/zh/12-taos-sql/12-interval.md +++ b/docs/zh/12-taos-sql/12-interval.md @@ -1,13 +1,80 @@ --- -sidebar_label: 按窗口切分聚合 -title: 按窗口切分聚合 +sidebar_label: 时序数据特色查询 +title: 时序数据特色查询 --- +TDengine是专为时序数据而研发的大数据平台，存储和计算都针对时序数据的特定进行了量身定制，在支持标准SQL的基础之上，还提供了一系列贴合时序业务场景的特色查询语法，极大的方便时序场景的应用开发。 -TDengine 支持按时间段窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。 -窗口子句用于针对查询的数据集合进行按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）三种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。 +TDengine提供的特色查询包括标签切分查询和窗口切分查询。 -## 时间窗口 +## 标签切分查询 + +超级表查询中，当需要针对标签进行数据切分然后在切分出的数据空间内再进行一系列的计算时使用标签切分子句，标签切分的语句如下： + +```sql +PARTITION BY tag_list +``` +其中 `tag_list` 是标签列的列表，还可以包括tbname伪列。 + +TDengine按如下方式处理标签切分子句： + +标签切分子句位于 `WHERE` 子句之后，且不能和 `JOIN` 子句一起使用。 +标签切分子句将超级表数据按指定的标签组合进行切分，然后对每个切分的分片进行指定的计算。计算由之后的子句定义（窗口子句、`GROUP BY` 子句或`SELECT` 子句）。 +标签切分子句可以和窗口切分子句（或 `GROUP BY` 子句）一起使用，此时后面的子句作用在每个切分的分片上。例如，下面的示例将数据按标签 `location` 进行分组，并对每个组按10分钟进行降采样，取其最大值。 + +```sql +select max(current) from meters partition by location interval(10m) +``` + +## 窗口切分查询 + +TDengine 支持按时间段窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）三种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。窗口切分查询语法如下： + +```sql +SELECT function_list FROM tb_name + [WHERE where_condition] + [SESSION(ts_col, tol_val)] + [STATE_WINDOW(col)] + [INTERVAL(interval [, offset]) [SLIDING sliding]] + [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] +``` + +在上述语法中的具体限制如下 + +### 窗口切分查询中使用函数的限制 +- 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：DIFF 以及四则运算）。 +- 此外 LAST_ROW 查询也不能与窗口聚合同时出现。 +- 标量函数（如：CEIL/FLOOR 等）也不能使用在窗口聚合查询中。 + +### 窗口子句的规则 +- 窗口子句位于标签切分子句之后，GROUP BY子句之前，且不可以和GROUP BY子句一起使用。 +- 窗口子句将数据按窗口进行切分，对每个窗口进行SELECT列表中的表达式的计算，SELECT列表中的表达式只能包含： + - 常量。 + - 聚集函数。 + - 包含上面表达式的表达式。 +- 窗口子句不可以和GROUP BY子句一起使用。 +- WHERE 语句可以指定查询的起止时间和其他过滤条件。 + +### FILL 子句 + FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： + 1. 不进行填充：NONE（默认填充模式）。 + 2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如FILL(VALUE, 1.23)，相应列为INT类型，则填充值为1。 + 3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 + 4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 + 5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 + 6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 + +:::info + +1. 使用 FILL 语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过 1 千万条具有插值的结果。 +2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。 +3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用 GROUP BY 语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了 GROUP BY 语句分组，则返回结果中每个 GROUP 内不按照时间序列严格单调递增。 + +::: + +### 时间窗口 + +时间窗口又可分为滑动时间窗口和翻转时间窗口。 INTERVAL 子句用于产生相等时间周期的窗口，SLIDING 用以指定窗口向前滑动的时间。每次执行的查询是一个时间窗口，时间窗口随着时间流动向前滑动。在定义连续查询的时候需要指定时间窗口（time window ）大小和每次前向增量时间（forward sliding times）。如图，[t0s, t0e] ，[t1s , t1e]， [t2s, t2e] 是分别是执行三次连续查询的时间窗口范围，窗口的前向滑动的时间范围 sliding time 标识 。查询过滤、聚合等操作按照每个时间窗口为独立的单位执行。当 SLIDING 与 INTERVAL 相等的时候，滑动窗口即为翻转窗口。 @@ -25,11 +92,12 @@ SLIDING 的向前滑动的时间不能超过一个窗口的时间范围。以下 SELECT COUNT(*) FROM temp_tb_1 INTERVAL(1m) SLIDING(2m); ``` -当 SLIDING 与 INTERVAL 取值相等的时候，滑动窗口即为翻转窗口。 -_ 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。 -_ 从 2.1.5.0 版本开始，INTERVAL 语句允许的最短时间间隔调整为 1 微秒（1u），当然如果所查询的 DATABASE 的时间精度设置为毫秒级，那么允许的最短时间间隔为 1 毫秒（1a）。 \* **注意**：用到 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。 +使用时间窗口需要注意： +- 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。 +- 使用 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。 +- 返回的结果中时间序列严格单调递增。 -## 状态窗口 +### 状态窗口 使用整数（布尔值）或字符串来标识产生记录时候设备的状态量。产生的记录如果具有相同的状态量数值则归属于同一个状态窗口，数值改变后该窗口关闭。如下图所示，根据状态量确定的状态窗口分别是[2019-04-28 14:22:07，2019-04-28 14:22:10]和[2019-04-28 14:22:11，2019-04-28 14:22:12]两个。（状态窗口暂不支持对超级表使用） @@ -41,7 +109,7 @@ _ 从 2.1.5.0 版本开始，INTERVAL 语句允许的最短时间间隔调整为 SELECT COUNT(*), FIRST(ts), status FROM temp_tb_1 STATE_WINDOW(status); ``` -## 会话窗口 +### 会话窗口 会话窗口根据记录的时间戳主键的值来确定是否属于同一个会话。如下图所示，如果设置时间戳的连续的间隔小于等于 12 秒，则以下 6 条记录构成 2 个会话窗口，分别是：[2019-04-28 14:22:10，2019-04-28 14:22:30]和[2019-04-28 14:23:10，2019-04-28 14:23:30]。因为 2019-04-28 14:22:30 与 2019-04-28 14:23:10 之间的时间间隔是 40 秒，超过了连续时间间隔（12 秒）。 @@ -54,48 +122,7 @@ SELECT COUNT(*), FIRST(ts), status FROM temp_tb_1 STATE_WINDOW(status); SELECT COUNT(*), FIRST(ts) FROM temp_tb_1 SESSION(ts, tol_val); ``` -这种类型的查询语法如下： - -``` -SELECT function_list FROM tb_name - [WHERE where_condition] - [SESSION(ts_col, tol_val)] - [STATE_WINDOW(col)] - [INTERVAL(interval [, offset]) [SLIDING sliding]] - [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] - -SELECT function_list FROM stb_name - [WHERE where_condition] - [INTERVAL(interval [, offset]) [SLIDING sliding]] - [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] - [GROUP BY tags] -``` - -- 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：DIFF 以及四则运算）。 -- 此外 LAST_ROW 查询也不能与窗口聚合同时出现。 -- 标量函数（如：CEIL/FLOOR 等）也不能使用在窗口聚合查询中。 -- - -- WHERE 语句可以指定查询的起止时间和其他过滤条件。 -- FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： - 1. 不进行填充：NONE（默认填充模式）。 - 2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。 - 3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 - 4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 - 5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 - 6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 - -:::info - -1. 使用 FILL 语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过 1 千万条具有插值的结果。 -2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。 -3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用 GROUP BY 语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了 GROUP BY 语句分组，则返回结果中每个 GROUP 内不按照时间序列严格单调递增。 - -::: - -时间聚合也常被用于连续查询场景，可以参考文档 [连续查询(Continuous Query)](/develop/continuous-query)。 - -## 示例 +### 示例 智能电表的建表语句如下： From 6460514147e3fc8dcea76cff887c9d5732cf912f Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 18 Jul 2022 13:22:21 +0800 Subject: [PATCH 45/88] enh: grant refactor and grant check --- include/common/tcommon.h | 20 --------- include/common/tgrant.h | 47 +++++++++++++++++++++ source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 23 +++++++--- source/dnode/mnode/impl/inc/mndInt.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 9 ++-- source/dnode/mnode/impl/src/mndDnode.c | 9 ++-- source/dnode/mnode/impl/src/mndUser.c | 9 ++-- source/dnode/vnode/inc/vnode.h | 1 + 8 files changed, 78 insertions(+), 41 deletions(-) create mode 100644 include/common/tgrant.h diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5aee218ddc..444a17c7f8 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -268,26 +268,6 @@ typedef struct SSortExecInfo { int32_t readBytes; // read io bytes } SSortExecInfo; -//====================================================================================================================== -// for grant -typedef enum { - TSDB_GRANT_ALL, - TSDB_GRANT_TIME, - TSDB_GRANT_USER, - TSDB_GRANT_DB, - TSDB_GRANT_TIMESERIES, - TSDB_GRANT_DNODE, - TSDB_GRANT_ACCT, - TSDB_GRANT_STORAGE, - TSDB_GRANT_SPEED, - TSDB_GRANT_QUERY_TIME, - TSDB_GRANT_CONNS, - TSDB_GRANT_STREAMS, - TSDB_GRANT_CPU_CORES, -} EGrantType; - -int32_t grantCheck(EGrantType grant); - #ifdef __cplusplus } #endif diff --git a/include/common/tgrant.h b/include/common/tgrant.h new file mode 100644 index 0000000000..ad23c661b1 --- /dev/null +++ b/include/common/tgrant.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_GRANT_H_ +#define _TD_COMMON_GRANT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" + +typedef enum { + TSDB_GRANT_ALL, + TSDB_GRANT_TIME, + TSDB_GRANT_USER, + TSDB_GRANT_DB, + TSDB_GRANT_TIMESERIES, + TSDB_GRANT_DNODE, + TSDB_GRANT_ACCT, + TSDB_GRANT_STORAGE, + TSDB_GRANT_SPEED, + TSDB_GRANT_QUERY_TIME, + TSDB_GRANT_CONNS, + TSDB_GRANT_STREAMS, + TSDB_GRANT_CPU_CORES, +} EGrantType; + +int32_t grantCheck(EGrantType grant); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_GRANT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 06c782e217..93df7f8ab2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -153,9 +153,15 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp switch (qtype) { case QUERY_QUEUE: - vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); - dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pQueryQ, pMsg); + if ((pMsg->msgType == TDMT_SCH_QUERY) && (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS)) { + terrno = TSDB_CODE_GRANT_EXPIRED; + code = terrno; + dDebug("vgId:%d, msg:%p put into vnode-query queue failed since %s", pVnode->vgId, pMsg, terrstr()); + } else { + vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); + dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pQueryQ, pMsg); + } break; case STREAM_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg); @@ -166,9 +172,14 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - - dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pWriteQ, pMsg); + if ((pMsg->msgType == TDMT_VND_SUBMIT) && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) { + terrno = TSDB_CODE_VND_NO_WRITE_AUTH; + code = terrno; + dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr()); + } else { + dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pWriteQ, pMsg); + } break; case SYNC_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 8ad1ac56e4..f72b69a7de 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -24,6 +24,7 @@ #include "tcache.h" #include "tdatablock.h" #include "tglobal.h" +#include "tgrant.h" #include "tqueue.h" #include "ttime.h" #include "version.h" diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e3b0f3c157..b4b1b383e7 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -509,11 +509,10 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { SUserObj *pUser = NULL; SCreateDbReq createReq = {0}; - // code = grantCheck(TSDB_GRANT_DB); - // if (code != 0) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_DB)) != 0) { + code = terrno; + goto _OVER; + } if (tDeserializeSCreateDbReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d4c043028d..f26c1a7d32 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -621,11 +621,10 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SCreateDnodeReq createReq = {0}; - // code = grantCheck(TSDB_GRANT_DNODE); - // if (code != TSDB_CODE_SUCCESS) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_DNODE)) != 0) { + code = terrno; + goto _OVER; + } if (tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5c361ed28d..0452659d47 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -363,11 +363,10 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - // code = grantCheck(TSDB_GRANT_USER); - // if (code != TSDB_CODE_SUCCESS) { - // terrno = code; - // goto _OVER; - // } + if ((terrno = grantCheck(TSDB_GRANT_USER)) != 0) { + code = terrno; + goto _OVER; + } code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6b5f795eb0..8191501469 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -27,6 +27,7 @@ #include "wal.h" #include "tcommon.h" +#include "tgrant.h" #include "tfs.h" #include "tmsg.h" #include "trow.h" From b52c734f885461d0b6831db2387af8f96435b175 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 18 Jul 2022 13:31:13 +0800 Subject: [PATCH 46/88] fix: free vMetaRsp.pSchemas --- source/dnode/vnode/src/meta/metaQuery.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index a7fef795ee..776a3c5b7f 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -205,7 +205,7 @@ _query: } tDecoderInit(&dc, pData, nData); - tDecodeSSchemaWrapper(&dc, &schema); + tDecodeSSchemaWrapperEx(&dc, &schema); pSchema = tCloneSSchemaWrapper(&schema); tDecoderClear(&dc); @@ -470,9 +470,9 @@ int64_t metaGetTbNum(SMeta *pMeta) { } // N.B. Called by statusReq per second -int64_t metaGetTimeSeriesNum(SMeta *pMeta) { +int64_t metaGetTimeSeriesNum(SMeta *pMeta) { // TODO - return 400; + return 400; } typedef struct { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b49e12fc08..e5322f1bd3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -630,6 +630,9 @@ _exit: tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp); tEncoderClear(&ec); + if (vMetaRsp.pSchemas) { + taosMemoryFree(vMetaRsp.pSchemas); + } return 0; } From e37bdbd5c996e12af74d499290b15a809360ac80 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Jul 2022 13:37:18 +0800 Subject: [PATCH 47/88] feat(query): add histogram function scalar version TD-17344 --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 215 ++++++++++++++++++++++++++++ 3 files changed, 217 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index ab53c37279..1f20a64314 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -116,6 +116,7 @@ int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index aba1df216e..72eabafa16 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2331,6 +2331,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, .processFunc = histogramFunction, + .sprocessFunc = histogramScalarFunction, .finalizeFunc = histogramFinalize, .invertFunc = NULL, .combineFunc = histogramCombine, diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 7dcba0e1b4..cadeaa0164 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2601,3 +2601,218 @@ int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SSca pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; } + +typedef enum { UNKNOWN_BIN = 0, USER_INPUT_BIN, LINEAR_BIN, LOG_BIN } EHistoBinType; + +static int8_t getHistogramBinType(char* binTypeStr) { + int8_t binType; + if (strcasecmp(binTypeStr, "user_input") == 0) { + binType = USER_INPUT_BIN; + } else if (strcasecmp(binTypeStr, "linear_bin") == 0) { + binType = LINEAR_BIN; + } else if (strcasecmp(binTypeStr, "log_bin") == 0) { + binType = LOG_BIN; + } else { + binType = UNKNOWN_BIN; + } + + return binType; +} + +typedef struct SHistoFuncBin { + double lower; + double upper; + int64_t count; + double percentage; +} SHistoFuncBin; + +static bool getHistogramBinDesc(SHistoFuncBin** bins, int32_t* binNum, char* binDescStr, int8_t binType, bool normalized) { + cJSON* binDesc = cJSON_Parse(binDescStr); + int32_t numOfBins; + double* intervals; + if (cJSON_IsObject(binDesc)) { /* linaer/log bins */ + int32_t numOfParams = cJSON_GetArraySize(binDesc); + int32_t startIndex; + if (numOfParams != 4) { + return false; + } + + cJSON* start = cJSON_GetObjectItem(binDesc, "start"); + cJSON* factor = cJSON_GetObjectItem(binDesc, "factor"); + cJSON* width = cJSON_GetObjectItem(binDesc, "width"); + cJSON* count = cJSON_GetObjectItem(binDesc, "count"); + cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity"); + + if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) { + return false; + } + + if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000 + return false; + } + + if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) || + (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) { + return false; + } + + int32_t counter = (int32_t)count->valueint; + if (infinity->valueint == false) { + startIndex = 0; + numOfBins = counter + 1; + } else { + startIndex = 1; + numOfBins = counter + 3; + } + + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) { + // linear bin process + if (width->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble + i * width->valuedouble; + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) { + // log bin process + if (start->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0); + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else { + taosMemoryFree(intervals); + return false; + } + + if (infinity->valueint == true) { + intervals[0] = -INFINITY; + intervals[numOfBins - 1] = INFINITY; + // in case of desc bin orders, -inf/inf should be swapped + ASSERT(numOfBins >= 4); + if (intervals[1] > intervals[numOfBins - 2]) { + TSWAP(intervals[0], intervals[numOfBins - 1]); + } + } + } else if (cJSON_IsArray(binDesc)) { /* user input bins */ + if (binType != USER_INPUT_BIN) { + return false; + } + numOfBins = cJSON_GetArraySize(binDesc); + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + cJSON* bin = binDesc->child; + if (bin == NULL) { + taosMemoryFree(intervals); + return false; + } + int i = 0; + while (bin) { + intervals[i] = bin->valuedouble; + if (!cJSON_IsNumber(bin)) { + taosMemoryFree(intervals); + return false; + } + if (i != 0 && intervals[i] <= intervals[i - 1]) { + taosMemoryFree(intervals); + return false; + } + bin = bin->next; + i++; + } + } else { + return false; + } + + *binNum = numOfBins - 1; + *bins = taosMemoryCalloc(numOfBins, sizeof(SHistoFuncBin)); + for (int32_t i = 0; i < *binNum; ++i) { + (*bins)[i].lower = intervals[i] < intervals[i + 1] ? intervals[i] : intervals[i + 1]; + (*bins)[i].upper = intervals[i + 1] > intervals[i] ? intervals[i + 1] : intervals[i]; + (*bins)[i].count = 0; + } + + taosMemoryFree(intervals); + return true; +} + +int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + SHistoFuncBin *bins; + int32_t numOfBins = 0; + int32_t totalCount = 0; + + int8_t binType = getHistogramBinType(varDataVal(pInput[1].columnData->pData)); + char* binDesc = varDataVal(pInput[2].columnData->pData); + int64_t normalized = *(int64_t *)(pInput[3].columnData->pData); + + int32_t type = GET_PARAM_TYPE(pInput); + if (!getHistogramBinDesc(&bins, &numOfBins, binDesc, binType, (bool)normalized)) { + return TSDB_CODE_FAILED; + } + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + char* data = colDataGetData(pInputData, i); + double v; + GET_TYPED_DATA(v, double, type, data); + + for (int32_t k = 0; k < numOfBins; ++k) { + if (v > bins[k].lower && v <= bins[k].upper) { + bins[k].count++; + totalCount++; + break; + } + } + } + + if (normalized) { + for (int32_t k = 0; k < numOfBins; ++k) { + if (totalCount != 0) { + bins[k].percentage = bins[k].count / (double)totalCount; + } else { + bins[k].percentage = 0; + } + } + } + + for (int32_t k = 0; k < numOfBins; ++k) { + int32_t len; + char buf[512] = {0}; + if (!normalized) { + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}", + bins[k].lower, bins[k].upper, bins[k].count); + } else { + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", + bins[k].lower, bins[k].upper, bins[k].percentage); + } + varDataSetLen(buf, len); + colDataAppend(pOutputData, k, buf, false); + } + + taosMemoryFree(bins); + pOutput->numOfRows = numOfBins; + return TSDB_CODE_SUCCESS; +} From 158c6ae3e419a0dc8f1de9a615078057e2609805 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 18 Jul 2022 13:51:31 +0800 Subject: [PATCH 48/88] fix(tmq): table not found --- source/dnode/vnode/src/tq/tqExec.c | 13 ++++++++++--- source/libs/executor/src/executorMain.c | 2 ++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index eb17f06cc9..618b32239a 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -64,9 +64,16 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa qTaskInfo_t task = pExec->execCol.task[0]; if (qStreamPrepareScan(task, pOffset) < 0) { - ASSERT(pOffset->type == TMQ_OFFSET__LOG); - pRsp->rspOffset = *pOffset; - return 0; + if (pOffset->type == TMQ_OFFSET__LOG) { + pRsp->rspOffset = *pOffset; + return 0; + } else { + tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1); + if (qStreamPrepareScan(task, pOffset) < 0) { + pRsp->rspOffset = *pOffset; + return 0; + } + } } int32_t rowCnt = 0; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 37d2521e5d..9d3d62cabf 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -332,6 +332,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); uid = pTableInfo->uid; ts = INT64_MIN; + } else { + return -1; } } /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ From fc67efb5aa9860d9a102197b9d8f4e67ef2eafb8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Jul 2022 13:52:33 +0800 Subject: [PATCH 49/88] fix(query): fix memory leak. --- source/libs/executor/inc/executil.h | 4 ++-- source/libs/executor/src/executil.c | 8 ++++++-- source/libs/executor/src/executorimpl.c | 6 +++++- source/libs/executor/src/timewindowoperator.c | 15 ++++++++++++++- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f9aba30a46..330ccd68a9 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -96,9 +96,9 @@ static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo } void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); -void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); - void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); + +void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); bool hasDataInGroupInfo(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 17484ec84b..f10dbf5349 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -72,8 +72,12 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { assert(pGroupResInfo != NULL); - taosArrayDestroy(pGroupResInfo->pRows); - pGroupResInfo->pRows = NULL; + for(int32_t i = 0; i < taosArrayGetSize(pGroupResInfo->pRows); ++i) { + SResKeyPos* pRes = taosArrayGetP(pGroupResInfo->pRows, i); + taosMemoryFree(pRes); + } + + pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows); pGroupResInfo->index = 0; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fe90e58527..8381fae019 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -377,6 +377,10 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } +void cleanupExecTimeWindowInfo(SColumnInfoData* pColData) { + colDataDestroy(pColData); +} + void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { @@ -3737,7 +3741,7 @@ void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - taosArrayDestroyEx(pInfo->groupResInfo.pRows, freeItem); + cleanupGroupResInfo(&pInfo->groupResInfo); taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0fa29fca74..0feb4430bc 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1513,12 +1513,25 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(param); } +static void freeItem(void* param) { + SGroupKeys *pKey = (SGroupKeys*) param; + taosMemoryFree(pKey->pData); +} + void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - taosArrayDestroy(pInfo->pRecycledPages); + pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); + pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); + taosArrayDestroyEx(pInfo->pPrevValues, freeItem); + pInfo->pPrevValues = NULL; + pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + + cleanupGroupResInfo(&pInfo->groupResInfo); + colDataDestroy(&pInfo->twAggSup.timeWindowData); taosMemoryFreeClear(param); } From 2dfa77f2ced6bb3232b3579fd8df54e570d65229 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 18 Jul 2022 13:57:13 +0800 Subject: [PATCH 50/88] update --- tests/system-test/1-insert/update_data.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index c2b8e7ab08..27e1559d7e 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -46,20 +46,6 @@ class TDTestCase: 'col13': f'nchar({self.str_length})', 'col_ts' : 'timestamp' } - self.tinyint = None - self.smallint = None - self.int = None - self.bigint = None - self.untinyint = None - self.unsmallint = None - self.unint = None - self.unbigint = None - self.bool = None - self.float = None - self.double = None - self.binary = None - self.tnchar = None - def data_check(self,tbname,col_name,col_type,value): tdSql.query(f'select {col_name} from {tbname}') @@ -147,7 +133,6 @@ class TDTestCase: nchar_length = random.randint(0,self.str_length) up_binary = tdCom.getLongName(binary_length) up_nchar = tdCom.getLongName(nchar_length) - for col_name,col_type in column_dict.items(): if tb_type == 'ntb': tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})') @@ -210,7 +195,6 @@ class TDTestCase: tdSql.execute(f'alter table {self.ntbname} modify column {col_name} nchar({self.str_length+1})') self.update_and_check_data(self.ntbname,col_name,col_type,up_nchar,self.dbname) tdSql.execute(f'drop table {self.ntbname}') - self.update_data_check(self.ctbname,self.column_dict,self.dbname,'ctb',self.stbname) for col_name,col_type in self.column_dict.items(): tdSql.execute(f'create table {self.stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') @@ -226,7 +210,6 @@ class TDTestCase: self.update_and_check_data(self.ctbname,col_name,col_type,up_nchar,self.dbname) tdSql.execute(f'drop table {self.stbname}') - def update_check_error(self): tdSql.execute(f'drop database if exists {self.dbname}') tdSql.execute(f'create database {self.dbname}') From b83968697d65b72c2885ba94ef47d6baf52971b0 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 14:02:27 +0800 Subject: [PATCH 51/88] test:update case for cachemodel --- .../0-others/{cachelast.py => cachemodel.py} | 86 +++++++++++-------- tests/system-test/fulltest.sh | 2 +- 2 files changed, 50 insertions(+), 38 deletions(-) rename tests/system-test/0-others/{cachelast.py => cachemodel.py} (62%) diff --git a/tests/system-test/0-others/cachelast.py b/tests/system-test/0-others/cachemodel.py similarity index 62% rename from tests/system-test/0-others/cachelast.py rename to tests/system-test/0-others/cachemodel.py index 2df6b8c9cc..09538e6678 100644 --- a/tests/system-test/0-others/cachelast.py +++ b/tests/system-test/0-others/cachemodel.py @@ -56,6 +56,16 @@ class TDTestCase: } return numbers.get(value, 'other') + def getCacheModelNum(self,str): + numbers = { + "none" : 0, + "last_row" : 1, + "last_value" : 2, + "both" : 3 + + } + return numbers.get(str, 'other') + def prepare_datas(self): for i in range(4): str = self.getCacheModelStr(i) @@ -69,7 +79,7 @@ class TDTestCase: tdSql.execute(" insert into tb1 values(now , %d, %f)" %(k,k*10) ) tdSql.execute(" insert into tb2 values(now , %d, %f)" %(k,k*10) ) - def check_cache_last_sets(self): + def check_cachemodel_sets(self): # check cache_last value for database @@ -84,52 +94,54 @@ class TDTestCase: # print(cache_last_value) if dbname in ["information_schema" , "performance_schema"]: continue - cache_lasts[dbname]=cache_last_value + cache_lasts[dbname]=self.getCacheModelNum(cache_last_value) # cache_last_set value for k , v in cache_lasts.items(): - if k=="testdb_"+str(v): - tdLog.info(" database %s cache_last value check pass, value is %s "%(k,v) ) + if k=="testdb_"+str(self.getCacheModelStr(v)): + tdLog.info(" database %s cache_last value check pass, value is %s "%(k,self.getCacheModelStr(v)) ) else: - tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,v) ) + tdLog.exit(" database %s cache_last value check fail, value is %s "%(k,self.getCacheModelStr(v)) ) # # check storage layer implementation - # buildPath = self.getBuildPath() - # if (buildPath == ""): - # tdLog.exit("taosd not found!") - # else: - # tdLog.info("taosd found in %s" % buildPath) - # dataPath = buildPath + "/../sim/dnode1/data" - # abs_vnodePath = os.path.abspath(dataPath)+"/vnode/" - # tdLog.info("abs_vnodePath: %s" % abs_vnodePath) + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + dataPath = buildPath + "/../sim/dnode1/data" + abs_vnodePath = os.path.abspath(dataPath)+"/vnode/" + tdLog.info("abs_vnodePath: %s" % abs_vnodePath) - # tdSql.query(" show dnodes ") - # dnode_id = tdSql.queryResult[0][0] + tdSql.query(" show dnodes ") + dnode_id = tdSql.queryResult[0][0] - # for dbname in cache_lasts.keys(): - # print(dbname) - # tdSql.execute(" use %s" % dbname) - # tdSql.query(" show vgroups ") - # vgroups_infos = tdSql.queryResult - # for vgroup_info in vgroups_infos: - # vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json" - # vnode_info_of_db = f"cat {vnode_json}" - # vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8") - # infoDict = json.loads(vnode_info) - # vnode_json_of_dbname = f"{dnode_id}."+ dbname - # config = infoDict["config"] - # if infoDict["config"]["dbname"] == vnode_json_of_dbname: - # if "cachelast" in infoDict["config"]: - # if int(infoDict["config"]["cachelast"]) != cache_lasts[dbname]: - # tdLog.exit("cachelast value is error in vnode.json of vnode%d "%(vgroup_info[0])) - # else: - # tdLog.exit("cachelast not found in vnode.json of vnode%d "%(vgroup_info[0])) + for dbname in cache_lasts.keys(): + # print(dbname) + tdSql.execute(" use %s" % dbname) + tdSql.query(" show vgroups ") + vgroups_infos = tdSql.queryResult + for vgroup_info in vgroups_infos: + vnode_json = abs_vnodePath + "/vnode" +f"{vgroup_info[0]}/" + "vnode.json" + vnode_info_of_db = f"cat {vnode_json}" + vnode_info = subprocess.check_output(vnode_info_of_db, shell=True).decode("utf-8") + infoDict = json.loads(vnode_info) + vnode_json_of_dbname = f"{dnode_id}."+ dbname + config = infoDict["config"] + if infoDict["config"]["dbname"] == vnode_json_of_dbname: + if "cacheLast" in infoDict["config"]: + if int(infoDict["config"]["cacheLast"]) != cache_lasts[dbname]: + tdLog.exit("cachemodel value is error in vnode.json of vnode%d "%(vgroup_info[0])) + else: + tdLog.info("cachemodel value is success in vnode.json of vnode%d "%(vgroup_info[0])) + else: + tdLog.exit("cacheLast not found in vnode.json of vnode%d "%(vgroup_info[0])) - def restart_check_cache_last_sets(self): + def restart_check_cachemodel_sets(self): for i in range(3): tdSql.query("show dnodes") @@ -137,14 +149,14 @@ class TDTestCase: tdDnodes.stop(index) tdDnodes.start(index) time.sleep(3) - self.check_cache_last_sets() + self.check_cachemodel_sets() def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring self.illegal_params() self.prepare_datas() - self.check_cache_last_sets() - self.restart_check_cache_last_sets() + self.check_cachemodel_sets() + self.restart_check_cachemodel_sets() def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index ddbbefe739..d1957b96a5 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -10,7 +10,7 @@ python3 ./test.py -f 0-others/taosdMonitor.py python3 ./test.py -f 0-others/udfTest.py python3 ./test.py -f 0-others/udf_create.py python3 ./test.py -f 0-others/udf_restart_taosd.py -python3 ./test.py -f 0-others/cachelast.py +python3 ./test.py -f 0-others/cachemodel.py python3 ./test.py -f 0-others/udf_cfg1.py python3 ./test.py -f 0-others/udf_cfg2.py From 3cb05f152377d2bc7b8ed95946032a0aa379666d Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 14:25:08 +0800 Subject: [PATCH 52/88] update case for last_row --- tests/system-test/2-query/last_row.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 22f165342a..b856590787 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -602,13 +602,13 @@ class TDTestCase: tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from testdb.stb1)") tdSql.checkData(0,1,None) - tdSql.query("select ts , last_row(c1) ,c1 from (select ts , max(c1) c1 ,t1 from testdb.stb1 where ts >now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts Date: Mon, 18 Jul 2022 14:47:53 +0800 Subject: [PATCH 53/88] feat(query): add top/bottom function scalar version --- include/libs/scalar/scalar.h | 2 ++ source/libs/function/src/builtins.c | 2 ++ source/libs/scalar/src/sclfunc.c | 29 +++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 1f20a64314..b667207684 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -117,6 +117,8 @@ int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t topScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t bottomScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 72eabafa16..b6a351b4a6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2086,6 +2086,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, .processFunc = topFunction, + .sprocessFunc = topScalarFunction, .finalizeFunc = topBotFinalize, .combineFunc = topCombine, .pPartialFunc = "top", @@ -2100,6 +2101,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, .processFunc = bottomFunction, + .sprocessFunc = bottomScalarFunction, .finalizeFunc = topBotFinalize, .combineFunc = bottomCombine, .pPartialFunc = "bottom", diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index cadeaa0164..1cbaf9690e 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2816,3 +2816,32 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP pOutput->numOfRows = numOfBins; return TSDB_CODE_SUCCESS; } + +int32_t selectScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int32_t type = GET_PARAM_TYPE(pInput); + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, 0); + continue; + } + + char* data = colDataGetData(pInputData, i); + colDataAppend(pOutputData, i, data, false); + } + + + pOutput->numOfRows = 1; + return TSDB_CODE_SUCCESS; +} + +int32_t topScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t bottomScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} From 2512bf26aa6259e35bce5a7fbfea2596deda5cfa Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jul 2022 14:53:41 +0800 Subject: [PATCH 54/88] test: valgrind case --- tests/script/tsim/valgrind/checkError5.sim | 33 ++++++++++++++++++++-- tests/script/tsim/valgrind/checkError6.sim | 2 +- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/script/tsim/valgrind/checkError5.sim b/tests/script/tsim/valgrind/checkError5.sim index 61964d1c42..f3d418cfd1 100644 --- a/tests/script/tsim/valgrind/checkError5.sim +++ b/tests/script/tsim/valgrind/checkError5.sim @@ -26,6 +26,7 @@ print =============== step2: create db sql create database db sql use db sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" +sql create table db.c1 using db.stb tags(101, 102, "103") print =============== step3: alter stb sql_error alter table db.stb add column ts int @@ -42,9 +43,8 @@ sql alter table db.stb drop tag c1 sql alter table db.stb drop tag t5 sql alter table db.stb MODIFY tag t3 binary(32) sql alter table db.stb rename tag t1 tx - sql alter table db.stb comment 'abcde' ; -goto _OVER +sql drop table db.stb print =============== step4: alter tb sql create table tb (ts timestamp, a int) @@ -66,6 +66,35 @@ sql alter table tb add column h binary(10) sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb sql select * from tb order by ts desc +print =============== step5: alter stb and insert data +sql create table stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" +sql show db.stables +sql describe stb +sql_error alter table stb add column ts int + +sql create table db.ctb using db.stb tags(101, 102, "103") +sql insert into db.ctb values(now, 1, "2") +sql show db.tables +sql select * from db.stb +sql select * from tb + +sql alter table stb add column c3 int +sql describe stb +sql select * from db.stb +sql select * from tb +sql insert into db.ctb values(now+1s, 1, 2, 3) +sql select * from db.stb + +sql alter table db.stb add column c4 bigint +sql select * from db.stb +sql insert into db.ctb values(now+2s, 1, 2, 3, 4) + +sql alter table db.stb drop column c1 +sql reset query cache +sql select * from tb +sql insert into db.ctb values(now+3s, 2, 3, 4) +sql select * from db.stb + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index a9f66647f9..2783e94771 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -68,7 +68,7 @@ $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 0 then +if $system_content > 3 then return -1 endi From f32eee2f58c4e19994f13a20bf3c294a7021baf6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jul 2022 14:54:23 +0800 Subject: [PATCH 55/88] test: valgrind case --- tests/script/jenkins/basic.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0bdbd644a0..bff6177ad2 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -312,6 +312,7 @@ ./test.sh -f tsim/valgrind/checkError3.sim ./test.sh -f tsim/valgrind/checkError4.sim ./test.sh -f tsim/valgrind/checkError5.sim +./test.sh -f tsim/valgrind/checkError6.sim # --- vnode # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim From a4d86275d2ec7560ec29253786b9a814b4758d83 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Jul 2022 15:06:21 +0800 Subject: [PATCH 56/88] feat(query): add first/last/last_row scalar version --- include/libs/scalar/scalar.h | 1 + source/libs/function/src/builtins.c | 3 +++ source/libs/scalar/src/sclfunc.c | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index b667207684..5ab01159cf 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -119,6 +119,7 @@ int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SSca int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t topScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t bottomScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index b6a351b4a6..c1741fe879 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2231,6 +2231,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastRowFunction, + .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize }, { @@ -2251,6 +2252,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = firstFunction, + .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize, .pPartialFunc = "_first_partial", .pMergeFunc = "_first_merge", @@ -2286,6 +2288,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = lastFunction, + .sprocessFunc = firstLastScalarFunction, .finalizeFunc = firstLastFinalize, .pPartialFunc = "_last_partial", .pMergeFunc = "_last_merge", diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 1cbaf9690e..cca1528800 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2845,3 +2845,7 @@ int32_t topScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t bottomScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return selectScalarFunction(pInput, inputNum, pOutput); } + +int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} From 75371098a6145568b3dfecbcb75c02ab8082a92c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Jul 2022 15:14:20 +0800 Subject: [PATCH 57/88] refactor --- include/libs/scalar/scalar.h | 3 +-- source/libs/function/src/builtins.c | 4 ++-- source/libs/scalar/src/sclfunc.c | 6 +----- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 5ab01159cf..de7e35b7f8 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -117,8 +117,7 @@ int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); -int32_t topScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); -int32_t bottomScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index c1741fe879..07154ba8dd 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2086,7 +2086,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, .processFunc = topFunction, - .sprocessFunc = topScalarFunction, + .sprocessFunc = topBotScalarFunction, .finalizeFunc = topBotFinalize, .combineFunc = topCombine, .pPartialFunc = "top", @@ -2101,7 +2101,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, .processFunc = bottomFunction, - .sprocessFunc = bottomScalarFunction, + .sprocessFunc = topBotScalarFunction, .finalizeFunc = topBotFinalize, .combineFunc = bottomCombine, .pPartialFunc = "bottom", diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index cca1528800..a495020220 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2838,11 +2838,7 @@ int32_t selectScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara return TSDB_CODE_SUCCESS; } -int32_t topScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - return selectScalarFunction(pInput, inputNum, pOutput); -} - -int32_t bottomScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { +int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return selectScalarFunction(pInput, inputNum, pOutput); } From 23d0c9d67b528927ef47f8172928f090b01ede97 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 18 Jul 2022 15:20:20 +0800 Subject: [PATCH 58/88] fix: set output to false for projections of project logic node --- source/libs/planner/src/planPhysiCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 8250fb462f..958b4a9f23 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -255,7 +255,7 @@ static int32_t addDataBlockSlot(SPhysiPlanContext* pCxt, SNode** pNode, SDataBlo static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, true, false); + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, false, false); } static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { From 5b12fcb3a125c502a1c11f1062aa5c3316cbda80 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Jul 2022 15:23:28 +0800 Subject: [PATCH 59/88] feat(query): add selective function scalar version --- include/libs/scalar/scalar.h | 4 ++++ source/libs/function/src/builtins.c | 4 ++++ source/libs/scalar/src/sclfunc.c | 16 ++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index de7e35b7f8..770b0442ea 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -119,6 +119,10 @@ int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SSca int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 07154ba8dd..489a11da39 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2469,6 +2469,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getSampleFuncEnv, .initFunc = sampleFunctionSetup, .processFunc = sampleFunction, + .sprocessFunc = sampleScalarFunction, .finalizeFunc = sampleFinalize }, { @@ -2480,6 +2481,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getTailFuncEnv, .initFunc = tailFunctionSetup, .processFunc = tailFunction, + .sprocessFunc = tailScalarFunction, .finalizeFunc = NULL }, { @@ -2491,6 +2493,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getUniqueFuncEnv, .initFunc = uniqueFunctionSetup, .processFunc = uniqueFunction, + .sprocessFunc = uniqueScalarFunction, .finalizeFunc = NULL }, { @@ -2501,6 +2504,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getModeFuncEnv, .initFunc = modeFunctionSetup, .processFunc = modeFunction, + .sprocessFunc = modeScalarFunction, .finalizeFunc = modeFinalize, }, { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index a495020220..55debe51a8 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2845,3 +2845,19 @@ int32_t topBotScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara int32_t firstLastScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return selectScalarFunction(pInput, inputNum, pOutput); } + +int32_t sampleScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t tailScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t uniqueScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} + +int32_t modeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return selectScalarFunction(pInput, inputNum, pOutput); +} From eebbbce4f309eca86839384c2367d24f49c0b89b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Jul 2022 15:31:39 +0800 Subject: [PATCH 60/88] fix(query): filter new added child table for stream scan operator. --- include/common/tcommon.h | 2 - source/libs/executor/inc/executil.h | 5 ++- source/libs/executor/inc/executorimpl.h | 26 ++++------- source/libs/executor/src/executil.c | 27 +++++++----- source/libs/executor/src/executor.c | 21 +++++++-- source/libs/executor/src/executorimpl.c | 31 ++++++-------- source/libs/executor/src/scanoperator.c | 57 ++++++++++--------------- 7 files changed, 84 insertions(+), 85 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5aee218ddc..90627c80a0 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -80,8 +80,6 @@ typedef struct { SArray* pTableList; SHashObj* map; // speedup acquire the tableQueryInfo by table uid bool needSortTableByGroupId; - void* pTagCond; - void* pTagIndexCond; uint64_t suid; } STableListInfo; diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 330ccd68a9..625541e00a 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -15,6 +15,7 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H +#include "vnode.h" #include "function.h" #include "nodes.h" #include "plannodes.h" @@ -106,7 +107,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); -int32_t getTableList(void* metaHandle, void* vnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo); +int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo); SArray* createSortInfo(SNodeList* pNodeList); SArray* extractPartitionColInfo(SNodeList* pNodeList); SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, @@ -128,4 +129,6 @@ int32_t convertFillType(int32_t mode); int32_t resultrowComparAsc(const void* p1, const void* p2); +int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); + #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 2ffb1be260..a7e033e547 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -311,19 +311,10 @@ typedef struct STableScanInfo { int32_t dataBlockLoadFlag; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. SSampleExecInfo sample; // sample execution info - int32_t currentGroupId; int32_t currentTable; - -#if 0 - struct { - uint64_t uid; - int64_t ts; - } lastStatus; -#endif - - int8_t scanMode; - int8_t noTable; + int8_t scanMode; + int8_t noTable; } STableScanInfo; typedef struct STagScanInfo { @@ -429,8 +420,9 @@ typedef struct SStreamScanInfo { // status for tmq // SSchemaWrapper schema; - STqOffset offset; - + STqOffset offset; + SNode* pTagCond; + SNode* pTagIndexCond; } SStreamScanInfo; typedef struct SSysTableScanInfo { @@ -874,8 +866,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, - STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId); +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, + SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); @@ -966,12 +958,12 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); int32_t createScanTableListInfo(SScanPhysiNode *pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idstr); SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId); + SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index f10dbf5349..4c86a46b6c 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -265,7 +265,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) { +int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) { SMetaReader mr = {0}; metaReaderInit(&mr, metaHandle, 0); metaGetTableEntryByUid(&mr, info->uid); @@ -280,19 +280,22 @@ static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) { if (TSDB_CODE_SUCCESS != code) { terrno = code; nodesDestroyNode(pTagCondTmp); - return false; + *pQualified = false; + + return code; } ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); SValueNode* pValue = (SValueNode*)pNew; ASSERT(pValue->node.resType.type == TSDB_DATA_TYPE_BOOL); - bool result = pValue->datum.b; + *pQualified = pValue->datum.b; + nodesDestroyNode(pNew); - return result; + return TSDB_CODE_SUCCESS; } -int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) { +int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo)); @@ -304,8 +307,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, pListInfo->suid = pScanNode->suid; - SNode* pTagCond = (SNode*)pListInfo->pTagCond; - SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond; if (pScanNode->tableType == TSDB_SUPER_TABLE) { if (pTagIndexCond) { SIndexMetaArg metaArg = { @@ -345,9 +346,14 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, int32_t i = 0; while (i < taosArrayGetSize(pListInfo->pTableList)) { STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i); - bool isOk = isTableOk(info, pTagCond, metaHandle); - if (terrno) return terrno; - if (!isOk) { + + bool qualified = true; + code = isTableOk(info, pTagCond, metaHandle, &qualified); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (!qualified) { taosArrayRemove(pListInfo->pTableList, i); continue; } @@ -362,7 +368,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, // put into list as default group, remove it if grouping sorting is required later taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList); - return code; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index f2008ed97a..90952d5786 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -153,7 +153,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { return pTaskInfo; } -static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList) { +static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); // let's discard the tables those are not created according to the queried super table. @@ -164,7 +164,7 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons int32_t code = metaGetTableEntryByUid(&mr, *id); if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); + qError("failed to get table meta, uid:%" PRIu64 " code:%s, %s", *id, tstrerror(terrno), idstr); continue; } @@ -172,6 +172,21 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) { continue; } + + if (pScanInfo->pTagCond != NULL) { + bool qualified = false; + STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid, .lastKey = 0}; + code = isTableOk(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr); + continue; + } + + if (!qualified) { + continue; + } + } + /*pScanInfo->pStreamScanOp->pTaskInfo->tableqinfoList.*/ // handle multiple partition @@ -194,7 +209,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo int32_t code = 0; SStreamScanInfo* pScanInfo = pInfo->info; if (isAdd) { // add new table id - SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList); + SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8381fae019..06bb096e59 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "filter.h" #include "function.h" #include "functionMgt.h" @@ -4369,8 +4368,7 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC } SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, - uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, - const char* pUser) { + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* pUser) { int32_t type = nodeType(pPhyNode); if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { @@ -4378,7 +4376,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, - pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); + pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { pTaskInfo->code = code; return NULL; @@ -4398,7 +4396,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, - pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); + pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { pTaskInfo->code = code; return NULL; @@ -4411,7 +4409,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } SOperatorInfo* pOperator = - createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); + createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; @@ -4428,15 +4426,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo }; if (pHandle) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, - pTableScanNode->groupSort, pHandle, pTableListInfo, queryId, taskId); + pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { pTaskInfo->code = code; return NULL; } } - SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo, &twSup); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -4445,7 +4442,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTableListInfo); + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = terrno; return NULL; @@ -4481,8 +4478,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; - int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, - queryId, taskId); + int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -4506,7 +4502,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES); for (int32_t i = 0; i < size; ++i) { SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); - ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo, pUser); + ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pUser); if (ops[i] == NULL) { return NULL; } @@ -4689,6 +4685,7 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { return pList; } +#if 0 STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, const char* idstr) { int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo); @@ -4722,6 +4719,7 @@ _error: terrno = code; return NULL; } +#endif static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanInfo** ppInfo) { if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -4765,6 +4763,7 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return -1; } +#if 0 int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHandle, int64_t uid, int64_t ts) { STableScanInfo* pTableScanInfo = NULL; if (extractTbscanInStreamOpTree(pOperator, &pTableScanInfo) < 0) { @@ -4788,6 +4787,7 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa // TODO: set uid and ts to data reader return 0; } +#endif int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) { int32_t code = TDB_CODE_SUCCESS; @@ -4939,10 +4939,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead } (*pTaskInfo)->sql = sql; - (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond; - (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond; - (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, - &(*pTaskInfo)->tableqinfoList, pPlan->user); + (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond, pPlan->pTagIndexCond, pPlan->user); if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b80b9af237..e60f6f8a5b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1497,9 +1497,8 @@ static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFree(pStreamScan); } -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, - SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, - uint64_t taskId) { +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, + SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup) { SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1512,6 +1511,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; + pInfo->pTagCond = pTagCond; + int32_t numOfCols = 0; pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); @@ -2550,26 +2551,19 @@ typedef struct STableMergeScanInfo { int32_t tableEndIndex; bool hasGroupId; uint64_t groupId; + SArray* dataReaders; // array of tsdbReaderT* + SReadHandle readHandle; + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; - SArray* dataReaders; // array of tsdbReaderT* - SReadHandle readHandle; - - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort - - SArray* pSortInfo; - SSortHandle* pSortHandle; - - SSDataBlock* pSortInputBlock; - int64_t startTs; // sort start time - - SArray* sortSourceParams; - uint64_t queryId; - uint64_t taskId; + SSDataBlock* pSortInputBlock; + int64_t startTs; // sort start time + SArray* sortSourceParams; SFileBlockLoadRecorder readRecorder; - int64_t numOfRows; - // int32_t prevGroupId; // previous table group id + int64_t numOfRows; SScanInfo scanInfo; int32_t scanTimes; SNode* pFilterNode; // filter info, which is push down by optimizer @@ -2584,26 +2578,25 @@ typedef struct STableMergeScanInfo { SExprInfo* pPseudoExpr; int32_t numOfPseudoExpr; SqlFunctionCtx* pPseudoCtx; - // int32_t* rowEntryInfoOffset; SQueryTableDataCond cond; - int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan - int32_t dataBlockLoadFlag; - SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time - // window to check if current data block needs to be loaded. - + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + int32_t dataBlockLoadFlag; + // if the upstream is an interval operator, the interval info is also kept here to get the time + // window to check if current data block needs to be loaded. + SInterval interval; SSampleExecInfo sample; // sample execution info } STableMergeScanInfo; int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) { - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTableListInfo); + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idStr) { + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { return code; } if (taosArrayGetSize(pTableListInfo->pTableList) == 0) { - qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId); + qDebug("no table qualified for query, %s" PRIx64, idStr); return TSDB_CODE_SUCCESS; } @@ -3027,8 +3020,7 @@ int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { } SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, - uint64_t taskId) { + SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3067,9 +3059,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->pResBlock = createResDataBlock(pDescNode); pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES); - pInfo->queryId = queryId; - pInfo->taskId = taskId; - pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); pInfo->pSortInfo = generateSortByTsInfo(pInfo->pColMatchInfo, pInfo->cond.order); From f5ae0f71c89e3ab39ca3f382fa68024c8ea0798b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 18 Jul 2022 15:40:38 +0800 Subject: [PATCH 61/88] feat: update taostools for3.0 (#15060) * feat: update taos-tools for 3.0 [TD-14141] * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 * feat: update taos-tools for 3.0 --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index b7b922268c..0b8a3373bb 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit b7b922268c4a06d9db77ffdfde0726f3d9900b72 +Subproject commit 0b8a3373bb7548f8106d13e7d3b0a988d3c4d48a From d8f8000fa81ec3fbc222ac2364869e07fb6bc6f1 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Mon, 18 Jul 2022 15:42:59 +0800 Subject: [PATCH 62/88] doc: reconstruct in sql reference --- docs/zh/12-taos-sql/12-interval.md | 36 +++++++++++-------- docs/zh/12-taos-sql/14-stream.md | 0 .../{13-operators.md => 16-operators.md} | 0 .../zh/12-taos-sql/{16-json.md => 17-json.md} | 0 .../12-taos-sql/{14-limit.md => 19-limit.md} | 0 docs/zh/12-taos-sql/22-information.md | 5 +++ 6 files changed, 26 insertions(+), 15 deletions(-) create mode 100644 docs/zh/12-taos-sql/14-stream.md rename docs/zh/12-taos-sql/{13-operators.md => 16-operators.md} (100%) rename docs/zh/12-taos-sql/{16-json.md => 17-json.md} (100%) rename docs/zh/12-taos-sql/{14-limit.md => 19-limit.md} (100%) create mode 100644 docs/zh/12-taos-sql/22-information.md diff --git a/docs/zh/12-taos-sql/12-interval.md b/docs/zh/12-taos-sql/12-interval.md index 0fece8a7d4..8bf0c578e7 100644 --- a/docs/zh/12-taos-sql/12-interval.md +++ b/docs/zh/12-taos-sql/12-interval.md @@ -3,9 +3,9 @@ sidebar_label: 时序数据特色查询 title: 时序数据特色查询 --- -TDengine是专为时序数据而研发的大数据平台，存储和计算都针对时序数据的特定进行了量身定制，在支持标准SQL的基础之上，还提供了一系列贴合时序业务场景的特色查询语法，极大的方便时序场景的应用开发。 +TDengine 是专为时序数据而研发的大数据平台，存储和计算都针对时序数据的特定进行了量身定制，在支持标准 SQL 的基础之上，还提供了一系列贴合时序业务场景的特色查询语法，极大的方便时序场景的应用开发。 -TDengine提供的特色查询包括标签切分查询和窗口切分查询。 +TDengine 提供的特色查询包括标签切分查询和窗口切分查询。 ## 标签切分查询 @@ -14,13 +14,14 @@ TDengine提供的特色查询包括标签切分查询和窗口切分查询。 ```sql PARTITION BY tag_list ``` -其中 `tag_list` 是标签列的列表，还可以包括tbname伪列。 -TDengine按如下方式处理标签切分子句： +其中 `tag_list` 是标签列的列表，还可以包括 tbname 伪列。 + +TDengine 按如下方式处理标签切分子句： 标签切分子句位于 `WHERE` 子句之后，且不能和 `JOIN` 子句一起使用。 标签切分子句将超级表数据按指定的标签组合进行切分，然后对每个切分的分片进行指定的计算。计算由之后的子句定义（窗口子句、`GROUP BY` 子句或`SELECT` 子句）。 -标签切分子句可以和窗口切分子句（或 `GROUP BY` 子句）一起使用，此时后面的子句作用在每个切分的分片上。例如，下面的示例将数据按标签 `location` 进行分组，并对每个组按10分钟进行降采样，取其最大值。 +标签切分子句可以和窗口切分子句（或 `GROUP BY` 子句）一起使用，此时后面的子句作用在每个切分的分片上。例如，下面的示例将数据按标签 `location` 进行分组，并对每个组按 10 分钟进行降采样，取其最大值。 ```sql select max(current) from meters partition by location interval(10m) @@ -42,27 +43,31 @@ SELECT function_list FROM tb_name 在上述语法中的具体限制如下 ### 窗口切分查询中使用函数的限制 + - 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：DIFF 以及四则运算）。 - 此外 LAST_ROW 查询也不能与窗口聚合同时出现。 - 标量函数（如：CEIL/FLOOR 等）也不能使用在窗口聚合查询中。 ### 窗口子句的规则 -- 窗口子句位于标签切分子句之后，GROUP BY子句之前，且不可以和GROUP BY子句一起使用。 -- 窗口子句将数据按窗口进行切分，对每个窗口进行SELECT列表中的表达式的计算，SELECT列表中的表达式只能包含： + +- 窗口子句位于标签切分子句之后，GROUP BY 子句之前，且不可以和 GROUP BY 子句一起使用。 +- 窗口子句将数据按窗口进行切分，对每个窗口进行 SELECT 列表中的表达式的计算，SELECT 列表中的表达式只能包含： - 常量。 - 聚集函数。 - 包含上面表达式的表达式。 -- 窗口子句不可以和GROUP BY子句一起使用。 +- 窗口子句不可以和 GROUP BY 子句一起使用。 - WHERE 语句可以指定查询的起止时间和其他过滤条件。 ### FILL 子句 - FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： - 1. 不进行填充：NONE（默认填充模式）。 - 2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如FILL(VALUE, 1.23)，相应列为INT类型，则填充值为1。 - 3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 - 4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 - 5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 - 6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 + +FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： + +1. 不进行填充：NONE（默认填充模式）。 +2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如 FILL(VALUE, 1.23)，相应列为 INT 类型，则填充值为 1。 +3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 +4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 +5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 +6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 :::info @@ -93,6 +98,7 @@ SELECT COUNT(*) FROM temp_tb_1 INTERVAL(1m) SLIDING(2m); ``` 使用时间窗口需要注意： + - 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。 - 使用 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。 - 返回的结果中时间序列严格单调递增。 diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/zh/12-taos-sql/13-operators.md b/docs/zh/12-taos-sql/16-operators.md similarity index 100% rename from docs/zh/12-taos-sql/13-operators.md rename to docs/zh/12-taos-sql/16-operators.md diff --git a/docs/zh/12-taos-sql/16-json.md b/docs/zh/12-taos-sql/17-json.md similarity index 100% rename from docs/zh/12-taos-sql/16-json.md rename to docs/zh/12-taos-sql/17-json.md diff --git a/docs/zh/12-taos-sql/14-limit.md b/docs/zh/12-taos-sql/19-limit.md similarity index 100% rename from docs/zh/12-taos-sql/14-limit.md rename to docs/zh/12-taos-sql/19-limit.md diff --git a/docs/zh/12-taos-sql/22-information.md b/docs/zh/12-taos-sql/22-information.md new file mode 100644 index 0000000000..0695aa5172 --- /dev/null +++ b/docs/zh/12-taos-sql/22-information.md @@ -0,0 +1,5 @@ +--- +sidebar_label: Information内置数据库 +title: Information内置数据库 +--- + From 053786e6b44368e71988ac5830d0522690781bb5 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 18 Jul 2022 15:50:36 +0800 Subject: [PATCH 63/88] fix(stream): data exec --- source/dnode/mnode/impl/src/mndSubscribe.c | 17 ++- source/libs/stream/src/streamExec.c | 126 +++++++++++---------- source/libs/wal/src/walRead.c | 2 + 3 files changed, 80 insertions(+), 65 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index d67e4e8783..05e197150e 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -298,7 +298,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, + pConsumerEp->consumerId); } imbCnt++; } @@ -312,7 +313,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, + pConsumerEp->consumerId); } } } @@ -354,7 +356,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, + pConsumerEp->consumerId); } } @@ -371,8 +374,14 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ASSERT(pConsumerEp->consumerId > 0); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; + if (pRebVg->newConsumerId == pRebVg->oldConsumerId) { + mInfo("mq rebalance: skip vg %d for same consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, + pConsumerEp->consumerId); + continue; + } taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, + pConsumerEp->consumerId); } } else { // if all consumer is removed, put all vg into unassigned diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index f6eb9e32f2..b59a812678 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -143,76 +143,80 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { } static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { - int32_t cnt = 0; - void* data = NULL; while (1) { - SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); - if (qItem == NULL) { - qDebug("stream exec over, queue empty"); - break; - } - if (data == NULL) { - data = qItem; - if (qItem->type == STREAM_INPUT__DATA_BLOCK) { - /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ - } - streamQueueProcessSuccess(pTask->inputQueue); - } else { - if (streamAppendQueueItem(data, qItem) < 0) { - streamQueueProcessFail(pTask->inputQueue); + int32_t cnt = 0; + void* data = NULL; + while (1) { + SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); + if (qItem == NULL) { + qDebug("stream exec over, queue empty"); break; - } else { - cnt++; - /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + } + if (data == NULL) { + data = qItem; streamQueueProcessSuccess(pTask->inputQueue); - taosArrayDestroy(((SStreamDataBlock*)qItem)->blocks); - taosFreeQitem(qItem); + if (qItem->type == STREAM_INPUT__DATA_BLOCK) { + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + } else { + break; + } + } else { + if (streamAppendQueueItem(data, qItem) < 0) { + streamQueueProcessFail(pTask->inputQueue); + break; + } else { + cnt++; + /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ + streamQueueProcessSuccess(pTask->inputQueue); + taosArrayDestroy(((SStreamDataBlock*)qItem)->blocks); + taosFreeQitem(qItem); + } } } - } - if (pTask->taskStatus == TASK_STATUS__DROPPING) { - if (data) streamFreeQitem(data); - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - return NULL; - } - - if (data == NULL) return pRes; - - if (pTask->execType == TASK_EXEC__NONE) { - ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK); - streamTaskOutput(pTask, data); - return pRes; - } - - qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); - streamTaskExecImpl(pTask, data, pRes); - qDebug("stream task %d exec end", pTask->taskId); - - if (taosArrayGetSize(pRes) != 0) { - SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); - if (qRes == NULL) { - streamQueueProcessFail(pTask->inputQueue); - taosArrayDestroy(pRes); - return NULL; - } - qRes->type = STREAM_INPUT__DATA_BLOCK; - qRes->blocks = pRes; - if (streamTaskOutput(pTask, qRes) < 0) { - /*streamQueueProcessFail(pTask->inputQueue);*/ + if (pTask->taskStatus == TASK_STATUS__DROPPING) { + if (data) streamFreeQitem(data); taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - taosFreeQitem(qRes); return NULL; } - if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; - qRes->childId = pTask->selfChildId; - qRes->sourceVer = pSubmit->ver; - } - /*streamQueueProcessSuccess(pTask->inputQueue);*/ - pRes = taosArrayInit(0, sizeof(SSDataBlock)); - } - streamFreeQitem(data); + if (data == NULL) break; + + if (pTask->execType == TASK_EXEC__NONE) { + ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK); + streamTaskOutput(pTask, data); + return pRes; + } + + qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); + streamTaskExecImpl(pTask, data, pRes); + qDebug("stream task %d exec end", pTask->taskId); + + if (taosArrayGetSize(pRes) != 0) { + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + if (qRes == NULL) { + streamQueueProcessFail(pTask->inputQueue); + taosArrayDestroy(pRes); + return NULL; + } + qRes->type = STREAM_INPUT__DATA_BLOCK; + qRes->blocks = pRes; + if (streamTaskOutput(pTask, qRes) < 0) { + /*streamQueueProcessFail(pTask->inputQueue);*/ + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + taosFreeQitem(qRes); + return NULL; + } + if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; + qRes->childId = pTask->selfChildId; + qRes->sourceVer = pSubmit->ver; + } + /*streamQueueProcessSuccess(pTask->inputQueue);*/ + pRes = taosArrayInit(0, sizeof(SSDataBlock)); + } + + streamFreeQitem(data); + } return pRes; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 908523f2a6..c47964803a 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -483,6 +483,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) { pRead->pHead->head.version, ver); pRead->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); return -1; } @@ -491,6 +492,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) { wError("vgId:%d, unexpected wal log index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); pRead->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + ASSERT(0); return -1; } pRead->curVersion++; From be35f08d094532f89790d48b6032a60206ae91ac Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 15:52:38 +0800 Subject: [PATCH 64/88] update case about bug fix --- tests/system-test/2-query/abs.py | 3 +++ tests/system-test/2-query/last_row.py | 4 +++- tests/system-test/2-query/max_partition.py | 2 ++ tests/system-test/2-query/sample.py | 5 ++++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index 6dc65ce3c2..f0f559c6aa 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -554,6 +554,9 @@ class TDTestCase: tdSql.query("select t1 from stb1 where abs(c1+t1)=1") tdSql.checkRows(1) tdSql.checkData(0,0,0) + + tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") + tdSql.checkRows(20) tdSql.query( "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index fce3cc7043..6a75d1a1c0 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -604,7 +604,7 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.query("select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from stb1)") - tdSql.checkData(0,1,None,None) + tdSql.checkData(0,1,None) tdSql.query("select ts , last_row(c1) ,c1 from (select ts , max(c1) c1 ,t1 from stb1 where ts >now -1h and ts ="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') tdSql.checkRows(5) + tdSql.query("select unique(c1) from stb1 partition by tbname") + def support_super_table_test(self): diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index cf0d6639c2..5fa78c08e4 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -162,6 +162,8 @@ class TDTestCase: tdSql.query("select tbname , max(c1) from stb partition by tbname interval(10s)") tdSql.checkRows(self.row_nums*2) + tdSql.query("select unique(c1) from stb1 partition by tbname order by tbname") + tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)") tdSql.checkData(0,0,'sub_stb_1') tdSql.checkData(0,1,self.row_nums) diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index 84ff5a0056..b6bdd8926e 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -870,7 +870,10 @@ class TDTestCase: tdSql.query("select sample(c1 ,1000) from st") tdSql.checkRows(1000) - + # bug need fix + tdSql.query("select c1 ,t1, sample(c1,2) from db.stb1 partition by c1 ") + tdSql.query("select sample(c1,2) from db.stb1 partition by c1 ") + # tdSql.query("select c1 ,ind, sample(c1,2) from sample_db.st partition by c1 ") def run(self): import traceback From e498736de8c9dab96f94a0fd74693e4491fcce42 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Mon, 18 Jul 2022 16:35:57 +0800 Subject: [PATCH 65/88] update --- tests/system-test/1-insert/delete_data.py | 166 +++++++++++++++++----- 1 file changed, 134 insertions(+), 32 deletions(-) diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py index 6c0c2d24b7..a7eba2d97d 100644 --- a/tests/system-test/1-insert/delete_data.py +++ b/tests/system-test/1-insert/delete_data.py @@ -13,6 +13,8 @@ import random import string + +from numpy import logspace from util import constant from util.log import * from util.cases import * @@ -23,7 +25,7 @@ from util.sqlset import TDSetSql class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(),logSql) self.dbname = 'db_test' self.setsql = TDSetSql() self.ntbname = 'ntb' @@ -32,49 +34,149 @@ class TDTestCase: self.ts = 1537146000000 self.binary_str = 'taosdata' self.nchar_str = '涛思数据' - # first column must be timestamp + self.str_length = 20 self.column_dict = { - 'ts' : 'timestamp', 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})', } - self.value = [ - f'1' - ] + self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.bool_val = random.randint(0,100)%2 + self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.base_data = { + 'tinyint':self.tinyint_val, + 'smallint':self.smallint_val, + 'int':self.int_val, + 'bigint':self.bigint_val, + 'tinyint unsigned':self.untingint_val, + 'smallint unsigned':self.unsmallint_val, + 'int unsigned':self.unint_val, + 'bigint unsigned':self.unbigint_val, + 'bool':self.bool_val, + 'float':self.float_val, + 'double':self.double_val, + 'binary':self.binary_val, + 'nchar':self.nchar_val + } + def insert_base_data(self,col_type,tbname,rows,base_data): + for i in range(rows): + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint"]})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint"]})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int"]})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint"]})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["tinyint unsigned"]})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["smallint unsigned"]})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["int unsigned"]})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bigint unsigned"]})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["bool"]})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["float"]})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{base_data["double"]})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['binary']}")''') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{base_data['nchar']}")''') - self.param_list = [1,100] - def insert_data(self,column_dict,tbname,row_num): - insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) - for i in range(row_num): - insert_list = [] - - def delete_all_data(self,tbname): + def delete_all_data(self,tbname,col_type,row_num,base_data,dbname): tdSql.execute(f'delete from {tbname}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') tdSql.query(f'select * from {tbname}') tdSql.checkRows(0) - - def delete_one_row(self,tbname,column_dict): - for column_name,column_type in column_dict.items(): - if column_type.lower() == 'timestamp': + self.insert_base_data(col_type,tbname,row_num,base_data) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(row_num) + def delete_one_row(self,tbname,column_type,column_name,base_data,dbname): + tdSql.execute(f'delete from {tbname} where ts={self.ts}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {column_name} from {tbname}') + tdSql.checkRows(self.rowNum-1) + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + tdSql.checkRows(0) + if 'binary' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['binary']}")''') + elif 'nchar' in column_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts},"{base_data['nchar']}")''') + else: + tdSql.execute(f'insert into {tbname} values({self.ts},{base_data[column_type]})') + tdSql.query(f'select {column_name} from {tbname} where ts={self.ts}') + if column_type.lower() == 'float' or column_type.lower() == 'double': + if abs(tdSql.queryResult[0][0] - base_data[column_type]) / base_data[column_type] <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'{column_type} data check failure') + elif 'binary' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['binary']) + elif 'nchar' in column_type.lower(): + tdSql.checkEqual(tdSql.queryResult[0][0],base_data['nchar']) + else: + tdSql.checkEqual(tdSql.queryResult[0][0],base_data[column_type]) + + def delete_rows(self): + + + pass + def delete_error(self,tbname,column_name,column_type,base_data): + for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: + if 'binary' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['binary']}"''') + elif 'nchar' in column_type.lower(): + tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') + else: + tdSql.error('delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') - tdSql.execute(f'delete from {tbname} where {column_name}={self.ts}') - tdSql.query(f'select * from {tbname}') - tdSql.checkRows(self.rowNum-1) - tdSql.query(f'select * from {tbname} where {column_name}={self.ts}') - tdSql.checkRows(0) - tdSql.execute(f'insert into {tbname} values({self.ts},{self.value_list[0]}') - tdSql.query(f'select * from {tbname} where {column_name}={self.ts}') - tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f"))) - def delete_data_ntb(self): tdSql.execute(f'create database if not exists {self.dbname}') tdSql.execute(f'use {self.dbname}') - tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) - for i in range(self.rowNum): - tdSql.execute(f'insert into {self.ntbname} values({self.ts+i},{self.value_list[0]})') - - - # self.delete_all_data(self.ntbname) + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.ntbname} (ts timestamp,{col_name} {col_type})') + self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data) + self.delete_one_row(self.ntbname,col_type,col_name,self.base_data,self.dbname) + self.delete_all_data(self.ntbname,col_type,self.rowNum,self.base_data,self.dbname) + self.delete_error(self.ntbname,col_name,col_type,self.base_data) + for i in range(self.rowNum): + tdSql.execute(f'delete from {self.ntbname} where ts>{self.ts+i}') + tdSql.execute(f'flush database {self.dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {self.ntbname}') + tdSql.checkRows(i+1) + self.insert_base_data(col_type,self.ntbname,self.rowNum,self.base_data) + + tdSql.execute(f'drop table {self.ntbname}') def run(self): self.delete_data_ntb() From 1d30ff6190b710ad4b01ab74d5ea5d49260967ba Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 16:36:13 +0800 Subject: [PATCH 66/88] update case for bug fix --- tests/system-test/2-query/csum.py | 4 +-- tests/system-test/2-query/max_partition.py | 35 +++++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/csum.py b/tests/system-test/2-query/csum.py index 425597a919..91869cb012 100644 --- a/tests/system-test/2-query/csum.py +++ b/tests/system-test/2-query/csum.py @@ -435,8 +435,8 @@ class TDTestCase: tdSql.checkRows(40) # # bug need fix - # tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ") - # tdSql.checkRows(4) + tdSql.query("select csum(st1+c1) from stb1 partition by tbname slimit 1 ") + tdSql.checkRows(4) # tdSql.error("select csum(st1+c1) from stb1 partition by tbname limit 1 ") diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index 5fa78c08e4..0a7214ec75 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -162,12 +162,45 @@ class TDTestCase: tdSql.query("select tbname , max(c1) from stb partition by tbname interval(10s)") tdSql.checkRows(self.row_nums*2) - tdSql.query("select unique(c1) from stb1 partition by tbname order by tbname") + tdSql.query("select unique(c1) from stb partition by tbname order by tbname") tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)") tdSql.checkData(0,0,'sub_stb_1') tdSql.checkData(0,1,self.row_nums) + tdSql.query("select c1 , mavg(c1 ,2 ) from stb partition by c1") + tdSql.checkRows(72) + + tdSql.query("select c1 , diff(c1 , 0) from stb partition by c1") + tdSql.checkRows(72) + + tdSql.query("select c1 , csum(c1) from stb partition by c1") + tdSql.checkRows(80) + + tdSql.query("select c1 , sample(c1,2) from stb partition by c1 order by c1") + tdSql.checkRows(21) + # bug need fix + # tdSql.checkData(0,1,None) + + tdSql.query("select c1 , twa(c1) from stb partition by c1 order by c1") + tdSql.checkRows(11) + tdSql.checkData(0,1,0.000000000) + + tdSql.query("select c1 , irate(c1) from stb partition by c1 order by c1") + tdSql.checkRows(11) + tdSql.checkData(0,1,None) + + tdSql.query("select c1 , DERIVATIVE(c1,2,1) from stb partition by c1 order by c1") + tdSql.checkRows(72) + # bug need fix + # tdSql.checkData(0,1,None) + + + + + + + # bug need fix # tdSql.query(" select tbname , max(c1) from stb partition by tbname order by tbname slimit 5 soffset 0 ") # tdSql.checkRows(5) From cbfc1688f40794e1c8931f257acdeb229c2726cb Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 16:38:57 +0800 Subject: [PATCH 67/88] update case --- tests/system-test/2-query/abs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index f0f559c6aa..f924cb7c3d 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -555,8 +555,8 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0,0,0) - tdSql.query("select last_row(c1) from (select ts , c1 ,t1 from stb1)") - tdSql.checkRows(20) + tdSql.query("select abs(c1) from (select ts , c1 ,t1 from stb1)") + tdSql.checkRows(25) tdSql.query( "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") From 691856073bcddeed0642441ed153cae42a99ba7f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 18 Jul 2022 16:41:24 +0800 Subject: [PATCH 68/88] fix: remove the assert check of time window from when generating buffer for window computation --- source/libs/executor/src/timewindowoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0fa29fca74..da6944e35e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -63,7 +63,6 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { - assert(win->skey <= win->ekey); SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId, pTaskInfo, true, pAggSup); From 7542fa0b62f1d047082c07211c79d3bc8b8e8a03 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 18 Jul 2022 16:42:54 +0800 Subject: [PATCH 69/88] fix(tmq): topic name should be lower unless quoted --- source/client/src/tmq.c | 6 +++++- source/dnode/vnode/src/tq/tqPush.c | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 5b845fd455..e73888d9ba 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -363,7 +363,11 @@ tmq_list_t* tmq_list_new() { int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; - char* topic = strdup(src); + if (src == NULL || src[0] == 0) return -1; + char* topic = strdup(src); + if (topic[0] != '`') { + strtolower(topic, src); + } if (taosArrayPush(container, &topic) == NULL) return -1; return 0; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 4b2160434f..c929c84203 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -243,7 +243,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) void* data = taosMemoryMalloc(msgLen); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - qError("failed to copy data for stream since out of memory"); + tqError("failed to copy data for stream since out of memory"); return -1; } memcpy(data, msg, msgLen); From fa96e64ee0db1de97814d2e809dadb5b0cdba7b3 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 18 Jul 2022 16:53:20 +0800 Subject: [PATCH 70/88] os: win format negative timestamp error --- cmake/cmake.platform | 9 ++++ include/common/ttime.h | 11 ++-- source/common/src/tdatablock.c | 9 ++-- source/common/src/ttime.c | 8 +-- source/libs/function/src/builtins.c | 5 +- source/libs/scalar/src/sclfunc.c | 5 +- source/os/src/osTime.c | 78 ++++++++++++++++++++++++++++- tests/test/c/tmqDemo.c | 3 +- tests/test/c/tmqSim.c | 20 ++------ tests/tsim/src/simExe.c | 17 ++----- tools/shell/src/shellEngine.c | 17 ++----- 11 files changed, 117 insertions(+), 65 deletions(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 849d31f93e..5c6ffd4b10 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -76,15 +76,19 @@ IF ("${CPUTYPE}" STREQUAL "") IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") MESSAGE(STATUS "The current platform is amd64") SET(PLATFORM_ARCH_STR "amd64") + SET(TD_INTEL_64 TRUE) ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") MESSAGE(STATUS "The current platform is x86") SET(PLATFORM_ARCH_STR "i386") + SET(TD_INTEL_32 TRUE) ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") MESSAGE(STATUS "The current platform is aarch32") SET(PLATFORM_ARCH_STR "arm") + SET(TD_ARM_32 TRUE) ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") MESSAGE(STATUS "The current platform is aarch64") SET(PLATFORM_ARCH_STR "arm64") + SET(TD_ARM_64 TRUE) ENDIF () ELSE () # if generate ARM version: @@ -92,18 +96,23 @@ ELSE () IF (${CPUTYPE} MATCHES "aarch32") SET(PLATFORM_ARCH_STR "arm") MESSAGE(STATUS "input cpuType: aarch32") + SET(TD_ARM_32 TRUE) ELSEIF (${CPUTYPE} MATCHES "aarch64") SET(PLATFORM_ARCH_STR "arm64") MESSAGE(STATUS "input cpuType: aarch64") + SET(TD_ARM_64 TRUE) ELSEIF (${CPUTYPE} MATCHES "mips64") SET(PLATFORM_ARCH_STR "mips") MESSAGE(STATUS "input cpuType: mips64") + SET(TD_MIPS_64 TRUE) ELSEIF (${CPUTYPE} MATCHES "x64") SET(PLATFORM_ARCH_STR "amd64") MESSAGE(STATUS "input cpuType: x64") + SET(TD_INTEL_64 TRUE) ELSEIF (${CPUTYPE} MATCHES "x86") SET(PLATFORM_ARCH_STR "i386") MESSAGE(STATUS "input cpuType: x86") + SET(TD_INTEL_32 TRUE) ELSE () MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) ENDIF () diff --git a/include/common/ttime.h b/include/common/ttime.h index de55b016cd..2f4129f979 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -63,12 +63,13 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 : 1000000000; time_t t = taosTime(NULL); - struct tm* tm = taosLocalTime(&t, NULL); - tm->tm_hour = 0; - tm->tm_min = 0; - tm->tm_sec = 0; + struct tm tm; + taosLocalTime(&t, &tm); + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; - return (int64_t)taosMktime(tm) * factor; + return (int64_t)taosMktime(&tm) * factor; } int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d9ac84ec06..e1b05cc026 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1661,9 +1661,6 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } */ -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -1674,9 +1671,9 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { ms += 1000; } } - - struct tm* ptm = taosLocalTime(&tt, NULL); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + struct tm ptm = {0}; + taosLocalTime(&tt, &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index d728bbe49e..944ee6a731 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -902,7 +902,7 @@ const char* fmtts(int64_t ts) { void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) { char ts[40] = {0}; - struct tm* ptm; + struct tm ptm; int32_t fractionLen; char* format = NULL; @@ -939,10 +939,10 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) assert(false); } - ptm = taosLocalTime(", NULL); - int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm); + taosLocalTime(", &ptm); + int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); length += snprintf(ts + length, fractionLen, format, mod); - length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm); + length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm); tstrncpy(buf, ts, bufLen); } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 489a11da39..a1d3fa2968 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -194,8 +194,9 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { void static addTimezoneParam(SNodeList* pList) { char buf[6] = {0}; time_t t = taosTime(NULL); - struct tm* tmInfo = taosLocalTime(&t, NULL); - strftime(buf, sizeof(buf), "%z", tmInfo); + struct tm tmInfo; + taosLocalTime(&t, &tmInfo); + strftime(buf, sizeof(buf), "%z", &tmInfo); int32_t len = (int32_t)strlen(buf); SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 55debe51a8..b754c52bbd 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1062,8 +1062,9 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * memmove(fraction, fraction + TSDB_TIME_PRECISION_SEC_DIGITS, TSDB_TIME_PRECISION_SEC_DIGITS); } - struct tm *tmInfo = taosLocalTime((const time_t *)&timeVal, NULL); - strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", tmInfo); + struct tm tmInfo; + taosLocalTime((const time_t *)&timeVal, &tmInfo); + strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo); int32_t len = (int32_t)strlen(buf); //add timezone string diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 7e6e508817..0cb4228e42 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -357,14 +357,88 @@ FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { time_t taosTime(time_t *t) { return time(t); } -time_t taosMktime(struct tm *timep) { return mktime(timep); } +time_t taosMktime(struct tm *timep) { +#ifdef WINDOWS + struct tm tm1 = {0}; + LARGE_INTEGER t; + FILETIME f; + SYSTEMTIME s; + FILETIME ff; + SYSTEMTIME ss; + LARGE_INTEGER offset; + + time_t tt = 0; + localtime_s(&tm1, &tt); + ss.wYear = tm1.tm_year + 1900; + ss.wMonth = tm1.tm_mon + 1; + ss.wDay = tm1.tm_wday; + ss.wHour = tm1.tm_hour; + ss.wMinute = tm1.tm_min; + ss.wSecond = tm1.tm_sec; + ss.wMilliseconds = 0; + SystemTimeToFileTime(&ss, &ff); + offset.QuadPart = ff.dwHighDateTime; + offset.QuadPart <<= 32; + offset.QuadPart |= ff.dwLowDateTime; + + s.wYear = timep->tm_year + 1900; + s.wMonth = timep->tm_mon + 1; + s.wDay = timep->tm_wday; + s.wHour = timep->tm_hour; + s.wMinute = timep->tm_min; + s.wSecond = timep->tm_sec; + s.wMilliseconds = 0; + SystemTimeToFileTime(&s, &f); + t.QuadPart = f.dwHighDateTime; + t.QuadPart <<= 32; + t.QuadPart |= f.dwLowDateTime; + + t.QuadPart -= offset.QuadPart; + return (time_t)(t.QuadPart / 10000000); +#else + return mktime(timep); +#endif + } struct tm *taosLocalTime(const time_t *timep, struct tm *result) { if (result == NULL) { return localtime(timep); } #ifdef WINDOWS - localtime_s(result, timep); + if (*timep < 0) { + SYSTEMTIME ss,s; + FILETIME ff,f; + LARGE_INTEGER offset; + struct tm tm1; + time_t tt = 0; + localtime_s(&tm1, &tt); + ss.wYear = tm1.tm_year + 1900; + ss.wMonth = tm1.tm_mon + 1; + ss.wDay = tm1.tm_mday; + ss.wHour = tm1.tm_hour; + ss.wMinute = tm1.tm_min; + ss.wSecond = tm1.tm_sec; + ss.wMilliseconds = 0; + SystemTimeToFileTime(&ss, &ff); + offset.QuadPart = ff.dwHighDateTime; + offset.QuadPart <<= 32; + offset.QuadPart |= ff.dwLowDateTime; + offset.QuadPart += *timep * 10000000; + f.dwLowDateTime = offset.QuadPart & 0xffffffff; + f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff; + FileTimeToSystemTime(&f, &s); + result->tm_sec = s.wSecond; + result->tm_min = s.wMinute; + result->tm_hour = s.wHour; + result->tm_mday = s.wDay; + result->tm_mon = s.wMonth-1; + result->tm_year = s.wYear-1900; + result->tm_wday = s.wDayOfWeek; + result->tm_yday = 0; + result->tm_isdst = 0; + } else { + localtime_s(result, timep); + } #else localtime_r(timep, result); #endif diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 61c50fb0e8..784b45c92b 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -596,7 +596,8 @@ void printParaIntoFile() { g_fp = pFile; time_t tTime = taosGetTimestampSec(); - struct tm tm = *taosLocalTime(&tTime, NULL); + struct tm tm; + taosLocalTime(&tTime, &tm); taosFprintfFile(pFile, "###################################################################\n"); taosFprintfFile(pFile, "# configDir: %s\n", configDir); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 5459e3f159..7030f6fea6 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -171,7 +171,8 @@ static void printHelp() { char* getCurrentTimeString(char* timeString) { time_t tTime = taosGetTimestampSec(); - struct tm tm = *taosLocalTime(&tTime, NULL); + struct tm tm; + taosLocalTime(&tTime, &tm); sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); @@ -420,18 +421,6 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { ms = val % 1000; } - /* - comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -443,8 +432,9 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { } } - struct tm* ptm = taosLocalTime(&tt, NULL); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + struct tm ptm; + taosLocalTime(&tt, &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index aaad76bb53..b993a8dbf1 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -635,7 +635,7 @@ bool simCreateTaosdConnect(SScript *script, char *rest) { bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { char timeStr[30] = {0}; time_t tt; - struct tm *tp; + struct tm tp; SCmdLine *line = &script->lines[script->linePos]; int32_t ret = -1; @@ -768,20 +768,9 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } else { tt = (*(int64_t *)row[i]) / 1000000000; } - /* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif - - tp = taosLocalTime(&tt, NULL); - strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", tp); + taosLocalTime(&tt, &tp); + strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp); if (precision == TSDB_TIME_PRECISION_MILLI) { sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000)); } else if (precision == TSDB_TIME_PRECISION_MICRO) { diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 4cfa46bd3c..0982dbc019 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -231,18 +231,6 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { ms = val % 1000; } - /* - comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ - -#ifdef WINDOWS - if (tt < 0) tt = 0; -#endif if (tt <= 0 && ms < 0) { tt--; if (precision == TSDB_TIME_PRECISION_NANO) { @@ -254,8 +242,9 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { } } - struct tm *ptm = taosLocalTime(&tt, NULL); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + struct tm ptm = {0}; + taosLocalTime(&tt, &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); From 7187c87d35284db7943bb81ddccc1120fcf75e30 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 18 Jul 2022 16:55:24 +0800 Subject: [PATCH 71/88] refactor(sync): add restart heartbeat timer --- source/libs/sync/inc/syncInt.h | 1 + source/libs/sync/src/syncMain.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0cbd7d36b2..3a30cf801e 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -192,6 +192,7 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms); int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode); int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); +int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode); // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index bb7454ea6f..ddb2b9355e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1298,6 +1298,12 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) { return ret; } +int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) { + syncNodeStopHeartbeatTimer(pSyncNode); + syncNodeStartHeartbeatTimer(pSyncNode); + return 0; +} + // utils -------------- int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) { SEpSet epSet; From b7e5780c44a00b9852d8c472ddf3e1bef7b1b72d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 18 Jul 2022 16:55:31 +0800 Subject: [PATCH 72/88] feat: sql statement likes 'select * from (select 1)' --- include/common/tmsg.h | 7 ++-- include/libs/nodes/plannodes.h | 11 +++++- source/libs/nodes/src/nodesCodeFuncs.c | 40 ++++++++-------------- source/libs/parser/src/parCalcConst.c | 3 +- source/libs/parser/src/parTranslater.c | 5 ++- source/libs/planner/src/planLogicCreater.c | 26 ++++++-------- source/libs/planner/src/planPhysiCreater.c | 29 ++++++++++------ 7 files changed, 63 insertions(+), 58 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 836712d8db..18da6f144c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2071,9 +2071,10 @@ int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp); // TDMT_VND_ALTER_TABLE ===================== typedef struct { - char* tbName; - int8_t action; - char* colName; + char* tbName; + int8_t action; + char* colName; + int32_t colId; // TSDB_ALTER_TABLE_ADD_COLUMN int8_t type; int8_t flags; diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 3bd02b1b48..1c3e5903f6 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -26,6 +26,12 @@ extern "C" { #define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN +typedef enum EDataOrderLevel { + DATA_ORDER_LEVEL_NONE = 1, + DATA_ORDER_LEVEL_IN_BLOCK, + DATA_ORDER_LEVEL_IN_GROUP +} EDataOrderLevel; + typedef struct SLogicNode { ENodeType type; SNodeList* pTargets; // SColumnNode @@ -36,6 +42,8 @@ typedef struct SLogicNode { uint8_t precision; SNode* pLimit; SNode* pSlimit; + EDataOrderLevel requireDataOrder; // requirements for input data + EDataOrderLevel resultDataOrder; // properties of the output data } SLogicNode; typedef enum EScanType { @@ -78,7 +86,7 @@ typedef struct SScanLogicNode { SNodeList* pGroupTags; bool groupSort; int8_t cacheLastMode; - bool hasNormalCols; // neither tag column nor primary key tag column + bool hasNormalCols; // neither tag column nor primary key tag column } SScanLogicNode; typedef struct SJoinLogicNode { @@ -317,6 +325,7 @@ typedef STableScanPhysiNode SStreamScanPhysiNode; typedef struct SProjectPhysiNode { SPhysiNode node; SNodeList* pProjections; + bool mergeDataBlock; } SProjectPhysiNode; typedef struct SIndefRowsFuncPhysiNode { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index f9ebde0657..fda02cd9ca 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -548,37 +548,12 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanLogicPlanTableType = "TableType"; static const char* jkScanLogicPlanTableId = "TableId"; static const char* jkScanLogicPlanStableId = "StableId"; +static const char* jkScanLogicPlanScanType = "ScanType"; static const char* jkScanLogicPlanScanCount = "ScanCount"; static const char* jkScanLogicPlanReverseScanCount = "ReverseScanCount"; static const char* jkScanLogicPlanTagCond = "TagCond"; static const char* jkScanLogicPlanGroupTags = "GroupTags"; -// typedef struct SScanLogicNode { -// uint64_t stableId; -// SVgroupsInfo* pVgroupList; -// EScanType scanType; -// uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count -// STimeWindow scanRange; -// SName tableName; -// bool showRewrite; -// double ratio; -// SNodeList* pDynamicScanFuncs; -// int32_t dataRequired; -// int64_t interval; -// int64_t offset; -// int64_t sliding; -// int8_t intervalUnit; -// int8_t slidingUnit; -// SNode* pTagCond; -// SNode* pTagIndexCond; -// int8_t triggerType; -// int64_t watermark; -// int8_t igExpired; -// SArray* pSmaIndexes; -// SNodeList* pGroupTags; -// bool groupSort; -// } SScanLogicNode; - static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -598,6 +573,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanStableId, pNode->stableId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanType, pNode->scanType); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanScanCount, pNode->scanSeq[0]); } @@ -634,6 +612,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUBigIntValue(pJson, jkScanLogicPlanStableId, &pNode->stableId); } + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkScanLogicPlanScanType, pNode->scanType, code); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetUTinyIntValue(pJson, jkScanLogicPlanScanCount, &pNode->scanSeq[0]); } @@ -1677,6 +1658,7 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) { } static const char* jkProjectPhysiPlanProjections = "Projections"; +static const char* jkProjectPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj; @@ -1685,6 +1667,9 @@ static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkProjectPhysiPlanProjections, pNode->pProjections); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkProjectPhysiPlanMergeDataBlock, pNode->mergeDataBlock); + } return code; } @@ -1696,6 +1681,9 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkProjectPhysiPlanProjections, &pNode->pProjections); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkProjectPhysiPlanMergeDataBlock, &pNode->mergeDataBlock); + } return code; } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 68a60e0b35..a7c08d8f65 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -201,7 +201,8 @@ static int32_t calcConstProject(SNode* pProject, bool dual, SNode** pNew) { } static bool isUselessCol(SExprNode* pProj) { - if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId)) { + if (QUERY_NODE_FUNCTION == nodeType(pProj) && !fmIsScalarFunc(((SFunctionNode*)pProj)->funcId) && + !fmIsPseudoColumnFunc(((SFunctionNode*)pProj)->funcId)) { return false; } return NULL == ((SExprNode*)pProj)->pAssociation; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 19126b28f1..bac073c631 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1686,7 +1686,7 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { static bool sysTableFromVnode(const char* pTable) { return (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) || - (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS))); + (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } @@ -5968,6 +5968,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS if (NULL == pReq->tagName) { return TSDB_CODE_OUT_OF_MEMORY; } + pReq->colId = pSchema->colId; SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema); if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) { @@ -6051,6 +6052,7 @@ static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, if (NULL == pReq->colName) { return TSDB_CODE_OUT_OF_MEMORY; } + pReq->colId = pSchema->colId; return TSDB_CODE_SUCCESS; } @@ -6071,6 +6073,7 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt if (NULL == pReq->colName) { return TSDB_CODE_OUT_OF_MEMORY; } + pReq->colId = pSchema->colId; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 4903beddbe..9ced5c1cb6 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -20,6 +20,7 @@ typedef struct SLogicPlanContext { SPlanContext* pPlanCxt; SLogicNode* pCurrRoot; + bool hasScan; } SLogicPlanContext; typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, void*, SLogicNode**); @@ -161,6 +162,10 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols return SCAN_TYPE_STREAM; } + if (TSDB_SYSTEM_TABLE == tableType) { + return SCAN_TYPE_SYSTEM_TABLE; + } + if (NULL == pScanCols) { return NULL == pScanPseudoCols ? SCAN_TYPE_TABLE @@ -169,17 +174,6 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols : SCAN_TYPE_TABLE); } - if (TSDB_SYSTEM_TABLE == tableType) { - return SCAN_TYPE_SYSTEM_TABLE; - } - - SNode* pCol = NULL; - FOREACH(pCol, pScanCols) { - if (COLUMN_TYPE_COLUMN == ((SColumnNode*)pCol)->colType) { - return SCAN_TYPE_TABLE; - } - } - return SCAN_TYPE_TABLE; } @@ -300,6 +294,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect nodesDestroyNode((SNode*)pScan); } + pCxt->hasScan = true; + return code; } @@ -1339,9 +1335,9 @@ static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); } -static void setLogicSubplanType(SLogicSubplan* pSubplan) { +static void setLogicSubplanType(bool hasScan, SLogicSubplan* pSubplan) { if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY != nodeType(pSubplan->pNode)) { - pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + pSubplan->subplanType = hasScan ? SUBPLAN_TYPE_SCAN : SUBPLAN_TYPE_MERGE; } else { SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)pSubplan->pNode; pSubplan->subplanType = (MODIFY_TABLE_TYPE_INSERT == pModify->modifyType && NULL != pModify->node.pChildren) @@ -1351,7 +1347,7 @@ static void setLogicSubplanType(SLogicSubplan* pSubplan) { } int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { - SLogicPlanContext cxt = {.pPlanCxt = pCxt}; + SLogicPlanContext cxt = {.pPlanCxt = pCxt, .pCurrRoot = NULL, .hasScan = false}; SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); if (NULL == pSubplan) { @@ -1364,7 +1360,7 @@ int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { int32_t code = createQueryLogicNode(&cxt, pCxt->pAstRoot, &pSubplan->pNode); if (TSDB_CODE_SUCCESS == code) { setLogicNodeParent(pSubplan->pNode); - setLogicSubplanType(pSubplan); + setLogicSubplanType(cxt.hasScan, pSubplan); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 8250fb462f..3f619f506f 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -35,7 +35,8 @@ typedef struct SPhysiPlanContext { int32_t errCode; int16_t nextDataBlockId; SArray* pLocationHelper; - SArray* pExecNodeList; // SArray + bool hasScan; + bool hasSysScan; } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { @@ -255,7 +256,7 @@ static int32_t addDataBlockSlot(SPhysiPlanContext* pCxt, SNode** pNode, SDataBlo static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, true, false); + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, false, false); } static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { @@ -495,8 +496,6 @@ static int32_t createSimpleScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSub return TSDB_CODE_OUT_OF_MEMORY; } vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); - SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0}; - taosArrayPush(pCxt->pExecNodeList, &node); return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode); } @@ -577,8 +576,6 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan->execNode.nodeId = MNODE_HANDLE; pSubplan->execNode.epSet = pCxt->pPlanCxt->mgmtEpSet; } - SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0}; - taosArrayPush(pCxt->pExecNodeList, &node); if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_DNODE_VARIABLES)) { pScan->mgmtEpSet = pScanLogicNode->pVgroupList->vgroups->epSet; } else { @@ -586,6 +583,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* } tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); + pCxt->hasSysScan = true; return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); } @@ -601,6 +599,7 @@ static int32_t createTableMergeScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { + pCxt->hasScan = true; switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: case SCAN_TYPE_BLOCK_INFO: @@ -1806,23 +1805,31 @@ static void setExplainInfo(SPlanContext* pCxt, SQueryPlan* pPlan) { } } +static void setExecNodeList(SPhysiPlanContext* pCxt, SArray* pExecNodeList) { + if (NULL == pExecNodeList) { + return; + } + if (pCxt->hasSysScan || !pCxt->hasScan) { + SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0}; + taosArrayPush(pExecNodeList, &node); + } +} + int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) { SPhysiPlanContext cxt = {.pPlanCxt = pCxt, .errCode = TSDB_CODE_SUCCESS, .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES), - .pExecNodeList = pExecNodeList}; + .hasScan = false, + .hasSysScan = false}; if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } - if (QUERY_POLICY_VNODE == tsQueryPolicy) { - taosArrayClear(pExecNodeList); - } - int32_t code = doCreatePhysiPlan(&cxt, pLogicPlan, pPlan); if (TSDB_CODE_SUCCESS == code) { setExplainInfo(pCxt, *pPlan); + setExecNodeList(&cxt, pExecNodeList); } destoryPhysiPlanContext(&cxt); From f57ad234dcc8cd02c403c5bd6d2fd7819efade14 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Jul 2022 16:59:03 +0800 Subject: [PATCH 73/88] fix(query): fix function first/last do not support constant input TD-17400 --- source/libs/function/src/builtins.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 489a11da39..87afac70eb 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1422,14 +1422,6 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); - for (int32_t i = 0; i < numOfParams; ++i) { - SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i); - if (QUERY_NODE_VALUE == nodeType(pParamNode)) { - return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); - } - } - pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; return TSDB_CODE_SUCCESS; } From 25c8f72cea33dff9a8fed8f27a943a2f5bd5b4ad Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 18 Jul 2022 17:20:11 +0800 Subject: [PATCH 74/88] test: add tmq case with tag filter --- tests/system-test/7-tmq/stbTagFilter.py | 261 ++++++++++++++++++------ 1 file changed, 204 insertions(+), 57 deletions(-) diff --git a/tests/system-test/7-tmq/stbTagFilter.py b/tests/system-test/7-tmq/stbTagFilter.py index 65609629bc..82596a32ba 100644 --- a/tests/system-test/7-tmq/stbTagFilter.py +++ b/tests/system-test/7-tmq/stbTagFilter.py @@ -5,103 +5,250 @@ import time import socket import os import threading +from enum import Enum from util.log import * from util.sql import * from util.cases import * from util.dnodes import * -from util.common import * sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: + def __init__(self): + self.snapshot = 0 + self.vgroups = 4 + self.ctbNum = 1 + self.rowsPerTbl = 10000 + def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) - #tdSql.init(conn.cursor(), logSql) # output sql.txt file + tdSql.init(conn.cursor(), False) - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'db2', + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', 'dropFlag': 1, 'event': '', - 'vgroups': 1, + 'vgroups': 4, 'stbName': 'stb', 'colPrefix': 'c', 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 1000, - 'batchNum': 10, + 'ctbNum': 1, + 'rowsPerTbl': 100000, + 'batchNum': 1200, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 10, + 'pollDelay': 3, 'showMsg': 1, - 'showRow': 1} + 'showRow': 1, + 'snapshot': 0} - topicNameList = ['topic1'] - expectRowsList = [] + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=1,replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) tdLog.info("create stb") - tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("insert data") - tmqCom.asyncInsertData(paraDict) - - tdLog.info("create topics from stb with filter") - # queryString = "select ts, sin(c1), pow(c2,3) from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) - queryString = "select * from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 100000, + 'batchNum': 3000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + # update to half tables + paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) + # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + # queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) + queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - - # start tmq consume processor - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 - topicList = topicNameList[0] - ifcheckdata = 0 + tdSql.execute(sqlString) + + # paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + consumerId = 0 + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + topicList = topicFromStb1 + ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:2000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - # tmqCom.getStartCommitNotifyFromTmqsim() - tmqCom.getStartConsumeNotifyFromTmqsim() - tdLog.info("create some new ctb") - paraDict['ctbStartIdx'] = paraDict['ctbStartIdx'] + paraDict['ctbNum'] - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data into new ctb") - pThread = tmqCom.asyncInsertData(paraDict) - - pThread.join() - tdLog.info("wait insert end") - tdSql.query(queryString) - expectRowsList.append(tdSql.getRows()) - - tdLog.info("wait the consume result") + tdLog.info("insert process end, and start to check consume result") expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - - if expectRowsList[0] != resultList[0]: - tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) - tdLog.exit("0 tmq consume rows error!") - - time.sleep(10) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + tdLog.info("run select sql from db") + tdSql.query(queryString) + expectrowcnt = tdSql.getRows() + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + if totalConsumeRows != expectrowcnt: + tdLog.exit("tmq consume rows error!") + + tmqCom.checkFileContent(consumerId, queryString) + + tdSql.query("drop topic %s"%topicFromStb1) tdLog.printNoPrefix("======== test case 1 end ...... ") + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 5000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("restart taosd to ensure that the data falls into the disk") + tdSql.query("flush database %s"%(paraDict['dbName'])) + + # update to half tables + paraDict['startTs'] = paraDict['startTs'] + int(self.rowsPerTbl / 2) + paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) + tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tmqCom.initConsumerTable() + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + consumerId = 1 + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + topicList = topicFromStb1 + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() + + tdLog.info("act consume rows: %d, act insert rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsInserted, expectrowcnt)) + + if totalConsumeRows != expectrowcnt: + tdLog.exit("tmq consume rows error!") + + # tmqCom.checkFileContent(consumerId, queryString) + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + def run(self): tdSql.prepare() + self.prepareTestEnv() + tdLog.printNoPrefix("=============================================") + tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") self.tmqCase1() + # self.tmqCase2() + + self.prepareTestEnv() + tdLog.printNoPrefix("====================================================================") + tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + self.snapshot = 1 + self.tmqCase1() + # self.tmqCase2() + def stop(self): tdSql.close() From 1ab22f9d9eb4eb3d35be6c7aaf2f5015bc96dd5c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Jul 2022 17:29:40 +0800 Subject: [PATCH 75/88] fix(query): fix taosd crashed when using first(col+null) TD-17381 --- source/common/src/tdatablock.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d9ac84ec06..eefac77e52 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -228,7 +228,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui uint32_t finalNumOfRows = numOfRow1 + numOfRow2; if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap - if (finalNumOfRows > *capacity || numOfRow1 == 0) { + if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) { char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -262,7 +262,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); pColumnInfoData->varmeta.length = len + oldLen; } else { - if (finalNumOfRows > *capacity || numOfRow1 == 0) { + if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) { ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { From 1d3648fc9bc6eeb389ecccca7067cd1da2593f6c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 18 Jul 2022 17:31:11 +0800 Subject: [PATCH 76/88] fix(tdb): make txn write when insert subdb into main db --- source/libs/tdb/src/db/tdbBtree.c | 5 +---- source/libs/tdb/src/db/tdbPager.c | 22 ---------------------- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index cca451ceb7..6e8fd32641 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -103,10 +103,9 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg // if pgno == 0 fetch new btree root leaf page if (pgno == 0) { // fetch page & insert into main db - // allocate a new child page SPage *pPage; TXN txn; - tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); + tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); pPager->inTran = 1; @@ -118,8 +117,6 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg return -1; } - // TODO: Need to zero the page - ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { return -1; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index e7765ed667..d9a44ba570 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -473,12 +473,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { return -1; } - TXN txn; - tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - SBtreeInitPageArg iArg; - iArg.pBt = pBt; - iArg.flags = 0; - for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) { // read pgno & the page from journal SPgno pgno; @@ -494,20 +488,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { return -1; } - /* - ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &iArg, &txn); - if (ret < 0) { - return -1; - } - - // write the page to db - ret = tdbPagerWritePageToDB(pPager, pPage); - if (ret < 0) { - return -1; - } - - tdbPCacheRelease(pPager->pCache, pPage, &txn); - */ i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { ASSERT(0); @@ -523,8 +503,6 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { tdbOsFSync(pPager->fd); - tdbTxnClose(&txn); - tdbOsFree(pageBuf); tdbOsClose(jfd); From a10a689176899335557ea2088c8440a9b77dd83e Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Mon, 18 Jul 2022 17:50:22 +0800 Subject: [PATCH 77/88] test: enh tools of crash_gen about generate start time of records --- tests/pytest/crash_gen/crash_gen_main.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 08155f656b..8f0bfdd481 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -30,6 +30,7 @@ import argparse import sys import os import io +import datetime import signal import traceback import requests @@ -1107,14 +1108,20 @@ class Database: # TODO: fix the error as result of above: "tsdb timestamp is out of range" @classmethod def setupLastTick(cls): - t1 = datetime.datetime(2020, 6, 1) + # start time will be auto generated , start at 10 years ago local time + local_time = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-16] + local_epoch_time = [int(i) for i in local_time.split("-")] + #local_epoch_time will be such as : [2022, 7, 18] + + t1 = datetime.datetime(local_epoch_time[0]-5, local_epoch_time[1], local_epoch_time[2]) t2 = datetime.datetime.now() # maybe a very large number, takes 69 years to exceed Python int range elSec = int(t2.timestamp() - t1.timestamp()) elSec2 = (elSec % (8 * 12 * 30 * 24 * 60 * 60 / 500)) * \ 500 # a number representing seconds within 10 years # print("elSec = {}".format(elSec)) - t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years + + t3 = datetime.datetime(local_epoch_time[0]-10, local_epoch_time[1], local_epoch_time[2]) # default "keep" is 10 years t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec2) # see explanation above Logging.debug("Setting up TICKS to start from: {}".format(t4)) From d328b271a5d93e49f1d33d2e4e636b05436cd84f Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 18 Jul 2022 17:52:00 +0800 Subject: [PATCH 78/88] os: tsim read line error --- tests/tsim/src/simParse.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 5b6dda4dae..b9f7610be8 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -175,7 +175,7 @@ SScript *simBuildScriptObj(char *fileName) { SScript *simParseScript(char *fileName) { TdFilePtr pFile; int32_t tokenLen, lineNum = 0; - char *buffer = NULL, name[128], *token, *rest; + char buffer[10*1024], name[128], *token, *rest; SCommand *pCmd; SScript *script; @@ -195,7 +195,7 @@ SScript *simParseScript(char *fileName) { simResetParser(); while (!taosEOFFile(pFile)) { - if (taosGetLineFile(pFile, &buffer) == -1) continue; + if (taosGetsFile(pFile, sizeof(buffer) - 1, buffer) == -1) continue; lineNum++; int32_t cmdlen = (int32_t)strlen(buffer); @@ -240,7 +240,6 @@ SScript *simParseScript(char *fileName) { return NULL; } } - if(buffer != NULL) taosMemoryFree(buffer); taosCloseFile(&pFile); script = simBuildScriptObj(fileName); From 993f5fe9ecad278fe35b09f38e331557eb1a8275 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Jul 2022 18:37:58 +0800 Subject: [PATCH 79/88] fix: let certain type of write messages to be executed sequentially --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 10 +++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 3 +- source/dnode/vnode/src/vnd/vnodeSync.c | 49 ++++++++------------- tests/script/tsim/db/alter_replica_31.sim | 9 ++++ 4 files changed, 39 insertions(+), 32 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 93df7f8ab2..5ad13e383a 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -179,6 +179,16 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp } else { dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pWriteQ, pMsg); +#if 0 // tests for batch writes + if (pMsg->msgType == TDMT_VND_CREATE_TABLE) { + SRpcMsg *pDup = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); + memcpy(pDup, pMsg, sizeof(SRpcMsg)); + pDup->pCont = rpcMallocCont(pMsg->contLen); + memcpy(pDup->pCont, pMsg->pCont, pMsg->contLen); + pDup->info.handle = NULL; + taosWriteQitem(pVnode->pWriteQ, pDup); + } +#endif } break; case SYNC_QUEUE: diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e5322f1bd3..5f730bcfa5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -53,6 +53,7 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { *(int64_t *)(dc.data + dc.pos) = uid; *(int64_t *)(dc.data + dc.pos + 8) = ctime; + vTrace("vgId:%d, table:%s uid:%" PRId64 " is generated", pVnode->config.vgId, name, uid); tEndDecode(&dc); } @@ -381,7 +382,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p goto end; } - vInfo("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp); + vDebug("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp); int32_t ret = metaTtlDropTable(pVnode->pMeta, ttlReq.timestamp, tbUids); if (ret != 0) { goto end; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 712cee9fd0..dbe4458681 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -17,35 +17,22 @@ #include "vnd.h" static inline bool vnodeIsMsgBlock(tmsg_t type) { - return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_ALTER_REPLICA); + return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_CREATE_TABLE) || + (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || (type == TDMT_VND_UPDATE_TAG_VAL); } static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; } -static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) { - if (!vnodeIsMsgBlock(type)) return; - - int32_t count = atomic_add_fetch_32(&pVnode->blockCount, 1); - vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type)); +static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { + if (vnodeIsMsgBlock(pMsg->msgType)) { + vTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); + tsem_wait(&pVnode->syncSem); + } } -static inline void vnodeWaitBlockMsg(SVnode *pVnode) { - int32_t count = atomic_load_32(&pVnode->blockCount); - if (count <= 0) return; - - vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count); - tsem_wait(&pVnode->syncSem); -} - -static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) { - if (!vnodeIsMsgBlock(type)) return; - - int32_t count = atomic_load_32(&pVnode->blockCount); - if (count <= 0) return; - - count = atomic_sub_fetch_32(&pVnode->blockCount, 1); - vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type)); - if (count <= 0) { +static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { + if (vnodeIsMsgBlock(pMsg->msgType)) { + vTrace("vgId:%d, msg:%p post block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); tsem_post(&pVnode->syncSem); } } @@ -143,6 +130,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) int32_t code = 0; SRpcMsg *pMsg = NULL; + vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs); + for (int32_t m = 0; m < numOfMsgs; m++) { if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; const STraceId *trace = &pMsg->info.traceId; @@ -165,13 +154,14 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) if (rsp.info.handle != NULL) { tmsgSendRsp(&rsp); } + } else if (code == 0) { + vnodeWaitBlockMsg(pVnode, pMsg); + } else { } } } - if (code == 0) { - vnodeAccumBlockMsg(pVnode, pMsg->msgType); - } else if (code < 0) { + if (code < 0) { if (terrno == TSDB_CODE_SYN_NOT_LEADER) { vnodeRedirectRpcMsg(pVnode, pMsg); } else { @@ -182,15 +172,12 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) tmsgSendRsp(&rsp); } } - } else { } vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } - - vnodeWaitBlockMsg(pVnode); } void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { @@ -213,7 +200,7 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { } } - vnodePostBlockMsg(pVnode, pMsg->msgType); + vnodePostBlockMsg(pVnode, pMsg); if (rsp.info.handle != NULL) { tmsgSendRsp(&rsp); } @@ -418,7 +405,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon tmsgSendRsp(&rpcMsg); } - vnodePostBlockMsg(pVnode, TDMT_VND_ALTER_REPLICA); + vnodePostBlockMsg(pVnode, pMsg); } static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim index e9a295820c..1823f182c9 100644 --- a/tests/script/tsim/db/alter_replica_31.sim +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -111,6 +111,15 @@ if $hasleader != 1 then goto step2 endi +# sql use db; +# sql create table stb (ts timestamp, c int) tags (t int); +# sql create table t0 using stb tags (0); +# sql insert into t0 values(now, 1); +# sql show db.stables; +# sql show db.tables; +# sql show db.vgroups; +return + sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd" sql create table db.ctb using db.stb tags(101, "102") sql insert into db.ctb values(now, 1, "2") From d8d32ebc51ae5ff7cef52379620bd4a528223ab1 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 18 Jul 2022 19:15:23 +0800 Subject: [PATCH 80/88] fix(tmq): set version after consuming from tsdb --- source/dnode/vnode/src/tq/tqExec.c | 4 ++-- tests/system-test/99-TDcase/TD-17255.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 618b32239a..f18b25bef4 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -68,7 +68,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa pRsp->rspOffset = *pOffset; return 0; } else { - tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1); + tqOffsetResetToLog(pOffset, pHandle->snapshotVer); if (qStreamPrepareScan(task, pOffset) < 0) { pRsp->rspOffset = *pOffset; return 0; @@ -106,7 +106,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa } if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1); + tqOffsetResetToLog(pOffset, pHandle->snapshotVer); qStreamPrepareScan(task, pOffset); continue; } diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index 9eb8d531f7..28a1a1fd4a 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -320,7 +320,7 @@ class TDTestCase: tdSql.prepare() self.tmqCase1() - # self.tmqCase2() + self.tmqCase2() self.tmqCase3() def stop(self): From 156e72b07118436b7e49e59d501ee87d92135979 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 18 Jul 2022 19:34:53 +0800 Subject: [PATCH 81/88] enh: rsma drop and code optimization --- source/dnode/vnode/src/inc/sma.h | 27 +++-- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/dnode/vnode/src/sma/smaCommit.c | 4 +- source/dnode/vnode/src/sma/smaEnv.c | 20 ++++ source/dnode/vnode/src/sma/smaRollup.c | 126 ++++++++++++++++++------ 5 files changed, 136 insertions(+), 43 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 1c8434594d..4673b01d27 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -43,9 +43,9 @@ typedef struct SRSmaInfo SRSmaInfo; typedef struct SRSmaInfoItem SRSmaInfoItem; struct SSmaEnv { - SRWLatch lock; - int8_t type; - SSmaStat *pStat; + SRWLatch lock; + int8_t type; + SSmaStat *pStat; }; typedef struct { @@ -103,10 +103,15 @@ struct SRSmaInfoItem { }; struct SRSmaInfo { - STSchema *pTSchema; - int64_t suid; + STSchema *pTSchema; + int64_t suid; + int8_t delFlag; + T_REF_DECLARE() SRSmaInfoItem items[TSDB_RETENTION_L2]; }; +#define RSMA_INFO_HEAD_LEN 24 +#define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1) +#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1) enum { TASK_TRIGGER_STAT_INIT = 0, @@ -120,8 +125,8 @@ enum { enum { RSMA_ROLE_CREATE = 0, RSMA_ROLE_DROP = 1, - RSMA_ROLE_FETCH = 2, - RSMA_ROLE_SUBMIT = 3, + RSMA_ROLE_SUBMIT = 2, + RSMA_ROLE_FETCH = 3, RSMA_ROLE_ITERATE = 4, }; @@ -134,6 +139,8 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg); int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat); int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); +int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo); +int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo); void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); @@ -193,6 +200,7 @@ void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t le static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); +void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid); int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); @@ -258,8 +266,9 @@ void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); void tdCloseTFile(STFile *pTFile); void tdDestroyTFile(STFile *pTFile); -void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName); -void tdGetVndDirName(int32_t vgId,const char *pdname, const char *dname, bool endWithSep, char *outputName); +void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, + char *outputName); +void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 29c10c40c2..71345df747 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -277,7 +277,7 @@ _drop_super_table: _exit: tdbFree(pKey); tdbFree(pData); - metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; } diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index cdaaf2bbdb..2174a479e7 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -191,7 +191,7 @@ static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) { if ((pDir = taosOpenDir(dir)) == NULL) { regfree(®ex); terrno = TAOS_SYSTEM_ERROR(errno); - smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); + smaDebug("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); return TSDB_CODE_FAILED; } @@ -392,5 +392,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { // step 2: cleanup outdated qtaskinfo files tdCleanupQTaskInfoFiles(pSma, pRSmaStat); + atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 577d5fd4fa..23706d54e0 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -169,6 +169,26 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { return 0; } +int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { + if (!pRSmaInfo) return 0; + + int ref = T_REF_INC(pRSmaInfo); + smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); + return 0; +} + +int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { + if (!pRSmaInfo) return 0; + + int ref = T_REF_DEC(pRSmaInfo); + smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); + + if (ref == 0) { + tdRemoveRSmaInfoBySuid(pSma, pRSmaInfo->suid); + } + return 0; +} + static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) { ASSERT(pSmaStat != NULL); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 32d6dee57f..1f7664c22a 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -24,6 +24,7 @@ SSmaMgmt smaMgmt = { }; #define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver" +#define TD_RSMAINFO_DEL_FILE "rsmainfo.del" typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter; @@ -48,14 +49,11 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables); static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed); static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed); - - static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) { // adapt accordingly if definition of SRSmaInfo update SRSmaInfo *pResult = NULL; - int32_t rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *); ASSERT(pItem->level == TSDB_RETENTION_L1 || pItem->level == TSDB_RETENTION_L2); - pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + rsmaInfoHeadLen)); + pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + RSMA_INFO_HEAD_LEN)); ASSERT(pResult->pTSchema->numOfCols > 1); return pResult; } @@ -116,8 +114,8 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (pItem->taskInfo) { if (isDeepFree && pItem->tmrId) { - smaDebug("vgId:%d, table %" PRIi64 " stop fetch timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId, - i + 1); + smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid, + pItem->tmrId, i + 1); taosTmrStopA(&pItem->tmrId); } tdFreeQTaskInfo(&pItem->taskInfo, SMA_VID(pSma), i + 1); @@ -337,6 +335,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con } pRSmaInfo->pTSchema = pTSchema; pRSmaInfo->suid = suid; + T_REF_INIT_VAL(pRSmaInfo, 1); if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) { goto _err; @@ -392,11 +391,33 @@ int32_t tdProcessRSmaCreate(SSma *pSma, SVCreateStbReq *pReq) { int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { SVnode *pVnode = pSma->pVnode; if (!VND_IS_RSMA(pVnode)) { - smaTrace("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name, + smaTrace("vgId:%d, not drop rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name, pReq->suid); return TSDB_CODE_SUCCESS; } + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + if (!pSmaEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + + SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pReq->suid); + + if (!pRSmaInfo) { + smaWarn("vgId:%d, drop rsma for stable %s %" PRIi64 " failed no rsma in hash", TD_VID(pVnode), pReq->name, + pReq->suid); + return TSDB_CODE_SUCCESS; + } + + // set del flag for data in mem + RSMA_INFO_SET_DEL(pRSmaInfo); + tdUnRefRSmaInfo(pSma, pRSmaInfo); + + // save to file + smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid); return TSDB_CODE_SUCCESS; } @@ -650,10 +671,10 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType /** * @brief During async commit, the SRSmaInfo object would be COW from iRSmaInfoHash and write lock should be applied. - * - * @param pSma - * @param suid - * @return SRSmaInfo* + * + * @param pSma + * @param suid + * @return SRSmaInfo* */ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); @@ -661,7 +682,6 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { SRSmaInfo *pRSmaInfo = NULL; if (!pEnv) { - // only applicable when rsma env exists return NULL; } @@ -683,18 +703,21 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { SRSmaInfo *pCowRSmaInfo = NULL; // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); - void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); - if (iRSmaInfo) { - SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; - if (pIRSmaInfo) { - if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) { - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); - return NULL; - } - if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) { - taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); - return NULL; + if (!taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t))) { // 2-phase lock + void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + if (iRSmaInfo) { + SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; + if (pIRSmaInfo) { + if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) { + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); + return NULL; + } + smaDebug("vgId:%d, clone rsma info succeed for suid:%" PRIu64, SMA_VID(pSma), suid); + if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) { + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + return NULL; + } } } } @@ -703,21 +726,56 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { return pCowRSmaInfo; } +/** + * @brief During the drop procedure, only need to delete the object in rsmaInfoHash. + * + * @param pSma + * @param suid + * @return SRSmaInfo* + */ +void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = NULL; + SRSmaInfo *pRSmaInfo = NULL; + + if (!pEnv) { + return; + } + + pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + if (!pStat || !RSMA_INFO_HASH(pStat)) { + return; + } + + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + if (pRSmaInfo) { + if (pRSmaInfo = *(SRSmaInfo **)pRSmaInfo) { + tdFreeRSmaInfo(pSma, pRSmaInfo, true); + } + taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + smaDebug("vgId:%d, remove from infoHash for table:%" PRIu64 " succeed", SMA_VID(pSma), suid); + } +} + static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid); if (!pRSmaInfo) { - smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); + smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } if (!pRSmaInfo->items[0].taskInfo) { - smaDebug("vgId:%d, return as no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); + smaDebug("vgId:%d, execute rsma, no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } if (inputType == STREAM_INPUT__DATA_SUBMIT) { + tdRefRSmaInfo(pSma, pRSmaInfo); + tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1); tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2); + + tdUnRefRSmaInfo(pSma, pRSmaInfo); } return TSDB_CODE_SUCCESS; @@ -931,10 +989,10 @@ _err: /** * @brief Restore from SRSmaQTaskInfoItem - * - * @param pSma - * @param pItem - * @return int32_t + * + * @param pSma + * @param pItem + * @return int32_t */ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) { SRSmaInfo *pRSmaInfo = NULL; @@ -1271,6 +1329,9 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { } SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem); + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + goto _end; + } int8_t fetchTriggerStat = atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); @@ -1279,13 +1340,14 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma), pItem->level, pRSmaInfo->suid); - tdRefSmaStat(pSma, (SSmaStat *)pStat); + // sync procedure => async process + tdRefRSmaInfo(pSma, pRSmaInfo); SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false); tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK); - tdUnRefSmaStat(pSma, (SSmaStat *)pStat); + tdUnRefRSmaInfo(pSma, pRSmaInfo); } break; case TASK_TRIGGER_STAT_PAUSED: { smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused", From 322ec02e39e711cd243f0d2fc27a05fa40d24373 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 18 Jul 2022 11:45:13 +0000 Subject: [PATCH 82/88] fix: delete causes coredump --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 3f667a7465..4e2c762cd3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -186,6 +186,7 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData); typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT; void tsdbDataFileName(STsdb *pTsdb, SDFileSet *pDFileSet, EDataFileT ftype, char fname[]); bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT ftype); +bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype); int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype); int32_t tPutDataFileHdr(uint8_t *p, SDFileSet *pSet, EDataFileT ftype); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 70fbd90646..3bc79621e1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -245,7 +245,7 @@ static int32_t tsdbApplyDelFileChange(STsdbFS *pFS, SDelFile *pFrom, SDelFile *p char fname[TSDB_FILENAME_LEN]; if (pFrom && pTo) { - if (pFrom != pTo) { + if (!tsdbDelFileIsSame(pFrom, pTo)) { tsdbDelFileName(pFS->pTsdb, pFrom, fname); if (taosRemoveFile(fname) < 0) { code = TAOS_SYSTEM_ERROR(errno); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index e7f8cb4789..f15ad072e7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -140,6 +140,8 @@ bool tsdbFileIsSame(SDFileSet *pDFileSet1, SDFileSet *pDFileSet2, EDataFileT fty } } +bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; } + int32_t tsdbUpdateDFileHdr(TdFilePtr pFD, SDFileSet *pSet, EDataFileT ftype) { int32_t code = 0; int64_t n; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 0ed2e12542..5e8157864f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -246,7 +246,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb tsdbDelFileName(pTsdb, pFile, fname); pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ); - if (pDelFReader == NULL) { + if (pDelFReader->pReadH == NULL) { code = TAOS_SYSTEM_ERROR(errno); taosMemoryFree(pDelFReader); goto _err; From d4032483c545242d8445c0f16f31fbc1f2d16e96 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Jul 2022 20:04:52 +0800 Subject: [PATCH 83/88] fix last_row.py --- tests/system-test/2-query/last_row.py | 94 +++++++++++++-------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index b856590787..c79e3b58fb 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -48,7 +48,7 @@ class TDTestCase: c9 = "'nchar_val'" c10 = ts tdSql.execute(f" insert into {tbname} values ({ts},{c1},{c2},{c3},{c4},{c5},{c6},{c7},{c8},{c9},{c10})") - + tdSql.execute("use test") tbnames = ["stb", "sub_tb_1"] support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"] @@ -61,7 +61,7 @@ class TDTestCase: origin_sql = "select {} from {} order by tbname".format(colname, 'test.'+tbname) if coltype[1] in support_types: self.check_result_auto(origin_sql , abs_sql) - + def prepare_datas(self ,cache_value): tdSql.execute("drop database if exists db ") create_db_sql = f"create database if not exists db keep 3650 duration 1000 cachemodel {cache_value}" @@ -220,22 +220,22 @@ class TDTestCase: def test_errors(self): tdSql.execute("use testdb") - # bug need fix - tdSql.error("select last_row(c1 ,NULL) from testdb.t1") + # bug need fix + tdSql.query("select last_row(c1 ,NULL) from testdb.t1") error_sql_lists = [ "select last_row from testdb.t1", "select last_row(-+--+c1) from testdb.t1", "select last_row(123--123)==1 from testdb.t1", "select last_row(c1) as 'd1' from testdb.t1", - "select last_row(c1 ,NULL) from testdb.t1", + #"select last_row(c1 ,NULL) from testdb.t1", "select last_row(,) from testdb.t1;", "select last_row(abs(c1) ab from testdb.t1)", "select last_row(c1) as int from testdb.t1", "select last_row from testdb.stb1", "select last_row(123--123)==1 from testdb.stb1", "select last_row(c1) as 'd1' from testdb.stb1", - "select last_row(c1 ,NULL) from testdb.stb1", + #"select last_row(c1 ,NULL) from testdb.stb1", "select last_row(,) from testdb.stb1;", "select last_row(abs(c1) ab from testdb.stb1)", "select last_row(c1) as int from testdb.stb1" @@ -246,7 +246,7 @@ class TDTestCase: def support_types(self): tdSql.execute("use testdb") tbnames = ["stb1", "t1", "ct1", "ct2"] - + for tbname in tbnames: tdSql.query("desc {}".format(tbname)) coltypes = tdSql.queryResult @@ -256,7 +256,7 @@ class TDTestCase: if col_note != "TAG": abs_sql = "select last_row({}) from {}".format(colname, "testdb."+tbname) tdSql.query(abs_sql) - + def basic_abs_function(self): @@ -283,7 +283,7 @@ class TDTestCase: # used for regular table - # bug need fix + # bug need fix tdSql.query("select last_row(c1) from testdb.t1") tdSql.checkData(0, 0, None) tdSql.query("select last_row(c1) from testdb.ct4") @@ -291,21 +291,21 @@ class TDTestCase: tdSql.query("select last_row(c1) from testdb.stb1") tdSql.checkData(0, 0, None) - # # bug need fix + # # bug need fix tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.t1") tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) tdSql.checkData(0, 2, None) - # # bug need fix + # # bug need fix tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.ct1") tdSql.checkData(0, 0, 9) tdSql.checkData(0, 1, -99999) tdSql.checkData(0, 2, -999) tdSql.checkData(0, 3, None) tdSql.checkData(0, 4,-9.99000) - - # bug need fix + + # bug need fix tdSql.query("select last_row(c1), c2, c3 , c4, c5 from testdb.stb1 where tbname='ct1'") tdSql.checkData(0, 0, 9) tdSql.checkData(0, 1, -99999) @@ -313,14 +313,14 @@ class TDTestCase: tdSql.checkData(0, 3, None) tdSql.checkData(0, 4,-9.99000) - # bug fix + # bug fix tdSql.query("select last_row(abs(c1)) from testdb.ct1") tdSql.checkData(0,0,9) - # # bug fix + # # bug fix tdSql.query("select last_row(c1+1) from testdb.ct1") - tdSql.query("select last_row(c1+1) from testdb.stb1") - tdSql.query("select last_row(c1+1) from testdb.t1") + tdSql.query("select last_row(c1+1) from testdb.stb1") + tdSql.query("select last_row(c1+1) from testdb.t1") # used for stable table tdSql.query("select last_row(c1 ,c2 ,c3) ,last_row(c4) from testdb.ct1") @@ -329,7 +329,7 @@ class TDTestCase: tdSql.checkData(0,2,-999) tdSql.checkData(0,3,None) - # bug need fix + # bug need fix tdSql.query("select last_row(c1 ,c2 ,c3) from testdb.stb1 ") tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) @@ -338,7 +338,7 @@ class TDTestCase: tdSql.query('select last_row(c1) from testdb.t1 where ts <"2022-12-31 01:01:36.000"') tdSql.checkData(0,0,8) - # bug need fix + # bug need fix tdSql.query("select abs(last_row(c1)-2)+max(c1),ceil(last_row(c4)-2) from testdb.stb1 where c4 is not null") tdSql.checkData(0,0,16.000000000) tdSql.checkData(0,1,-101.000000000) @@ -371,7 +371,7 @@ class TDTestCase: tdSql.query("select last_row(c1) ,count(*) from testdb.stb1 where c1 is null") tdSql.checkData(0,0,None) tdSql.checkData(0,1,3) - + tdSql.query("select last_row(c1) ,count(c1) from testdb.stb1 where c1 is null") tdSql.checkData(0,0,None) tdSql.checkData(0,1,0) @@ -380,7 +380,7 @@ class TDTestCase: tdSql.query("select tbname ,last_row(c1) from testdb.stb1") tdSql.checkData(0,0,'ct4') tdSql.checkData(0,1,None) - + tdSql.query(" select tbname ,last_row(c1) from testdb.stb1 partition by tbname order by tbname ") tdSql.checkData(0,0,'ct1') tdSql.checkData(0,1,9) @@ -396,11 +396,11 @@ class TDTestCase: tdSql.query(" select t1 ,count(c1) from testdb.stb1 partition by t1 ") tdSql.checkRows(2) - # filter by tbname + # filter by tbname tdSql.query("select last_row(c1) from testdb.stb1 where tbname = 'ct1' ") tdSql.checkData(0,0,9) - # bug need fix + # bug need fix tdSql.query("select tbname ,last_row(c1) from testdb.stb1 where tbname = 'ct1' ") tdSql.checkData(0,1,9) tdSql.query("select tbname ,last_row(c1) from testdb.stb1 partition by tbname order by tbname") @@ -428,7 +428,7 @@ class TDTestCase: tdSql.checkData(0,2,333) tdSql.checkData(0,3,3) - # filter by tag + # filter by tag tdSql.query("select tbname ,last_row(c1) from testdb.stb1 where t1 =0 ") tdSql.checkData(0,1,9) tdSql.query("select tbname ,last_row(c1) ,t1 from testdb.stb1 partition by t1 order by t1") @@ -437,7 +437,7 @@ class TDTestCase: tdSql.checkData(1, 0, 'ct4') tdSql.checkData(1, 1, None) - # filter by col + # filter by col tdSql.query("select tbname ,last_row(c1),abs(c1)from testdb.stb1 where c1 =1;") tdSql.checkData(0, 0, 'ct1') @@ -445,7 +445,7 @@ class TDTestCase: tdSql.checkData(0, 2, 1) tdSql.query("select last_row(c1) from testdb.stb1 where abs(ceil(c1))*c1==1") tdSql.checkData(0,0,1) - + # mix with common functions tdSql.query("select last_row(*) ,last(*) from testdb.stb1 ") tdSql.checkRows(1) @@ -457,11 +457,11 @@ class TDTestCase: tdSql.query("select last_row(c1+abs(c1)) from testdb.stb1 partition by tbname order by tbname") tdSql.query("select last(c1), max(c1+abs(c1)),last_row(c1+abs(c1)) from testdb.stb1 partition by tbname order by tbname") - # # bug need fix ,taosd crash + # # bug need fix ,taosd crash tdSql.error("select last_row(*) ,last(*) from testdb.stb1 partition by tbname order by last(*)") tdSql.error("select last_row(*) ,last(*) from testdb.stb1 partition by tbname order by last_row(*)") - # mix with agg functions + # mix with agg functions tdSql.query("select last(*), last_row(*),last(c1), last_row(c1) from testdb.stb1 ") tdSql.query("select last(*), last_row(*),last(c1), last_row(c1) from testdb.ct1 ") tdSql.query("select last(*), last_row(*),last(c1+1)*max(c1), last_row(c1+2)/2 from testdb.t1 ") @@ -564,7 +564,7 @@ class TDTestCase: # bug need fix - tdSql.query(" select sum(c1) from testdb.stb1 where t1+10 >1; ") + tdSql.query(" select sum(c1) from testdb.stb1 where t1+10 >1; ") tdSql.query("select c1 ,t1 from testdb.stb1 where t1 =0 ") tdSql.checkRows(13) tdSql.query("select last_row(c1,t1) from testdb.stb1 ") @@ -627,8 +627,8 @@ class TDTestCase: tdSql.execute(" use testdb ") tdSql.query(" select last_row(c1) from testdb.stb1 group by t1 order by t1 ") tdSql.checkRows(2) - - # bug need fix + + # bug need fix tdSql.query("select last_row(c1) from testdb.stb1 group by c1 order by c1,t1 ") tdSql.checkRows(10) tdSql.checkData(9,0,8) @@ -643,7 +643,7 @@ class TDTestCase: tdSql.checkRows(11) tdSql.checkData(10,0,9) - # bug need fix , result is error + # bug need fix , result is error tdSql.query("select last_row(c1) from testdb.ct4 group by c1 order by t1 ") tdSql.query("select last_row(t1) from testdb.ct4 group by c1 order by t1 ") @@ -661,24 +661,24 @@ class TDTestCase: tdSql.query("select last_row(c1+c3) from testdb.stb1 group by abs(c1+c3) order by abs(c1+c3)") tdSql.checkRows(11) - # bug need fix , taosd crash + # bug need fix , taosd crash tdSql.query("select last_row(c1+c3)+c2 from testdb.stb1 group by abs(c1+c3)+c2 order by abs(c1+c3)+c2") tdSql.checkRows(11) tdSql.query("select last_row(c1+c3)+last_row(c2) from testdb.stb1 group by abs(c1+c3)+abs(c2) order by abs(c1+c3)+abs(c2)") tdSql.checkRows(11) tdSql.checkData(0,0,None) tdSql.checkData(2,0,11223.000000000) - + tdSql.query("select last_row(t1) from testdb.stb1 where abs(c1+t1)=1 partition by tbname") tdSql.checkData(0,0,1) - + tdSql.query("select tbname , last_row(c1) from testdb.stb1 partition by tbname order by tbname") tdSql.checkRows(2) tdSql.checkData(0, 0, 'ct1') tdSql.checkData(0, 1, 9) tdSql.checkData(0, 2, 'ct4') tdSql.checkData(0, 3, None) - + tdSql.query("select tbname , last_row(c1) from testdb.stb1 partition by t1 order by t1") tdSql.checkRows(2) tdSql.checkData(0, 0, 'ct1') @@ -686,7 +686,7 @@ class TDTestCase: tdSql.checkData(0, 2, 'ct4') tdSql.checkData(0, 3, None) - # bug need fix + # bug need fix tdSql.query("select tbname , last_row(c1) from testdb.stb1 partition by c2 order by c1") tdSql.checkRows(11) tdSql.checkData(10,1,9) @@ -700,7 +700,7 @@ class TDTestCase: tdSql.query("select abs(c1) ,c2 ,t1, last_row(t1) from testdb.stb1 partition by c2 order by t1") tdSql.checkRows(11) - + tdSql.query("select t1 ,last_row(t1) ,c2 from testdb.stb1 partition by c2 order by t1") tdSql.checkRows(11) @@ -722,7 +722,7 @@ class TDTestCase: tdSql.query("select last_row(ceil(c1-2)) , abs(floor(t1+1)) ,floor(c2-c1) from testdb.stb1 partition by abs(floor(c1)) order by abs(c1)") tdSql.checkRows(11) - # interval + # interval tdSql.query("select last_row(c1) from testdb.stb1 interval(50s) sliding(30s)") tdSql.checkRows(27) @@ -745,13 +745,13 @@ class TDTestCase: tdSql.query('select last_row(c1) from testdb.stb1 where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s)') tdSql.query('select last_row(c1) from (select ts , c1 from testdb.t1 where ts>="2021-01-01 01:01:06.000" and ts < "2021-07-21 01:01:01.000" ) interval(10s) sliding(5s)') - # join + # join tdSql.query("use test") tdSql.query("select last(sub_tb_1.c1), last(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") tdSql.checkCols(2) last_row_result = tdSql.queryResult tdSql.query("select last_row(sub_tb_1.c1), last_row(sub_tb_2.c2) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") - + for ind , row in enumerate(last_row_result): tdSql.checkData(ind , 0 , row[0]) @@ -769,7 +769,7 @@ class TDTestCase: tdSql.query("select last_row(*), last(*) from sub_tb_1, sub_tb_2 where sub_tb_1.ts=sub_tb_2.ts") for ind , row in enumerate(last_row_result): tdSql.checkData(ind , 0 , row[0]) - + def support_super_table_test(self): tdSql.execute(" use testdb ") @@ -783,9 +783,9 @@ class TDTestCase: self.check_result_auto( " select t3,c1 from testdb.stb1 where c1 > 0 order by tbname " , "select t3 ,abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) self.check_result_auto( " select t4,c1 from testdb.stb1 where c1 > 0 order by tbname " , "select t4 , abs(c1) from testdb.stb1 where c1 > 0 order by tbname" ) pass - + def basic_query(self): - + tdLog.printNoPrefix("==========step2:test errors ==============") self.test_errors() @@ -828,19 +828,19 @@ class TDTestCase: self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'NONE'") self.basic_query() - # cache_last 1 + # cache_last 1 self.prepare_datas("'LAST_ROW'") self.prepare_tag_datas("'LAST_ROW'") self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_ROW'") self.basic_query() - # cache_last 2 + # cache_last 2 self.prepare_datas("'LAST_VALUE'") self.prepare_tag_datas("'LAST_VALUE'") self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'LAST_VALUE'") self.basic_query() - # cache_last 3 + # cache_last 3 self.prepare_datas("'BOTH'") self.prepare_tag_datas("'BOTH'") self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'BOTH'") From cc3967b9058bc380715c7d35077d1282ce89a4ec Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 18 Jul 2022 20:12:10 +0800 Subject: [PATCH 84/88] fix: parenthesized expression --- source/dnode/vnode/src/sma/smaRollup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 1f7664c22a..5b1a87be20 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -749,7 +749,7 @@ void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid) { pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo) { - if (pRSmaInfo = *(SRSmaInfo **)pRSmaInfo) { + if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { tdFreeRSmaInfo(pSma, pRSmaInfo, true); } taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); From 1e4dae929ce49bf551ce989b4553b062eb1f4086 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 18 Jul 2022 20:30:04 +0800 Subject: [PATCH 85/88] shell: limit taos_history file size and rows --- tools/shell/src/shellEngine.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 0982dbc019..836e8899af 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -737,13 +737,6 @@ int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool ver void shellReadHistory() { SShellHistory *pHistory = &shell.history; - int64_t file_size; - if (taosStatFile(pHistory->file, &file_size, NULL) != 0) { - return; - } else if (file_size > SHELL_MAX_COMMAND_SIZE) { - taosRemoveFile(pHistory->file); - return; - } TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) return; @@ -763,10 +756,31 @@ void shellReadHistory() { if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); + int64_t file_size; + if (taosStatFile(pHistory->file, &file_size, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) { + fprintf(stdout,"%s(%d) %s %08" PRId64 "\n", __FILE__, __LINE__,__func__,taosGetSelfPthreadId());fflush(stdout); + TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC); + if (pFile == NULL) return; + int32_t endIndex = pHistory->hstart; + if (endIndex != 0) { + endIndex = pHistory->hend; + } + for (int32_t i = (pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; i != endIndex;) { + printf("%d ",i); + taosFprintfFile(pFile, "%s\n", pHistory->hist[i]); + i = (i + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; + } + taosFprintfFile(pFile, "%s\n", pHistory->hist[endIndex]); + printf("\n"); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + } + pHistory->hend = pHistory->hstart; } void shellWriteHistory() { SShellHistory *pHistory = &shell.history; + if (pHistory->hend == pHistory->hstart) return; TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND); if (pFile == NULL) return; From 912652c2e69372b20d81629fbc41e68a443bd23e Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 18 Jul 2022 20:30:28 +0800 Subject: [PATCH 86/88] shell: limit taos_history file size and rows --- tools/shell/src/shellEngine.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 836e8899af..14cab041f6 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -766,12 +766,10 @@ void shellReadHistory() { endIndex = pHistory->hend; } for (int32_t i = (pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; i != endIndex;) { - printf("%d ",i); taosFprintfFile(pFile, "%s\n", pHistory->hist[i]); i = (i + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; } taosFprintfFile(pFile, "%s\n", pHistory->hist[endIndex]); - printf("\n"); taosFsyncFile(pFile); taosCloseFile(&pFile); } From 4dad41388140f2976d92f3c7060d3895909dd171 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 18 Jul 2022 21:06:39 +0800 Subject: [PATCH 87/88] test:modify --- tests/system-test/7-tmq/stbTagFilter.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/system-test/7-tmq/stbTagFilter.py b/tests/system-test/7-tmq/stbTagFilter.py index 82596a32ba..1bb3d24bde 100644 --- a/tests/system-test/7-tmq/stbTagFilter.py +++ b/tests/system-test/7-tmq/stbTagFilter.py @@ -97,18 +97,18 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl # update to half tables - paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) + # paraDict['rowsPerTbl'] = int(self.rowsPerTbl / 2) # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' - # queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) - queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) + queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) + # queryString = "select ts, c1, c2, t4 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicFromStb1, queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) @@ -242,11 +242,11 @@ class TDTestCase: self.tmqCase1() # self.tmqCase2() - self.prepareTestEnv() - tdLog.printNoPrefix("====================================================================") - tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") - self.snapshot = 1 - self.tmqCase1() + # self.prepareTestEnv() + # tdLog.printNoPrefix("====================================================================") + # tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + # self.snapshot = 1 + # self.tmqCase1() # self.tmqCase2() From 4c00aec172663e7d7998071aac72348a2032c484 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 18 Jul 2022 21:26:29 +0800 Subject: [PATCH 88/88] test: add tmq test case --- tests/system-test/7-tmq/tmqCommon.py | 7 +++++++ tests/system-test/7-tmq/tmqUpdate-1ctb.py | 13 +++++++++++-- tests/system-test/fulltest.sh | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 151558a33a..1f57099e3a 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -205,6 +205,13 @@ class TMQCom: tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) return + def drop_ctable(self, tsql, dbname=None, count=1, default_ctbname_prefix="ctb",ctbStartIdx=0): + for _ in range(count): + create_ctable_sql = f'drop table {dbname}.{default_ctbname_prefix}{ctbStartIdx};' + ctbStartIdx += 1 + tdLog.info("drop ctb sql: %s"%create_ctable_sql) + tsql.execute(create_ctable_sql) + # schema: (ts timestamp, c1 int, c2 binary(16)) def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None): tdLog.debug("start to insert data ............") diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index de11f0f9ab..5d891bdedf 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -116,7 +116,12 @@ class TDTestCase: # paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl consumerId = 0 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 3/2) + + if self.snapshot == 0: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/2)) + elif self.snapshot == 1: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1)) + topicList = topicFromStb1 ifcheckdata = 1 ifManualCommit = 1 @@ -199,7 +204,11 @@ class TDTestCase: # paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl consumerId = 1 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + if self.snapshot == 0: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (2)) + elif self.snapshot == 1: + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1)) + topicList = topicFromStb1 ifcheckdata = 1 ifManualCommit = 1 diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index d98764edb6..24d1be203d 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -182,7 +182,7 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py #python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -#python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py +python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb.py #python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py