From 41b9b98e3c4e11df8492335516be7d3083d76af1 Mon Sep 17 00:00:00 2001 From: wu champion Date: Tue, 6 Apr 2021 18:06:59 +0800 Subject: [PATCH 1/6] [TD-3652] add case for TD-3652 to resolve TD-3590 --- tests/pytest/fulltest.sh | 2 + tests/pytest/query/queryStddevWithGroupby.py | 68 ++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 tests/pytest/query/queryStddevWithGroupby.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 7ff41b13a6..3266a1ab8f 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -22,6 +22,7 @@ python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py python3 ./test.py -f insert/before_1970.py python3 bug2265.py +python3 ./test.py -f insert/bug3654.py #table python3 ./test.py -f table/alter_wal0.py @@ -216,6 +217,7 @@ python3 ./test.py -f query/floatCompare.py python3 ./test.py -f query/query1970YearsAf.py python3 ./test.py -f query/bug3351.py python3 ./test.py -f query/bug3375.py +python3 ./test.py -f query/queryStddevWithGroupby.py diff --git a/tests/pytest/query/queryStddevWithGroupby.py b/tests/pytest/query/queryStddevWithGroupby.py new file mode 100644 index 0000000000..aee88ca9c5 --- /dev/null +++ b/tests/pytest/query/queryStddevWithGroupby.py @@ -0,0 +1,68 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def querysqls(self): + tdSql.query("select stddev(c1) from t10 group by c1") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, 0) + tdSql.checkData(2, 0, 0) + tdSql.checkData(3, 0, 0) + tdSql.checkData(4, 0, 0) + tdSql.checkData(5, 0, 0) + tdSql.query("select stddev(c2) from t10") + tdSql.checkData(0, 0, 0.5) + + def run(self): + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db keep 36500") + tdSql.execute("use db") + + tdLog.printNoPrefix("==========step1:create table && insert data") + tdSql.execute("create stable stb1 (ts timestamp , c1 int ,c2 float) tags(t1 int)") + tdSql.execute("create table t10 using stb1 tags(1)") + tdSql.execute("insert into t10 values ('1969-12-31 00:00:00.000', 2,1)") + tdSql.execute("insert into t10 values ('1970-01-01 00:00:00.000', 3,1)") + tdSql.execute("insert into t10 values (0, 4,1)") + tdSql.execute("insert into t10 values (now-18725d, 1,2)") + tdSql.execute("insert into t10 values ('2021-04-06 00:00:00.000', 5,2)") + tdSql.execute("insert into t10 values (now+1d,6,2)") + + tdLog.printNoPrefix("==========step2:query and check") + self.querysqls() + + tdLog.printNoPrefix("==========step3:after wal,check again") + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + tdDnodes.stop(index) + tdDnodes.start(index) + self.querysqls() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 9f858447f74317054e0e67e19b4fccc8ec413702 Mon Sep 17 00:00:00 2001 From: wu champion Date: Tue, 6 Apr 2021 18:09:13 +0800 Subject: [PATCH 2/6] Update fulltest.sh --- tests/pytest/fulltest.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 3266a1ab8f..9d52b76a67 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -22,7 +22,6 @@ python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py python3 ./test.py -f insert/before_1970.py python3 bug2265.py -python3 ./test.py -f insert/bug3654.py #table python3 ./test.py -f table/alter_wal0.py From 7db5c29d1ee95bd91f9976ecf2f7211167a9c01c Mon Sep 17 00:00:00 2001 From: wu champion Date: Tue, 6 Apr 2021 18:18:10 +0800 Subject: [PATCH 3/6] [TD-3295] add case for TD-3295 --- tests/pytest/fulltest.sh | 2 +- tests/pytest/query/queryJoin10tables.py | 201 ++++++++++++++++++++++++ 2 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 tests/pytest/query/queryJoin10tables.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 7ff41b13a6..c2db4b3503 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -216,7 +216,7 @@ python3 ./test.py -f query/floatCompare.py python3 ./test.py -f query/query1970YearsAf.py python3 ./test.py -f query/bug3351.py python3 ./test.py -f query/bug3375.py - +python3 ./test.py -f query/queryJoin10tables.py #stream diff --git a/tests/pytest/query/queryJoin10tables.py b/tests/pytest/query/queryJoin10tables.py new file mode 100644 index 0000000000..01a7397d44 --- /dev/null +++ b/tests/pytest/query/queryJoin10tables.py @@ -0,0 +1,201 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def createtable(self): + + # create stbles + tdSql.execute("create table if not exists stb1 (ts timestamp, c1 int) tags(t11 int, t12 int)") + tdSql.execute("create table if not exists stb2 (ts timestamp, c2 int) tags(t21 int, t22 int)") + tdSql.execute("create table if not exists stb3 (ts timestamp, c3 int) tags(t31 int, t32 int)") + tdSql.execute("create table if not exists stb4 (ts timestamp, c4 int) tags(t41 int, t42 int)") + tdSql.execute("create table if not exists stb5 (ts timestamp, c5 int) tags(t51 int, t52 int)") + tdSql.execute("create table if not exists stb6 (ts timestamp, c6 int) tags(t61 int, t62 int)") + tdSql.execute("create table if not exists stb7 (ts timestamp, c7 int) tags(t71 int, t72 int)") + tdSql.execute("create table if not exists stb8 (ts timestamp, c8 int) tags(t81 int, t82 int)") + tdSql.execute("create table if not exists stb9 (ts timestamp, c9 int) tags(t91 int, t92 int)") + tdSql.execute("create table if not exists stb10 (ts timestamp, c10 int) tags(t101 int, t102 int)") + tdSql.execute("create table if not exists stb11 (ts timestamp, c11 int) tags(t111 int, t112 int)") + + # create normal tables + tdSql.execute("create table t10 using stb1 tags(0, 9)") + tdSql.execute("create table t11 using stb1 tags(1, 8)") + tdSql.execute("create table t12 using stb1 tags(2, 7)") + tdSql.execute("create table t13 using stb1 tags(3, 6)") + tdSql.execute("create table t14 using stb1 tags(4, 5)") + tdSql.execute("create table t15 using stb1 tags(5, 4)") + tdSql.execute("create table t16 using stb1 tags(6, 3)") + tdSql.execute("create table t17 using stb1 tags(7, 2)") + tdSql.execute("create table t18 using stb1 tags(8, 1)") + tdSql.execute("create table t19 using stb1 tags(9, 0)") + tdSql.execute("create table t110 using stb1 tags(10, 10)") + + tdSql.execute("create table t20 using stb2 tags(0, 9)") + tdSql.execute("create table t21 using stb2 tags(1, 8)") + tdSql.execute("create table t22 using stb2 tags(2, 7)") + + tdSql.execute("create table t30 using stb3 tags(0, 9)") + tdSql.execute("create table t31 using stb3 tags(1, 8)") + tdSql.execute("create table t32 using stb3 tags(2, 7)") + + def inserttable(self): + for i in range(100): + if i<60: + tdSql.execute(f"insert into t20 values('2020-10-01 00:00:{i}.000', {i})") + tdSql.execute(f"insert into t21 values('2020-10-01 00:00:{i}.000', {i})") + tdSql.execute(f"insert into t22 values('2020-10-01 00:00:{i}.000', {i})") + tdSql.execute(f"insert into t30 values('2020-10-01 00:00:{i}.000', {i})") + tdSql.execute(f"insert into t31 values('2020-10-01 00:00:{i}.000', {i})") + tdSql.execute(f"insert into t32 values('2020-10-01 00:00:{i}.000', {i})") + else: + tdSql.execute(f"insert into t20 values('2020-10-01 00:01:{i-60}.000', {i})") + tdSql.execute(f"insert into t21 values('2020-10-01 00:01:{i-60}.000', {i})") + tdSql.execute(f"insert into t22 values('2020-10-01 00:01:{i-60}.000', {i})") + tdSql.execute(f"insert into t30 values('2020-10-01 00:01:{i-60}.000', {i})") + tdSql.execute(f"insert into t31 values('2020-10-01 00:01:{i-60}.000', {i})") + tdSql.execute(f"insert into t32 values('2020-10-01 00:01:{i-60}.000', {i})") + for j in range(11): + if i<60: + tdSql.execute(f"insert into t1{j} values('2020-10-01 00:00:{i}.000', {i})") + else: + tdSql.execute(f"insert into t1{j} values('2020-10-01 00:01:{i-60}.000', {i})") + + def queryjointable(self): + tdSql.error( + '''select from t10,t11,t12,t13,t14,t15,t16,t17,t18,t19 + where t10.ts=t11.ts and t10.ts=t12.ts and t10.ts=t13.ts and t10.ts=t14.ts and t10.ts=t15.ts + and t10.ts=t16.ts and t10.ts=t17.ts and t10.ts=t18.ts and t10.ts=t19.ts''' + ) + tdSql.error("select * from t10 where t10.ts=t11.ts") + tdSql.error("select * from where t10.ts=t11.ts") + tdSql.error("select * from t10,t11,t12,t13,t14,t15,t16,t17,t18,t19") + tdSql.error("select * from stb1, stb2, stb3 where stb1.ts=stb2.ts and stb1.ts=stb3.ts") + tdSql.error("select * from stb1, stb2, stb3 where stb1.t11=stb2.t21 and stb1.t11=stb3.t31") + tdSql.error("select * from stb1, stb2, stb3") + tdSql.error( + '''select * from stb1 + join stb2 on stb1.ts=stb2.ts and stb1.t11=stb2.t21 + join stb3 on stb1.ts=stb3.ts and stb1.t11=stb3.t31''' + ) + tdSql.error("select * from t10 join t11 on t10.ts=t11.ts join t12 on t11.ts=t12.ts") + tdSql.query( + '''select * from stb1,stb2,stb3 + where stb1.ts=stb2.ts and stb1.ts=stb3.ts and stb1.t11=stb2.t21 and stb1.t11 =stb3.t31''' + ) + tdSql.checkRows(300) + tdSql.query("select * from t11,t12,t13 where t11.ts=t12.ts and t11.ts=t13.ts") + tdSql.checkRows(100) + tdSql.error("selec * from t11,t12,t13 where t11.ts=t12.ts and t11.ts=t13.ts") + tdSql.error("select * form t11,t12,t13 where t11.ts=t12.ts and t11.ts=t13.ts") + tdSql.error("select * from t11,t12,t13 when t11.ts=t12.ts and t11.ts=t13.ts") + tdSql.error("select * from t11,t12,t13 when t11.ts <> t12.ts and t11.ts=t13.ts") + tdSql.error("select * from t11,t12,t13 when t11.ts != t12.ts and t11.ts=t13.ts") + tdSql.error("select * from t11,t12,t13 when t11.ts=t12.ts or t11.ts=t13.ts") + tdSql.error("select * from t11,t12,t13 when t11.ts=t12.ts=t13.ts") + tdSql.error("select * from t11,t12,t13 when t11.c1=t12.c2 and t11.c1=t13.c3") + tdSql.error("select * from t11,t12,t13 when t11.ts=t12.ts and t11.ts=t13.c3 and t11.c1=t13.ts") + tdSql.error("select ts from t11,t12,t13 when t11.ts=t12.ts and t11.ts=t13.ts") + tdSql.error("select * from t11,t12,t13 when t11.ts=ts and t11.ts=t13.ts") + tdSql.error("select * from t11,t12,t13 when t11.ts=t12.ts and t11.ts=t13.ts and ts>100") + tdSql.error("select * from t11,t12,stb1 when t11.ts=t12.ts and t11.ts=stb1.ts") + tdSql.error("select t14.ts from t11,t12,t13 when t11.ts=t12.ts and t11.ts=t13.ts") + tdSql.error("select * from t11,t12,t13 when t11.ts=t12.ts and t11.ts=t13.ts1") + tdSql.error("select * from t11,t12,t13 when t11.ts=t12.ts and t11.ts=t14.ts") + tdSql.error("select * from t11,t12,t13 when t11.ts=t12.ts") + tdSql.error("select * from t11,t12,t13 when t11.ts=t12.ts and t11.ts=t13.ts and t11.c1=t13.c3") + tdSql.error( + '''select * from t10,t11,t12,t13,t14,t15,t16,t17,t18,t19,t20 + where t10.ts=t11.ts and t10.ts=t12.ts and t10.ts=t13.ts and t10.ts=t14.ts and t10.ts=t15.ts + and t10.ts=t16.ts and t10.ts=t17.ts and t10.ts=t18.ts and t10.ts=t19.ts and t10.ts=t20.ts''' + ) + tdSql.error( + '''select * from t10,t11,t12,t13,t14,t15,t16,t17,t18,t19,t20 + where t10.ts=t11.ts and t10.ts=t12.ts and t10.ts=t13.ts and t10.ts=t14.ts and t10.ts=t15.ts + and t10.ts=t16.ts and t10.ts=t17.ts and t10.ts=t18.ts and t10.ts=t19.ts''' + ) + tdSql.error( + '''select * from t10,t11,t12,t13,t14,t15,t16,t17,t18,t19 + where t10.ts=t11.ts and t10.ts=t12.ts and t10.ts=t13.ts and t10.ts=t14.ts and t10.ts=t15.ts + and t10.ts=t16.ts and t10.ts=t17.ts and t10.ts=t18.ts and t10.ts=t19.ts and t10.c1=t19.c1''' + ) + tdSql.error( + '''select * from stb1,stb2,stb3 + where stb1.ts=stb2.ts and stb1.ts=stb3.ts and stb1.t11=stb2.t21''' + ) + tdSql.error( + '''select * from stb1,stb2,stb3 + where stb1.ts=stb2.ts and stb1.t11=stb2.t21 and stb1.t11=stb3.t31''' + ) + tdSql.error( + '''select * from stb1,stb2,stb3 + where stb1.ts=stb2.ts and stb1.ts=stb3.ts and stb1.t11=stb2.t21 and stb1.t11=stb3.t31 + and stb1.t12=stb3=t32''' + ) + tdSql.error( + '''select * from stb1,stb2,stb3,stb4,stb5,stb6,stb7,stb8,stb9,stb10,stb11 + where stb1.ts=stb2.ts and stb1.ts=stb3.ts and stb1.ts=stb4.ts and stb1.ts=stb5.ts and stb1.ts=stb6.ts + and stb1.ts=stb7.ts and stb1.ts=stb8.ts and stb1.ts=stb9.ts and stb1.ts=stb10.ts and stb1.ts=stb11.ts + and stb1.t11=stb2.t21 and stb1.t11=stb3.t31 and stb1.t11=stb4.t41 and stb1.t11=stb5.t51 + and stb1.t11=stb6.t61 and stb1.t11=stb7.t71 and stb1.t11=stb8.t81 and stb1.t11=stb9.t91 + and stb1.t11=stb10.t101 and stb1.t11=stb11.t111''' + ) + tdSql.error( + '''select * from stb1,stb2,stb3,stb4,stb5,stb6,stb7,stb8,stb9,stb10 + where stb1.ts=stb2.ts and stb1.ts=stb3.ts and stb1.ts=stb4.ts and stb1.ts=stb5.ts and stb1.ts=stb6.ts + and stb1.ts=stb7.ts and stb1.ts=stb8.ts and stb1.ts=stb9.ts and stb1.ts=stb10.ts and stb1.t11=stb2.t21 + and stb1.t11=stb3.t31 and stb1.t11=stb4.t41 and stb1.t11=stb5.t51 and stb1.t11=stb6.t61 + and stb1.t11=stb7.t71 and stb1.t11=stb8.t81 and stb1.t11=stb9.t91 and stb1.t11=stb10.t101 + and stb1.t12=stb11.t102''' + ) + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + self.createtable() + + tdLog.printNoPrefix("==========step2:insert data") + self.inserttable() + + tdLog.printNoPrefix("==========step3:query timestamp type") + self.queryjointable() + + # after wal and sync, check again + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + tdDnodes.stop(index) + tdDnodes.start(index) + + tdLog.printNoPrefix("==========step4:query again after wal") + self.queryjointable() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 568a3ebc87a99e07e0e50633d97c44256861c037 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 7 Apr 2021 17:55:54 +0800 Subject: [PATCH 4/6] Hotfix/sangshuduo/td 3607 for master (#5712) * test * [TD-3677]: test pr message 1 * [TD-3671]change target branch * [TD-3677]: test pr message 2 * [TD-3677]: test pr message 3 * Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#5688) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix coverity scan issues. check super tbl info pointer. * [TD-3197] : fix coverity scan issues. move sub tbl query thread join into loop * [TD-3197] : fix coverity scan issues. remove unused variable * [TD-3197] : fix coverity scan issues. use more secure random library * [TD-3197] : fix coverity scan issues. use strncpy for more safe * [TD-3197] : fix taosdemo coverity scan issue. replace arc4random with rand(). * [TD-3197] : fix coverity scan issues. check stb info pointer for start time * [TD-3197] : fix coverity scan issues. fix strcpy vulnerability * [TD-3197] : fix taosdemo coverity scan issue. modify taosdemoTest2. try to check database continously. * [TD-3197] : taosdemo coverity scan issues. * [TD-3197] : fix memory leak when parsing arguments. * [TD-3197] : fix cmake strip arguments. * [TD-3197] : taosdemo coverity scan. fix cmake string manipulation. Co-authored-by: Shuduo Sang * remove useless file * fix changing target branch * fix * fix * [TD-3607]: fix taosdemo limit and offset. if offset+limit > count. * [TD-3607]: taosdemo limit and offset. if child tbl not exist, dont take limit and offset value. Co-authored-by: huili <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: liuyq-617 Co-authored-by: plum-lihui Co-authored-by: Elias Soong Co-authored-by: Shuduo Sang --- Jenkinsfile | 2 +- src/kit/taosdemo/taosdemo.c | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0d6939af31..dfe9ed4389 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -82,9 +82,9 @@ def pre_test(){ } } sh ''' - cd ${WK} git pull >/dev/null + export TZ=Asia/Harbin date git clean -dfx diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 713b39d98b..30f096954c 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3479,9 +3479,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (childTblExists && childTblExists->type == cJSON_String && childTblExists->valuestring != NULL) { - if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) { + if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3)) + && (g_Dbs.db[i].drop == false)) { g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; - } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) { + } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2) + || (g_Dbs.db[i].drop == true))) { g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; } else { g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; @@ -3527,7 +3529,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit"); - if ((childTbl_limit) && (g_Dbs.db[i].drop != true)) { + if ((childTbl_limit) && (g_Dbs.db[i].drop != true) + && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { if (childTbl_limit->type != cJSON_Number) { printf("ERROR: failed to read json, childtable_limit\n"); goto PARSE_OVER; @@ -3538,7 +3541,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset"); - if ((childTbl_offset) && (g_Dbs.db[i].drop != true)) { + if ((childTbl_offset) && (g_Dbs.db[i].drop != true) + && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) { printf("ERROR: failed to read json, childtable_offset\n"); goto PARSE_OVER; From bbf49cb8c63932825e727ec847eb62bfdb3dffdc Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 8 Apr 2021 10:50:26 +0800 Subject: [PATCH 5/6] [TD-3683]: reduce buffer size for more stable table creation. (#5720) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 30f096954c..8c0c755113 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2573,10 +2573,7 @@ static void* createTable(void *sarg) int64_t lastPrintTime = taosGetTimestampMs(); int buff_len; - if (superTblInfo) - buff_len = superTblInfo->maxSqlLen; - else - buff_len = BUFFER_SIZE; + buff_len = BUFFER_SIZE / 8; char *buffer = calloc(buff_len, 1); if (buffer == NULL) { @@ -2624,7 +2621,7 @@ static void* createTable(void *sarg) return NULL; } len += snprintf(buffer + len, - superTblInfo->maxSqlLen - len, + buff_len - len, "if not exists %s.%s%d using %s.%s tags %s ", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, @@ -2632,7 +2629,7 @@ static void* createTable(void *sarg) free(tagsValBuf); batchNum++; if ((batchNum < superTblInfo->batchCreateTableNum) - && ((superTblInfo->maxSqlLen - len) + && ((buff_len - len) >= (superTblInfo->lenOfTagOfOneRow + 256))) { continue; } @@ -5174,8 +5171,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, if ((superTblInfo->childTblExists == TBL_ALREADY_EXISTS) && (superTblInfo->childTblOffset >= 0)) { - if ((superTblInfo->childTblLimit < 0) - || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit) + if ((superTblInfo->childTblLimit < 0) + || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit) > (superTblInfo->childTblCount))) { superTblInfo->childTblLimit = superTblInfo->childTblCount - superTblInfo->childTblOffset; From 5a200ff74645be90469fe90b4d08c5f592d1b793 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 9 Apr 2021 12:04:57 +0800 Subject: [PATCH 6/6] [td-225]additional memory border check for the temp query output buffer --- src/query/inc/qUtil.h | 15 ++++++++++++--- src/query/src/qExecutor.c | 8 +++----- src/query/src/qUtil.c | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index cdd8b0707a..cb8c9679ec 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -52,11 +52,20 @@ static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int return pResultRowInfo->pResult[slot]; } -static FORCE_INLINE char *getPosInResultPage(SQuery *pQuery, tFilePage* page, int32_t rowOffset, int16_t offset) { - assert(rowOffset >= 0 && pQuery != NULL); +static FORCE_INLINE char* getPosInResultPage(SQueryRuntimeEnv* pRuntimeEnv, tFilePage* page, int32_t rowOffset, + int16_t offset, int32_t size) { + assert(rowOffset >= 0 && pRuntimeEnv != NULL); + + SQuery* pQuery = pRuntimeEnv->pQuery; + int64_t pageSize = pRuntimeEnv->pResultBuf->pageSize; int32_t numOfRows = (int32_t)GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pQuery->topBotQuery, pQuery->stableQuery); - return ((char *)page->data) + rowOffset + offset * numOfRows; + + // buffer overflow check + int64_t bufEnd = (rowOffset + offset * numOfRows + size); + assert(page->num <= pageSize && bufEnd <= page->num); + + return ((char*)page->data) + rowOffset + offset * numOfRows; } bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index bd5fdda0f9..20b3ce8f55 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3243,7 +3243,7 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe continue; } - pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQuery, bufPage, pResult->offset, offset); + pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv, bufPage, pResult->offset, offset, pCtx[i].outputBytes); offset += pCtx[i].outputBytes; int32_t functionId = pCtx[i].functionId; @@ -3301,7 +3301,7 @@ void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLF int16_t offset = 0; for (int32_t i = 0; i < numOfCols; ++i) { - pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQuery, page, pResult->offset, offset); + pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv, page, pResult->offset, offset, pCtx[i].outputBytes); offset += pCtx[i].outputBytes; int32_t functionId = pCtx[i].functionId; @@ -3509,8 +3509,6 @@ void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { */ static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { - SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t numOfResult = pBlock->info.rows; // there are already exists result rows @@ -3545,7 +3543,7 @@ static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* int32_t bytes = pColInfoData->info.bytes; char *out = pColInfoData->pData + numOfResult * bytes; - char *in = getPosInResultPage(pQuery, page, pRow->offset, offset); + char *in = getPosInResultPage(pRuntimeEnv, page, pRow->offset, offset, bytes); memcpy(out, in, bytes * numOfRowsToCopy); offset += bytes; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 9b0046fda0..aa793add84 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -140,7 +140,7 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 SResultRowCellInfo *pResultInfo = &pResultRow->pCellInfo[i]; int16_t size = pRuntimeEnv->pQuery->pExpr1[i].bytes; - char * s = getPosInResultPage(pRuntimeEnv->pQuery, page, pResultRow->offset, offset); + char * s = getPosInResultPage(pRuntimeEnv, page, pResultRow->offset, offset, size); memset(s, 0, size); offset += size;