From ae0641ec15645a4eb71b1279beb4152faa892f2e Mon Sep 17 00:00:00 2001 From: "pengrongkun94@qq.com" Date: Mon, 23 Dec 2024 10:26:12 +0800 Subject: [PATCH 1/9] add test case --- .../system-test/1-insert/db_tb_name_check.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/system-test/1-insert/db_tb_name_check.py b/tests/system-test/1-insert/db_tb_name_check.py index e217bbe183..2f7afef6ec 100644 --- a/tests/system-test/1-insert/db_tb_name_check.py +++ b/tests/system-test/1-insert/db_tb_name_check.py @@ -78,9 +78,34 @@ class TDTestCase: tdSql.error(f'create table {sql} (ts timestamp,c0 int)') tdSql.execute(f'trim database `{dbname}`') tdSql.execute(f'drop database `{dbname}`') + + def tb_name_len_check(self): + dbname = tdCom.getLongName(10) + tdSql.execute(f'create database if not exists `{dbname}` vgroups 1 replica 1') + tdSql.execute(f'use `{dbname}`') + tdSql.execute(f'CREATE STABLE `test_csv` (`ts` TIMESTAMP, `c1` VARCHAR(2000), `c2` VARCHAR(2000)) TAGS (`c3` VARCHAR(2000))') + tbname = "test_csv_a12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012" + tdSql.execute(f"INSERT INTO `{tbname}`\ + using `test_csv` (`c3`) tags('a12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890')\ + (`ts`,`c1`,`c2`) values(1591060628000,'a','1');") + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(1) + tdSql.execute(f'drop table {tbname}') + + tdSql.execute(f"INSERT INTO `{dbname}`.`{tbname}`\ + using `{dbname}`.`test_csv` (`c3`) tags('a12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890')\ + (`ts`,`c1`,`c2`) values(1591060628000,'a','1');") + tdSql.query(f'select * from {tbname}') + tdSql.checkRows(1) + tdSql.execute(f'drop table {tbname}') + + tdSql.execute(f'trim database `{dbname}`') + tdSql.execute(f'drop database `{dbname}`') + def run(self): self.db_name_check() self.tb_name_check() + self.tb_name_len_check() def stop(self): tdSql.close() From e5b148f20f59bd2ab71f89c8e9393fefac6abb29 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 23 Dec 2024 11:47:20 +0800 Subject: [PATCH 2/9] fix double free --- source/libs/index/src/indexTfile.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index e2693ff4a4..c6c091c630 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -44,10 +44,10 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval); static int tfileWriteFooter(TFileWriter* write); // handle file corrupt later -static int tfileReaderLoadHeader(TFileReader* reader); +static int tfileReaderLoadHeader(TFileReader* reader); static int32_t tfileReaderLoadFst(TFileReader* reader); -static int tfileReaderVerify(TFileReader* reader); -static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); +static int tfileReaderVerify(TFileReader* reader); +static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); static int32_t tfileGetFileList(const char* path, SArray** pResult); static int tfileRmExpireFile(SArray* result); @@ -106,6 +106,11 @@ TFileCache* tfileCacheCreate(SIndex* idx, const char* path) { SArray* files = NULL; int32_t code = tfileGetFileList(path, &files); + if (code != 0) { + indexError("failed to get file list since %s", tstrerror(code)); + goto End; + } + for (size_t i = 0; i < taosArrayGetSize(files); i++) { char* file = taosArrayGetP(files, i); @@ -1182,7 +1187,6 @@ _exception: TAOS_UNUSED(taosCloseDir(&pDir)); if (files != NULL) { taosArrayDestroyEx(files, tfileDestroyFileName); - taosArrayDestroy(files); } return code; } From 79b4b4bec151224c2ddc4d5f094100911e348294 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 24 Dec 2024 09:53:11 +0800 Subject: [PATCH 3/9] fix: use limit with join --- source/libs/executor/src/mergejoin.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/mergejoin.c b/source/libs/executor/src/mergejoin.c index 61809b99e0..adf1b4f0d1 100755 --- a/source/libs/executor/src/mergejoin.c +++ b/source/libs/executor/src/mergejoin.c @@ -46,7 +46,7 @@ static FORCE_INLINE bool mJoinBlkReachThreshold(SMJoinOperatorInfo* pInfo, int64 return blkRows >= pInfo->ctx.mergeCtx.blkThreshold; } - return (pInfo->execInfo.resRows + blkRows) >= pInfo->ctx.mergeCtx.limit; + return (pInfo->execInfo.resRows + blkRows) >= pInfo->ctx.mergeCtx.limit || blkRows >= pInfo->ctx.mergeCtx.blkThreshold; } From 4779f4a16fcb12c0af6a20363027152b69e367cf Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 24 Dec 2024 17:22:39 +0800 Subject: [PATCH 4/9] test case --- tests/parallel_test/cases.task | 1 + tests/pytest/util/cases.py | 38 ++++++++++++++++++++ tests/system-test/2-query/large_data.py | 46 +++++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 tests/system-test/2-query/large_data.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c9d28e0623..9e4e6f9c71 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -247,6 +247,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/td-32548.py +,,n,system-test,python3 ./test.py -f 2-query/large_data.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stddev_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stddev_test.py -Q 2 diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 536b8f30d3..8d31cda78c 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -19,6 +19,7 @@ import inspect import importlib import traceback from util.log import * +import platform class TDCase: @@ -146,5 +147,42 @@ class TDCases: tdLog.notice("total %d Cluster test case(s) executed" % (runNum)) + def getTaosBenchmarkPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] + else: + projPath = selfPath[: selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if (tool) in files: + rootRealPath = os.path.dirname(os.path.realpath(root)) + if "packaging" not in rootRealPath: + paths.append(os.path.join(root, tool)) + break + if len(paths) == 0: + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def taosBenchmarkExec(self, param): + buildPath = tdCases.getTaosBenchmarkPath() + + if (platform.system().lower() == 'windows'): + cmdStr1 = ' mintty -h never %s %s '%(buildPath, param) + tdLog.info(cmdStr1) + os.system(cmdStr1) + else: + cmdStr1 = '%s %s &'%(buildPath, param) + tdLog.info(cmdStr1) + os.system(cmdStr1) + + time.sleep(15) + tdCases = TDCases() diff --git a/tests/system-test/2-query/large_data.py b/tests/system-test/2-query/large_data.py new file mode 100644 index 0000000000..6a5e3ca53c --- /dev/null +++ b/tests/system-test/2-query/large_data.py @@ -0,0 +1,46 @@ +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def prepare_data(self): + tdSql.execute("drop database if exists test;") + + tdCases.taosBenchmarkExec("-t 2 -n 1000000 -b int,float,nchar -y") + + def ts5803(self): + tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 join test.d1 on d0.ts=d1.ts;") + num1 = tdSql.queryRows + + tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 join test.d1 on d0.ts=d1.ts limit 1000000;") + tdSql.checkRows(num1) + + tdSql.query("select d0.ts from test.d0 join test.d1 on d0.ts=d1.ts limit 1000000;") + tdSql.checkRows(num1) + + tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 left join test.d1 on d0.ts=d1.ts;") + num1 = tdSql.queryRows + + tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 left join test.d1 on d0.ts=d1.ts limit 1000000;") + tdSql.checkRows(num1) + + tdSql.query("select d0.ts from test.d0 left join test.d1 on d0.ts=d1.ts limit 1000000;") + tdSql.checkRows(num1) + + def run(self): + self.prepare_data() + self.ts5803() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From faaaa243afb4081e79c9591b781e467b48b9df4b Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 25 Dec 2024 09:09:27 +0800 Subject: [PATCH 5/9] fix: test case --- tests/system-test/2-query/large_data.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/system-test/2-query/large_data.py b/tests/system-test/2-query/large_data.py index 6a5e3ca53c..69b15ce821 100644 --- a/tests/system-test/2-query/large_data.py +++ b/tests/system-test/2-query/large_data.py @@ -14,6 +14,15 @@ class TDTestCase: tdSql.execute("drop database if exists test;") tdCases.taosBenchmarkExec("-t 2 -n 1000000 -b int,float,nchar -y") + + while True: + tdSql.query("select d0.ts from test.d0;") + num1 = tdSql.queryRows + tdSql.query("select d0.ts from test.d1;") + num2 = tdSql.queryRows + if num1 == 1000000 and num2 == 1000000: + break + time.sleep(1) def ts5803(self): tdSql.query("select d0.ts,d0.c1,d0.c2 from test.d0 join test.d1 on d0.ts=d1.ts;") From f4180ec3d0c0ba166cab9daaeba6263b3946c0cd Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 25 Dec 2024 09:11:54 +0800 Subject: [PATCH 6/9] fix: win system error desc --- source/util/src/terror.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index d2d551a539..b2a8c422f7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -908,12 +908,6 @@ const char* tstrerror(int32_t err) { (void)taosThreadOnce(&tsErrorInit, tsSortError); // this is a system errno - if ((err & 0x00ff0000) == 0x00ff0000) { - int32_t code = err & 0x0000ffff; - // strerror can handle any invalid code - // invalid code return Unknown error - return strerror(code); - } #ifdef WINDOWS if ((err & 0x01ff0000) == 0x01ff0000) { snprintf(WinAPIErrDesc, 256, "windows api error, code: 0x%08x", err & 0x0000ffff); @@ -923,6 +917,13 @@ const char* tstrerror(int32_t err) { return WinAPIErrDesc; } #endif + if ((err & 0x00ff0000) == 0x00ff0000) { + int32_t code = err & 0x0000ffff; + // strerror can handle any invalid code + // invalid code return Unknown error + return strerror(code); + } + int32_t s = 0; int32_t e = sizeof(errors) / sizeof(errors[0]); From 4a08d6915909a0f264b685bf1f705366d080d659 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 25 Dec 2024 09:18:57 +0800 Subject: [PATCH 7/9] fix: test case --- tests/pytest/util/cases.py | 2 +- tests/system-test/2-query/large_data.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 8d31cda78c..eee8809ad0 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -183,6 +183,6 @@ class TDCases: tdLog.info(cmdStr1) os.system(cmdStr1) - time.sleep(15) + time.sleep(5) tdCases = TDCases() diff --git a/tests/system-test/2-query/large_data.py b/tests/system-test/2-query/large_data.py index 69b15ce821..2279c3ce70 100644 --- a/tests/system-test/2-query/large_data.py +++ b/tests/system-test/2-query/large_data.py @@ -16,12 +16,13 @@ class TDTestCase: tdCases.taosBenchmarkExec("-t 2 -n 1000000 -b int,float,nchar -y") while True: - tdSql.query("select d0.ts from test.d0;") + tdSql.query("select ts from test.d0;") num1 = tdSql.queryRows - tdSql.query("select d0.ts from test.d1;") + tdSql.query("select ts from test.d1;") num2 = tdSql.queryRows if num1 == 1000000 and num2 == 1000000: break + tdLog.info(f"waiting for data ready, d0: {num1}, d1: {num2}") time.sleep(1) def ts5803(self): From c18444d3e26b4e7e678c31c8643615f0356fe396 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Wed, 25 Dec 2024 11:21:41 +0800 Subject: [PATCH 8/9] docs/optimize the description in 03-table.md --- docs/zh/14-reference/03-taos-sql/03-table.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-taos-sql/03-table.md b/docs/zh/14-reference/03-taos-sql/03-table.md index ca2170db8b..cf29ef1451 100644 --- a/docs/zh/14-reference/03-taos-sql/03-table.md +++ b/docs/zh/14-reference/03-taos-sql/03-table.md @@ -6,7 +6,7 @@ description: 对表的各种管理操作 ## 创建表 -`CREATE TABLE` 语句用于创建普通表和以超级表为模板创建子表。 +`CREATE TABLE` 语句用于创建普通表和以超级表为模板创建子表（也可以通过指定 TAGS 字段创建超级表）。 ```sql CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definition] ...) [table_options] From c1e93d499a7a83a624ca359bebf88624d14a85bc Mon Sep 17 00:00:00 2001 From: qevolg <2227465945@qq.com> Date: Wed, 25 Dec 2024 14:42:02 +0800 Subject: [PATCH 9/9] fix(keeper): rollback deps --- tools/keeper/go.mod | 6 +++--- tools/keeper/go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/keeper/go.mod b/tools/keeper/go.mod index 00a399768d..f8edf2709b 100644 --- a/tools/keeper/go.mod +++ b/tools/keeper/go.mod @@ -70,10 +70,10 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect golang.org/x/arch v0.7.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/tools/keeper/go.sum b/tools/keeper/go.sum index f879c731ba..8f6e9bd13a 100644 --- a/tools/keeper/go.sum +++ b/tools/keeper/go.sum @@ -424,8 +424,8 @@ golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -573,8 +573,8 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -583,8 +583,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=