From 49865ed294ec75a1d0cc1b5f10a958901de21b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 11 Nov 2024 14:57:06 +0800 Subject: [PATCH 01/62] test:add stream test cases --- tests/system-test/8-stream/checkpoint_info.py | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 tests/system-test/8-stream/checkpoint_info.py diff --git a/tests/system-test/8-stream/checkpoint_info.py b/tests/system-test/8-stream/checkpoint_info.py new file mode 100644 index 0000000000..00e5d1f688 --- /dev/null +++ b/tests/system-test/8-stream/checkpoint_info.py @@ -0,0 +1,139 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.cluster import * +import threading +# should be used by -N option +class TDTestCase: + + #updatecfgDict = {'checkpointInterval': 60 ,} + def init(self, conn, logSql, replicaVar=1): + print("========init========") + + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + def find_checkpoint_info_file(self, dirpath, checkpointid, task_id): + for root, dirs, files in os.walk(dirpath): + if f'checkpoint{checkpointid}' in dirs: + info_path = os.path.join(root, f'checkpoint{checkpointid}', 'info') + if os.path.exists(info_path): + if task_id in info_path: + return info_path + else: + continue + else: + return None + def get_dnode_info(self): + ''' + get a dict from vnode to dnode + ''' + self.vnode_dict = {} + sql = 'select dnode_id, vgroup_id from information_schema.ins_vnodes' + result = tdSql.getResult(sql) + for (dnode,vnode) in result: + self.vnode_dict[vnode] = dnode + def print_time_info(self): + ''' + sometimes, we need to wait for a while to check the info (for example, the checkpoint info file won't be created immediately after the redistribute) + ''' + times= 0 + while(True): + if(self.check_info()): + tdLog.success(f'Time to finish is {times}') + return + else: + if times > 200: + tdLog.exit("time out") + times += 10 + time.sleep(10) + def check_info(self): + ''' + first, check if the vnode is restored + ''' + while(True): + if(self.check_vnodestate()): + break + sql = 'select task_id, node_id, checkpoint_id, checkpoint_ver from information_schema.ins_stream_tasks where `level` = "source" or `level` = "agg" and node_type == "vnode"' + for task_id, vnode, checkpoint_id, checkpoint_ver in tdSql.getResult(sql): + dirpath = f"{cluster.dnodes[self.vnode_dict[vnode]-1].dataDir}/vnode/vnode{vnode}/" + info_path = self.find_checkpoint_info_file(dirpath, checkpoint_id, task_id) + if info_path is None: + return False + with open(info_path, 'r') as f: + info_id, info_ver = f.read().split() + if int(info_id) != int(checkpoint_id) or int(info_ver) != int(checkpoint_ver): + return False + return True + + def restart_stream(self): + tdLog.debug("========restart stream========") + for i in range(5): + tdSql.execute("pause stream s1") + time.sleep(2) + tdSql.execute("resume stream s1") + def initstream(self): + tdLog.debug("========case1 start========") + os.system("nohup taosBenchmark -y -B 1 -t 4 -S 500 -n 1000 -v 3 > /dev/null 2>&1 &") + time.sleep(5) + tdSql.execute("create snode on dnode 1") + tdSql.execute("use test") + tdSql.execute("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(1s)") + tdLog.debug("========create stream using snode and insert data ok========") + self.get_dnode_info() + def redistribute_vnode(self): + tdLog.debug("========redistribute vnode========") + tdSql.redistribute_db_all_vgroups() + self.get_dnode_info() + def replicate_db(self): + tdLog.debug("========replicate db========") + while True: + res = tdSql.getResult("SHOW TRANSACTIONS") + if res == []: + tdLog.debug("========== no transaction, begin to replicate db =========") + tdSql.execute("alter database test replica 3") + return + else: + time.sleep(5) + continue + def check_vnodestate(self): + sql = 'select distinct restored from information_schema.ins_vnodes' + if tdSql.getResult(sql) != [(True,)]: + tdLog.debug(f"vnode not restored, wait 5s") + time.sleep(5) + return False + else: + return True + def run(self): + print("========run========") + self.initstream() + self.restart_stream() + time.sleep(60) + self.print_time_info() + self.redistribute_vnode() + self.restart_stream() + time.sleep(60) + self.print_time_info() + + def stop(self): + print("========stop========") + tdSql.close() + tdLog.success(f"{__file__} successfully executed") +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 7eff826097036ab41a44d9faca8bb1a7e4f8e37a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 11 Nov 2024 14:57:19 +0800 Subject: [PATCH 02/62] test:add stream test cases --- .../system-test/8-stream/checkpoint_info2.py | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 tests/system-test/8-stream/checkpoint_info2.py diff --git a/tests/system-test/8-stream/checkpoint_info2.py b/tests/system-test/8-stream/checkpoint_info2.py new file mode 100644 index 0000000000..7f2e8b0672 --- /dev/null +++ b/tests/system-test/8-stream/checkpoint_info2.py @@ -0,0 +1,134 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.cluster import * + +# should be used by -N option +class TDTestCase: + updatecfgDict = {'checkpointInterval': 60 , + } + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + def find_checkpoint_info_file(self, dirpath, checkpointid, task_id): + for root, dirs, files in os.walk(dirpath): + if f'checkpoint{checkpointid}' in dirs: + info_path = os.path.join(root, f'checkpoint{checkpointid}', 'info') + if os.path.exists(info_path): + if task_id in info_path: + return info_path + else: + continue + else: + return None + def get_dnode_info(self): + ''' + get a dict from vnode to dnode + ''' + self.vnode_dict = {} + sql = 'select dnode_id, vgroup_id from information_schema.ins_vnodes' + result = tdSql.getResult(sql) + for (dnode,vnode) in result: + self.vnode_dict[vnode] = dnode + def print_time_info(self): + ''' + sometimes, we need to wait for a while to check the info (for example, the checkpoint info file won't be created immediately after the redistribute) + ''' + times= 0 + while(True): + if(self.check_info()): + tdLog.success(f'Time to finish is {times}') + return + else: + if times > 400: + tdLog.exit("time out") + times += 10 + time.sleep(10) + def check_info(self): + ''' + first, check if the vnode is restored + ''' + while(True): + if(self.check_vnodestate()): + break + sql = 'select task_id, node_id, checkpoint_id, checkpoint_ver from information_schema.ins_stream_tasks where `level` = "source" or `level` = "agg" and node_type == "vnode"' + for task_id, vnode, checkpoint_id, checkpoint_ver in tdSql.getResult(sql): + dirpath = f"{cluster.dnodes[self.vnode_dict[vnode]-1].dataDir}/vnode/vnode{vnode}/" + info_path = self.find_checkpoint_info_file(dirpath, checkpoint_id, task_id) + if info_path is None: + return False + with open(info_path, 'r') as f: + info_id, info_ver = f.read().split() + if int(info_id) != int(checkpoint_id) or int(info_ver) != int(checkpoint_ver): + return False + return True + + def restart_stream(self): + tdLog.debug("========restart stream========") + for i in range(5): + tdSql.execute("pause stream s1") + time.sleep(2) + tdSql.execute("resume stream s1") + def initstream(self): + tdLog.debug("========case1 start========") + os.system("nohup taosBenchmark -y -B 1 -t 4 -S 500 -n 1000 -v 3 > /dev/null 2>&1 &") + time.sleep(5) + tdSql.execute("create snode on dnode 1") + tdSql.execute("use test") + tdSql.execute("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(1s)") + tdLog.debug("========create stream using snode and insert data ok========") + self.get_dnode_info() + def redistribute_vnode(self): + tdLog.debug("========redistribute vnode========") + tdSql.redistribute_db_all_vgroups() + self.get_dnode_info() + def replicate_db(self): + tdLog.debug("========replicate db========") + while True: + res = tdSql.getResult("SHOW TRANSACTIONS") + if res == []: + tdLog.debug("========== no transaction, begin to replicate db =========") + tdSql.execute("alter database test replica 3") + return + else: + time.sleep(5) + continue + def check_vnodestate(self): + sql = 'select distinct restored from information_schema.ins_vnodes' + if tdSql.getResult(sql) != [(True,)]: + tdLog.debug(f"vnode not restored, wait 5s") + time.sleep(5) + return False + else: + return True + def run(self): + self.initstream() + self.replicate_db() + self.print_time_info() + self.restart_stream() + time.sleep(60) + self.print_time_info() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 26e9ed055f39758490b3a8c2ce82d974bccb7f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 11 Nov 2024 14:58:17 +0800 Subject: [PATCH 03/62] test:add insert test cases --- .../system-test/1-insert/test_multi_insert.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/system-test/1-insert/test_multi_insert.py diff --git a/tests/system-test/1-insert/test_multi_insert.py b/tests/system-test/1-insert/test_multi_insert.py new file mode 100644 index 0000000000..d1b6d28ffd --- /dev/null +++ b/tests/system-test/1-insert/test_multi_insert.py @@ -0,0 +1,32 @@ +from util.sql import * +from util.common import * +import taos +taos.taos_connect +class TDTestCase: + def init(self, conn, logSql, replicaVar = 1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + self.conn = conn + tdSql.init(conn.cursor(), logSql) + def initdb(self): + tdSql.execute("drop database if exists d0") + tdSql.execute("create database d0") + tdSql.execute("use d0") + tdSql.execute("create stable stb0 (ts timestamp, w_ts timestamp, opc nchar(100), quality int) tags(t0 int)") + tdSql.execute("create table t0 using stb0 tags(1)") + tdSql.execute("create table t1 using stb0 tags(2)") + def multi_insert(self): + for i in range(5): + tdSql.execute(f"insert into t1 values(1721265436000, now() + {i + 1}s, '0', 12) t1(opc, quality, ts) values ('opc2', 192, now()+ {i + 2}s) t1(ts, opc, quality) values(now() + {i + 3}s, 'opc4', 10) t1 values(1721265436000, now() + {i + 4}s, '1', 191) t1(opc, quality, ts) values('opc5', 192, now() + {i + 5}s) t1 values(now(), now() + {i + 6}s, '2', 192)") + tdSql.execute("insert into t0 values(1721265436000,now(),'0',192) t0(quality,w_ts,ts) values(192,now(),1721265326000) t0(quality,w_t\ +s,ts) values(190,now()+1s,1721265326000) t0 values(1721265436000,now()+2s,'1',191) t0(quality,w_ts,ts) values(192,now()+3s,\ +1721265326002) t0(ts,w_ts,opc,quality) values(1721265436003,now()+4s,'3',193) t0 values(now(), now() + 4s , '2', 192)") + def run(self): + self.initdb() + self.multi_insert() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From e2f43410dff3b4284a368dfcf19dce7e67c22dda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 11 Nov 2024 14:59:01 +0800 Subject: [PATCH 04/62] test:add stddev test cases --- tests/system-test/2-query/stddev_test.py | 54 ++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/system-test/2-query/stddev_test.py diff --git a/tests/system-test/2-query/stddev_test.py b/tests/system-test/2-query/stddev_test.py new file mode 100644 index 0000000000..c0cb51fe57 --- /dev/null +++ b/tests/system-test/2-query/stddev_test.py @@ -0,0 +1,54 @@ +import numpy as np +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +''' +Test case for TS-5150 +''' +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.ts = 1537146000000 + def initdabase(self): + tdSql.execute('create database if not exists db_test vgroups 2 buffer 10') + tdSql.execute('use db_test') + tdSql.execute('create stable stb(ts timestamp, delay int) tags(groupid int)') + tdSql.execute('create table t1 using stb tags(1)') + tdSql.execute('create table t2 using stb tags(2)') + tdSql.execute('create table t3 using stb tags(3)') + tdSql.execute('create table t4 using stb tags(4)') + tdSql.execute('create table t5 using stb tags(5)') + tdSql.execute('create table t6 using stb tags(6)') + def insert_data(self): + for i in range(5000): + tdSql.execute(f"insert into t1 values({self.ts + i * 1000}, {i%5})") + tdSql.execute(f"insert into t2 values({self.ts + i * 1000}, {i%5})") + tdSql.execute(f"insert into t3 values({self.ts + i * 1000}, {i%5})") + + def verify_stddev(self): + for i in range(20): + tdSql.query(f'SELECT MAX(CASE WHEN delay != 0 THEN delay ELSE NULL END) AS maxDelay,\ + MIN(CASE WHEN delay != 0 THEN delay ELSE NULL END) AS minDelay,\ + AVG(CASE WHEN delay != 0 THEN delay ELSE NULL END) AS avgDelay,\ + STDDEV(CASE WHEN delay != 0 THEN delay ELSE NULL END) AS jitter,\ + COUNT(CASE WHEN delay = 0 THEN 1 ELSE NULL END) AS timeoutCount,\ + COUNT(*) AS totalCount from stb where ts between {1537146000000 + i * 1000} and {1537146000000 + (i+10) * 1000}') + res = tdSql.queryResult[0][3] + assert res > 0.8 + def run(self): + self.initdabase() + self.insert_data() + self.verify_stddev() + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) + From b38411a0767c9ed7095dbeb27d81aa8f975763a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 11 Nov 2024 14:59:36 +0800 Subject: [PATCH 05/62] test:add stream test cases --- tests/pytest/util/sql.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 1d3333264a..46b7e1f795 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -843,9 +843,10 @@ class TDSql: tdSql.query("select * from information_schema.ins_vnodes") #result: dnode_id|vgroup_id|db_name|status|role_time|start_time|restored| + results = list(tdSql.queryResult) for vnode_group_id in db_vgroups_list: - print(tdSql.queryResult) - for result in tdSql.queryResult: + for result in results: + print(f'result[2] is {result[2]}, db_name is {db_name}, result[1] is {result[1]}, vnode_group_id is {vnode_group_id}') if result[2] == db_name and result[1] == vnode_group_id: tdLog.debug(f"dbname: {db_name}, vgroup :{vnode_group_id}, dnode is {result[0]}") print(useful_trans_dnodes_list) From 2dbd0bacb506d0cc166b9813247d6f02a49915f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Mon, 11 Nov 2024 15:00:10 +0800 Subject: [PATCH 06/62] test:add stream and query test cases --- tests/parallel_test/cases.task | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 151358aec3..2b45142156 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -230,6 +230,14 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/agg_group_NotReturnValue.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/td-32548.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stddev_test.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stddev_test.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stddev_test.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stddev_test.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/checkpoint_info.py -N 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/checkpoint_info2.py -N 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_multi_insert.py + ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 -i False ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 -i False ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreMnode.py -N 5 -M 3 -i False From 6c03a8c1ca4a951e3402d9937881c9395ef6c168 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 11 Nov 2024 15:06:17 +0800 Subject: [PATCH 07/62] fix double free --- source/dnode/mgmt/mgmt_snode/src/smWorker.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 8c33c5bb4b..1e882fc656 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -36,14 +36,15 @@ static void smProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO dTrace("msg:%p, get from snode-write queue", pMsg); int32_t code = sndProcessWriteMsg(pMgmt->pSnode, pMsg, NULL); - if (code < 0) { - dGError("snd, msg:%p failed to process write since %s", pMsg, tstrerror(code)); - if (pMsg->info.handle != NULL) { - tmsgSendRsp(pMsg); - } - } else { - smSendRsp(pMsg, 0); - } + // if (code < 0) { + // dGError("snd, msg:%p failed to process write since %s", pMsg, tstrerror(code)); + // if (pMsg->info.handle != NULL) { + // tmsgSendRsp(pMsg); + // } + // } else { + // smSendRsp(pMsg, 0); + // } + smSendRsp(pMsg, code); dTrace("msg:%p, is freed", pMsg); rpcFreeCont(pMsg->pCont); From 3c34ce6e96575fd28b650842e6563b2f3f989236 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 12 Nov 2024 14:08:23 +0800 Subject: [PATCH 08/62] fix/TD-32863-add-lock-for-allocate-primary --- source/dnode/mgmt/mgmt_vnode/inc/vmInt.h | 1 + source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 29 ++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 989adf84ac..f289035689 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -38,6 +38,7 @@ typedef struct SVnodeMgmt { SHashObj *hash; SHashObj *closedHash; TdThreadRwlock lock; + TdThreadMutex mutex; SVnodesStat state; STfs *pTfs; TdThread thread; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 682c179270..9440920866 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -66,6 +66,12 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { if (code != 0) { return code; } + + code = taosThreadMutexLock(&pMgmt->mutex); + if (code != 0) { + goto _OVER; + } + for (int32_t v = 0; v < numOfVnodes; v++) { SVnodeObj *pVnode = ppVnodes[v]; disks[pVnode->diskPrimary] += 1; @@ -81,6 +87,13 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { } } + code = taosThreadMutexUnlock(&pMgmt->mutex); + if (code != 0) { + goto _OVER; + } + +_OVER: + for (int32_t i = 0; i < numOfVnodes; ++i) { if (ppVnodes == NULL || ppVnodes[i] == NULL) continue; vmReleaseVnode(pMgmt, ppVnodes[i]); @@ -89,8 +102,13 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { taosMemoryFree(ppVnodes); } - dInfo("vgId:%d, alloc disk:%d of level 0. ndisk:%d, vnodes: %d", vgId, diskId, ndisk, numOfVnodes); - return diskId; + if (code != 0) { + dError("vgId:%d, failed to alloc disk since %s", vgId, tstrerror(code)); + return code; + } else { + dInfo("vgId:%d, alloc disk:%d of level 0. ndisk:%d, vnodes: %d", vgId, diskId, ndisk, numOfVnodes); + return diskId; + } } SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict) { @@ -622,6 +640,7 @@ static void vmCleanup(SVnodeMgmt *pMgmt) { vmStopWorker(pMgmt); vnodeCleanup(); (void)taosThreadRwlockDestroy(&pMgmt->lock); + (void)taosThreadMutexDestroy(&pMgmt->mutex); (void)taosThreadMutexDestroy(&pMgmt->fileLock); taosMemoryFree(pMgmt); } @@ -714,6 +733,12 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { goto _OVER; } + code = taosThreadMutexInit(&pMgmt->mutex, NULL); + if (code != 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _OVER; + } + code = taosThreadMutexInit(&pMgmt->fileLock, NULL); if (code != 0) { code = TAOS_SYSTEM_ERROR(errno); From 6720f850b0a1eb2a9e07c4ef0b47d6723e10b006 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 12 Nov 2024 10:12:30 +0800 Subject: [PATCH 09/62] fix: print multi dataDir for show variables --- source/common/src/tmisce.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index 4df458c2bb..1fd590bf79 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -267,7 +267,11 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { int8_t locked = 0; - TAOS_CHECK_GOTO(blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)), NULL, _exit); + SConfigItem* pDataDirItem = cfgGetItem(pConf, "dataDir"); + size_t exSize = TMAX(taosArrayGetSize(pDataDirItem->array), 1) - 1; + size_t index = 0; + + TAOS_CHECK_GOTO(blockDataEnsureCapacity(pBlock, cfgGetSize(pConf) + exSize), NULL, _exit); TAOS_CHECK_GOTO(cfgCreateIter(pConf, &pIter), NULL, _exit); @@ -275,6 +279,7 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { locked = 1; while ((pItem = cfgNextIter(pIter)) != NULL) { +_start: col = startCol; // GRANT_CFG_SKIP; @@ -291,7 +296,17 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; - TAOS_CHECK_GOTO(cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen), NULL, _exit); + + if (strcasecmp(pItem->name, "dataDir") == 0 && exSize > 0) { + char* buf = &value[VARSTR_HEADER_SIZE]; + SDiskCfg* pDiskCfg = taosArrayGet(pItem->array, index); + valueLen = snprintf(buf, TSDB_CONFIG_VALUE_LEN, "%s l:%d p:%d d:%" PRIi8, pDiskCfg->dir, pDiskCfg->level, + pDiskCfg->primary, pDiskCfg->disable); + index++; + } else { + TAOS_CHECK_GOTO(cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen), NULL, + _exit); + } varDataSetLen(value, valueLen); pColInfo = taosArrayGet(pBlock->pDataBlock, col++); @@ -314,7 +329,10 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit); numOfRows++; - } + if (index > 0 && index <= exSize) { + goto _start; + } +} pBlock->info.rows = numOfRows; _exit: if (locked) cfgUnLock(pConf); From a3fa03646a99b644482317d4ae16b4e05da0a487 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 12 Nov 2024 15:17:43 +0800 Subject: [PATCH 10/62] fix: print multi dataDir for log --- source/util/src/tconfig.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index d6852b0566..10bebc1907 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -881,11 +881,26 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { case CFG_DTYPE_CHARSET: case CFG_DTYPE_TIMEZONE: case CFG_DTYPE_NONE: + if (strcasecmp(pItem->name, "dataDir") == 0) { + size_t sz = taosArrayGetSize(pItem->array); + if (sz > 1) { + for (size_t j = 0; j < sz; ++j) { + SDiskCfg *pCfg = taosArrayGet(pItem->array, j); + if (dump) { + (void)printf("%s %s %s l:%d p:%d d:%"PRIi8"\n", src, name, pCfg->dir, pCfg->level, pCfg->primary, pCfg->disable); + } else { + uInfo("%s %s %s l:%d p:%d d:%"PRIi8, src, name, pCfg->dir, pCfg->level, pCfg->primary, pCfg->disable); + } + } + break; + } + } if (dump) { (void)printf("%s %s %s\n", src, name, pItem->str); } else { uInfo("%s %s %s", src, name, pItem->str); } + break; } } From 5f5e000160035005d094fb8031ebafdedb8f7f11 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 12 Nov 2024 16:37:14 +0800 Subject: [PATCH 11/62] fix: show local issue --- source/common/src/tmisce.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index 1fd590bf79..0ac59ef4a4 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -267,9 +267,12 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { int8_t locked = 0; - SConfigItem* pDataDirItem = cfgGetItem(pConf, "dataDir"); - size_t exSize = TMAX(taosArrayGetSize(pDataDirItem->array), 1) - 1; + size_t exSize = 0; size_t index = 0; + SConfigItem* pDataDirItem = cfgGetItem(pConf, "dataDir"); + if (pDataDirItem) { + exSize = TMAX(taosArrayGetSize(pDataDirItem->array), 1) - 1; + } TAOS_CHECK_GOTO(blockDataEnsureCapacity(pBlock, cfgGetSize(pConf) + exSize), NULL, _exit); From b141e0e2ba20791c9e14c30db83559d954013081 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 13 Nov 2024 12:37:09 +0800 Subject: [PATCH 12/62] fix/TD-32863-add-creating-hash --- source/dnode/mgmt/mgmt_vnode/inc/vmInt.h | 3 + source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 48 ++++++++++++++ source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 + source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 70 +++++++++++++++++++-- 4 files changed, 118 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index f289035689..7842077d88 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -37,6 +37,7 @@ typedef struct SVnodeMgmt { SSingleWorker mgmtMultiWorker; SHashObj *hash; SHashObj *closedHash; + SHashObj *creatingHash; TdThreadRwlock lock; TdThreadMutex mutex; SVnodesStat state; @@ -97,6 +98,7 @@ SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict); void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl); void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal, bool keepClosed); +void vmRemoveFromCreatingHash(SVnodeMgmt *pMgmt, int32_t vgId); // vmHandle.c SArray *vmGetMsgHandles(); @@ -114,6 +116,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt); int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes); int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes); +int32_t vmGetAllVnodeListFromHashWithCreating(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes); // vmWorker.c int32_t vmStartWorker(SVnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index 7566b69c02..b4453ad6fc 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -67,6 +67,54 @@ int32_t vmGetAllVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnod return 0; } +int32_t vmGetAllVnodeListFromHashWithCreating(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) { + (void)taosThreadRwlockRdlock(&pMgmt->lock); + + int32_t num = 0; + int32_t size = taosHashGetSize(pMgmt->hash); + int32_t creatingSize = taosHashGetSize(pMgmt->creatingHash); + size += creatingSize; + SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *)); + if (pVnodes == NULL) { + (void)taosThreadRwlockUnlock(&pMgmt->lock); + return terrno; + } + + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + SVnodeObj *pVnode = *ppVnode; + if (pVnode && num < size) { + int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + dTrace("vgId:%d,acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); + pVnodes[num++] = (*ppVnode); + pIter = taosHashIterate(pMgmt->hash, pIter); + } else { + taosHashCancelIterate(pMgmt->hash, pIter); + } + } + + pIter = taosHashIterate(pMgmt->creatingHash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + SVnodeObj *pVnode = *ppVnode; + if (pVnode && num < size) { + int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + dTrace("vgId:%d, acquire vnode, vnode:%p, ref:%d", pVnode->vgId, pVnode, refCount); + pVnodes[num++] = (*ppVnode); + pIter = taosHashIterate(pMgmt->creatingHash, pIter); + } else { + taosHashCancelIterate(pMgmt->creatingHash, pIter); + } + } + (void)taosThreadRwlockUnlock(&pMgmt->lock); + + *numOfVnodes = num; + *ppVnodes = pVnodes; + + return 0; +} + int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) { (void)taosThreadRwlockRdlock(&pMgmt->lock); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 006f44b349..c9512d06d4 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -421,6 +421,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { goto _OVER; } + vmRemoveFromCreatingHash(pMgmt, req.vgId); + _OVER: if (code != 0) { int32_t r = 0; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 9440920866..26ac901f58 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -62,7 +62,7 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { int32_t numOfVnodes = 0; SVnodeObj **ppVnodes = NULL; - code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes); + code = vmGetAllVnodeListFromHashWithCreating(pMgmt, &numOfVnodes, &ppVnodes); if (code != 0) { return code; } @@ -92,6 +92,25 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { goto _OVER; } + (void)taosThreadRwlockWrlock(&pMgmt->lock); + + SVnodeObj *pCreatingVnode = taosMemoryCalloc(1, sizeof(SVnodeObj)); + if (pCreatingVnode == NULL) { + (void)taosThreadRwlockUnlock(&pMgmt->lock); + dError("failed to alloc vnode since %s", terrstr()); + goto _OVER; + } + (void)memset(pCreatingVnode, 0, sizeof(SVnodeObj)); + + pCreatingVnode->vgId = vgId; + pCreatingVnode->diskPrimary = diskId; + + int32_t r = taosHashPut(pMgmt->creatingHash, &vgId, sizeof(int32_t), &pCreatingVnode, sizeof(SVnodeObj *)); + if (r != 0) { + dError("vgId:%d, failed to put vnode to creatingHash", vgId); + } + (void)taosThreadRwlockUnlock(&pMgmt->lock); + _OVER: for (int32_t i = 0; i < numOfVnodes; ++i) { @@ -234,12 +253,12 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal, } if (keepClosed) { SVnodeObj *pClosedVnode = taosMemoryCalloc(1, sizeof(SVnodeObj)); - (void)memset(pClosedVnode, 0, sizeof(SVnodeObj)); - if (pVnode == NULL) { - dError("vgId:%d, failed to alloc vnode since %s", pVnode->vgId, terrstr()); + if (pClosedVnode == NULL) { + dError("failed to alloc vnode since %s", terrstr()); (void)taosThreadRwlockUnlock(&pMgmt->lock); return; } + (void)memset(pClosedVnode, 0, sizeof(SVnodeObj)); pClosedVnode->vgId = pVnode->vgId; pClosedVnode->dropped = pVnode->dropped; @@ -445,11 +464,18 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { pMgmt->closedHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (pMgmt->hash == NULL) { + if (pMgmt->closedHash == NULL) { dError("failed to init vnode closed hash since %s", terrstr()); return TSDB_CODE_OUT_OF_MEMORY; } + pMgmt->creatingHash = + taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (pMgmt->creatingHash == NULL) { + dError("failed to init vnode creatingHash hash since %s", terrstr()); + return TSDB_CODE_OUT_OF_MEMORY; + } + SWrapperCfg *pCfgs = NULL; int32_t numOfVnodes = 0; if (vmGetVnodeListFromFile(pMgmt, &pCfgs, &numOfVnodes) != 0) { @@ -527,6 +553,28 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { return 0; } +void vmRemoveFromCreatingHash(SVnodeMgmt *pMgmt, int32_t vgId) { + (void)taosThreadRwlockWrlock(&pMgmt->lock); + SVnodeObj *pOld = NULL; + int32_t r = taosHashGetDup(pMgmt->closedHash, &vgId, sizeof(int32_t), (void *)&pOld); + if (r != 0) { + dError("vgId:%d, failed to get vnode from closedHash", vgId); + } + if (pOld) { + vmFreeVnodeObj(&pOld); + } + r = taosHashRemove(pMgmt->creatingHash, &vgId, sizeof(int32_t)); + if (r != 0) { + dError("vgId:%d, failed to remove vnode from hash", vgId); + } + (void)taosThreadRwlockUnlock(&pMgmt->lock); + +_OVER: + if (r != 0) { + dError("vgId:%d, failed to remove vnode from creatingHash since %s", vgId, tstrerror(r)); + } +} + static void *vmCloseVnodeInThread(void *param) { SVnodeThread *pThread = param; SVnodeMgmt *pMgmt = pThread->pMgmt; @@ -632,6 +680,18 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) { pMgmt->closedHash = NULL; } + pIter = taosHashIterate(pMgmt->creatingHash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + vmFreeVnodeObj(ppVnode); + pIter = taosHashIterate(pMgmt->creatingHash, pIter); + } + + if (pMgmt->creatingHash != NULL) { + taosHashCleanup(pMgmt->creatingHash); + pMgmt->creatingHash = NULL; + } + dInfo("total vnodes:%d are all closed", numOfVnodes); } From 67a547f65a1acaa5637d74dc3037181b37e6edad Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 13 Nov 2024 13:46:23 +0800 Subject: [PATCH 13/62] fix(stream): alwasy return success for resume req. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index f31dd28847..326e8d4ada 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -1186,10 +1186,12 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m streamMutexUnlock(&pHTask->lock); code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode); + tqDebug("s-task:%s resume complete, code:%s", pHTask->id.idStr, tstrerror(code)); + streamMetaReleaseTask(pMeta, pHTask); } - return code; + return TSDB_CODE_SUCCESS; } int32_t tqStreamTasksGetTotalNum(SStreamMeta* pMeta) { return taosArrayGetSize(pMeta->pTaskList); } From 156421f8a03aa36d35ca98c6b9ca1d5a767851f1 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 13 Nov 2024 13:53:55 +0800 Subject: [PATCH 14/62] fix/TD-32863-add-creating-hash-fix-case --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index c9512d06d4..90b3f0025d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -381,6 +381,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { if (vnodeCreate(path, &vnodeCfg, diskPrimary, pMgmt->pTfs) < 0) { dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr()); vmReleaseVnode(pMgmt, pVnode); + vmRemoveFromCreatingHash(pMgmt, req.vgId); (void)tFreeSCreateVnodeReq(&req); code = terrno != 0 ? terrno : -1; return code; @@ -421,9 +422,9 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { goto _OVER; } +_OVER: vmRemoveFromCreatingHash(pMgmt, req.vgId); -_OVER: if (code != 0) { int32_t r = 0; r = taosThreadRwlockWrlock(&pMgmt->lock); From 2b069db01fe2ff08e083b9eda0d9e4b17d717e6f Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 13 Nov 2024 14:26:44 +0800 Subject: [PATCH 15/62] fix/TD-32863-add-creating-hash-add-testcase --- tests/parallel_test/cases.task | 1 + .../0-others/multilevel_createdb.py | 89 +++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 tests/system-test/0-others/multilevel_createdb.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 67fb784b61..90b5f4fee9 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -351,6 +351,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_privilege_all.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/multilevel.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/multilevel_createdb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/ttl.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/ttlChangeOnWrite.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compress_tsz1.py diff --git a/tests/system-test/0-others/multilevel_createdb.py b/tests/system-test/0-others/multilevel_createdb.py new file mode 100644 index 0000000000..a4fd94f4c1 --- /dev/null +++ b/tests/system-test/0-others/multilevel_createdb.py @@ -0,0 +1,89 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import glob + +def scanFiles(pattern): + res = [] + for f in glob.iglob(pattern): + res += [f] + return res + +def checkFiles(pattern, state): + res = scanFiles(pattern) + tdLog.info(res) + num = len(res) + if num: + if state: + tdLog.info("%s: %d files exist. expect: files exist" % (pattern, num)) + else: + tdLog.exit("%s: %d files exist. expect: files not exist." % (pattern, num)) + else: + if state: + tdLog.exit("%s: %d files exist. expect: files exist" % (pattern, num)) + else: + tdLog.info("%s: %d files exist. expect: files not exist." % (pattern, num)) + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + def basic(self): + tdLog.info("============== basic test ===============") + cfg={ + '/mnt/data1 0 1 0' : 'dataDir', + '/mnt/data2 0 0 0' : 'dataDir', + '/mnt/data3 0 0 0' : 'dataDir', + '/mnt/data4 0 0 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + tdSql.createDir('/mnt/data3') + tdSql.createDir('/mnt/data4') + + tdDnodes.stop(1) + tdDnodes.deploy(1,cfg) + tdDnodes.start(1) + + checkFiles(r'/mnt/data1/*/*',1) + checkFiles(r'/mnt/data2/*/*',0) + + tdSql.execute('create database nws vgroups 20 buffer 3000 stt_trigger 1 wal_level 1 wal_retention_period 0') + + checkFiles(r'/mnt/data1/vnode/*/wal',5) + checkFiles(r'/mnt/data2/vnode/*/wal',5) + checkFiles(r'/mnt/data3/vnode/*/wal',5) + checkFiles(r'/mnt/data4/vnode/*/wal',5) + + def run(self): + self.basic() + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 0b9f15817489d3823c47e8a91e655463e8afb325 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 13 Nov 2024 14:48:57 +0800 Subject: [PATCH 16/62] fix/TD-32863-add-creating-hash-fix-case --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 26ac901f58..d28394e332 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -105,6 +105,7 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { pCreatingVnode->vgId = vgId; pCreatingVnode->diskPrimary = diskId; + dTrace("vgId:%d, put vnode into creating hash, pCreatingVnode:%p", vgId, pCreatingVnode); int32_t r = taosHashPut(pMgmt->creatingHash, &vgId, sizeof(int32_t), &pCreatingVnode, sizeof(SVnodeObj *)); if (r != 0) { dError("vgId:%d, failed to put vnode to creatingHash", vgId); @@ -556,13 +557,15 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { void vmRemoveFromCreatingHash(SVnodeMgmt *pMgmt, int32_t vgId) { (void)taosThreadRwlockWrlock(&pMgmt->lock); SVnodeObj *pOld = NULL; - int32_t r = taosHashGetDup(pMgmt->closedHash, &vgId, sizeof(int32_t), (void *)&pOld); + int32_t r = taosHashGetDup(pMgmt->creatingHash, &vgId, sizeof(int32_t), (void *)&pOld); if (r != 0) { - dError("vgId:%d, failed to get vnode from closedHash", vgId); + dError("vgId:%d, failed to get vnode from creating Hash", vgId); } if (pOld) { + dTrace("vgId:%d, free vnode pOld:%p", vgId, &pOld); vmFreeVnodeObj(&pOld); } + dTrace("vgId:%d, remove from creating Hash", vgId); r = taosHashRemove(pMgmt->creatingHash, &vgId, sizeof(int32_t)); if (r != 0) { dError("vgId:%d, failed to remove vnode from hash", vgId); From 7ff6884779cde6a77e201c5886e022ee1d740b26 Mon Sep 17 00:00:00 2001 From: dmchen Date: Wed, 13 Nov 2024 16:07:45 +0800 Subject: [PATCH 17/62] fix/TD-32863-add-creating-hash-change-lock --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 47 +++++++++++++++++------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index d28394e332..a81208c567 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" #include "libs/function/tudf.h" +#include "osMemory.h" #include "tfs.h" #include "vnd.h" @@ -87,17 +88,15 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { } } - code = taosThreadMutexUnlock(&pMgmt->mutex); - if (code != 0) { - goto _OVER; - } - - (void)taosThreadRwlockWrlock(&pMgmt->lock); - SVnodeObj *pCreatingVnode = taosMemoryCalloc(1, sizeof(SVnodeObj)); if (pCreatingVnode == NULL) { - (void)taosThreadRwlockUnlock(&pMgmt->lock); - dError("failed to alloc vnode since %s", terrstr()); + code = -1; + if (terrno != 0) code = terrno; + dError("failed to alloc vnode since %s", tstrerror(code)); + int32_t r = taosThreadMutexUnlock(&pMgmt->mutex); + if (r != 0) { + dError("vgId:%d, failed to unlock mutex since %s", vgId, tstrerror(r)); + } goto _OVER; } (void)memset(pCreatingVnode, 0, sizeof(SVnodeObj)); @@ -105,12 +104,32 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { pCreatingVnode->vgId = vgId; pCreatingVnode->diskPrimary = diskId; - dTrace("vgId:%d, put vnode into creating hash, pCreatingVnode:%p", vgId, pCreatingVnode); - int32_t r = taosHashPut(pMgmt->creatingHash, &vgId, sizeof(int32_t), &pCreatingVnode, sizeof(SVnodeObj *)); - if (r != 0) { - dError("vgId:%d, failed to put vnode to creatingHash", vgId); + code = taosThreadRwlockWrlock(&pMgmt->lock); + if (code != 0) { + int32_t r = taosThreadMutexUnlock(&pMgmt->mutex); + if (r != 0) { + dError("vgId:%d, failed to unlock mutex since %s", vgId, tstrerror(r)); + } + taosMemoryFree(pCreatingVnode); + goto _OVER; + } + + dTrace("vgId:%d, put vnode into creating hash, pCreatingVnode:%p", vgId, pCreatingVnode); + code = taosHashPut(pMgmt->creatingHash, &vgId, sizeof(int32_t), &pCreatingVnode, sizeof(SVnodeObj *)); + if (code != 0) { + dError("vgId:%d, failed to put vnode to creatingHash", vgId); + taosMemoryFree(pCreatingVnode); + } + + int32_t r = taosThreadRwlockUnlock(&pMgmt->lock); + if (r != 0) { + dError("vgId:%d, failed to unlock since %s", vgId, tstrerror(r)); + } + + code = taosThreadMutexUnlock(&pMgmt->mutex); + if (code != 0) { + goto _OVER; } - (void)taosThreadRwlockUnlock(&pMgmt->lock); _OVER: From 67da7b103660369620069b83bb0b7ffe5fc0b7bf Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 13 Nov 2024 16:51:40 +0800 Subject: [PATCH 18/62] feat: alter systable variablesSchema --- include/common/tmsg.h | 3 ++- include/libs/nodes/cmdnodes.h | 5 +++-- include/util/tdef.h | 10 +++++---- source/client/inc/clientInt.h | 3 ++- source/client/src/clientMsgHandler.c | 15 ++++++++++++-- source/common/src/systable.c | 3 ++- source/common/src/tmisce.c | 28 ++++++++++++++++++++------ source/dnode/mnode/impl/src/mndDnode.c | 1 + source/libs/command/src/command.c | 6 ++++++ source/libs/parser/src/parTranslater.c | 16 +++++++++------ 10 files changed, 67 insertions(+), 23 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a7da778513..70a32cd266 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2187,8 +2187,9 @@ int32_t tSerializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq typedef struct { char name[TSDB_CONFIG_OPTION_LEN + 1]; - char value[TSDB_CONFIG_VALUE_LEN + 1]; + char value[TSDB_CONFIG_PATH_LEN + 1]; char scope[TSDB_CONFIG_SCOPE_LEN + 1]; + char info[TSDB_CONFIG_INFO_LEN + 1]; } SVariablesInfo; typedef struct { diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 514eddbc24..0b617c7ce3 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -42,10 +42,11 @@ extern "C" { #define SHOW_CREATE_VIEW_RESULT_FIELD1_LEN (TSDB_VIEW_FNAME_LEN + 4 + VARSTR_HEADER_SIZE) #define SHOW_CREATE_VIEW_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN + VARSTR_HEADER_SIZE) -#define SHOW_LOCAL_VARIABLES_RESULT_COLS 3 +#define SHOW_LOCAL_VARIABLES_RESULT_COLS 4 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) -#define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) +#define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) +#define SHOW_LOCAL_VARIABLES_RESULT_FIELD4_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) #define COMPACT_DB_RESULT_COLS 3 #define COMPACT_DB_RESULT_FIELD1_LEN 32 diff --git a/include/util/tdef.h b/include/util/tdef.h index ba30e78c59..695b4159fd 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -195,9 +195,9 @@ static const EOperatorType OPERATOR_ARRAY[] = { OP_TYPE_MULTI, OP_TYPE_DIV, OP_TYPE_REM, - + OP_TYPE_MINUS, - + OP_TYPE_BIT_AND, OP_TYPE_BIT_OR, @@ -213,7 +213,7 @@ static const EOperatorType OPERATOR_ARRAY[] = { OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH, - + OP_TYPE_IS_NULL, OP_TYPE_IS_NOT_NULL, OP_TYPE_IS_TRUE, @@ -222,7 +222,7 @@ static const EOperatorType OPERATOR_ARRAY[] = { OP_TYPE_IS_NOT_TRUE, OP_TYPE_IS_NOT_FALSE, OP_TYPE_IS_NOT_UNKNOWN, - //OP_TYPE_COMPARE_MAX_VALUE, + //OP_TYPE_COMPARE_MAX_VALUE, OP_TYPE_JSON_GET_VALUE, OP_TYPE_JSON_CONTAINS, @@ -631,6 +631,8 @@ enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 }; #define TSDB_CONFIG_VALUE_LEN 64 #define TSDB_CONFIG_SCOPE_LEN 8 #define TSDB_CONFIG_NUMBER 16 +#define TSDB_CONFIG_PATH_LEN 128 +#define TSDB_CONFIG_INFO_LEN 64 #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 90505ed25a..da56dcf75e 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -47,10 +47,11 @@ enum { RES_TYPE__TMQ_BATCH_META, }; -#define SHOW_VARIABLES_RESULT_COLS 3 +#define SHOW_VARIABLES_RESULT_COLS 4 #define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) +#define SHOW_VARIABLES_RESULT_FIELD4_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) #define TD_RES_QUERY(res) (*(int8_t*)(res) == RES_TYPE__QUERY) #define TD_RES_TMQ(res) (*(int8_t*)(res) == RES_TYPE__TMQ) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 9a723218ff..e182cd97ee 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -541,6 +541,10 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD3_LEN; TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_VARIABLES_RESULT_FIELD4_LEN; + TSDB_CHECK_NULL(taosArrayPush(pBlock->pDataBlock, &infoData), code, line, END, terrno); + int32_t numOfCfg = taosArrayGetSize(pVars); code = blockDataEnsureCapacity(pBlock, numOfCfg); TSDB_CHECK_CODE(code, line, END); @@ -569,6 +573,13 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { TSDB_CHECK_NULL(pColInfo, code, line, END, terrno); code = colDataSetVal(pColInfo, i, scope, false); TSDB_CHECK_CODE(code, line, END); + + char info[TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(info, pInfo->info, TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + TSDB_CHECK_NULL(pColInfo, code, line, END, terrno); + code = colDataSetVal(pColInfo, i, info, false); + TSDB_CHECK_CODE(code, line, END); } pBlock->info.rows = numOfCfg; @@ -825,7 +836,7 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tscError("failed to post semaphore"); } } - return code; + return code; } __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) { @@ -845,7 +856,7 @@ __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) { case TDMT_MND_SHOW_VARIABLES: return processShowVariablesRsp; case TDMT_MND_COMPACT_DB: - return processCompactDbRsp; + return processCompactDbRsp; default: return genericRspCallback; } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 4993ece7c1..12b789f14e 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -327,8 +327,9 @@ static const SSysDbTableSchema configSchema[] = { static const SSysDbTableSchema variablesSchema[] = { {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "value", .bytes = TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "scope", .bytes = TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "info", .bytes = TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, }; static const SSysDbTableSchema topicSchema[] = { diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index 0ac59ef4a4..dfba2ebf6c 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -297,17 +297,16 @@ _start: TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit); - char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; + char value[TSDB_CONFIG_PATH_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; - + SDiskCfg* pDiskCfg = NULL; if (strcasecmp(pItem->name, "dataDir") == 0 && exSize > 0) { char* buf = &value[VARSTR_HEADER_SIZE]; - SDiskCfg* pDiskCfg = taosArrayGet(pItem->array, index); - valueLen = snprintf(buf, TSDB_CONFIG_VALUE_LEN, "%s l:%d p:%d d:%" PRIi8, pDiskCfg->dir, pDiskCfg->level, - pDiskCfg->primary, pDiskCfg->disable); + pDiskCfg = taosArrayGet(pItem->array, index); + valueLen = tsnprintf(buf, TSDB_CONFIG_PATH_LEN, "%s", pDiskCfg->dir); index++; } else { - TAOS_CHECK_GOTO(cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen), NULL, + TAOS_CHECK_GOTO(cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_PATH_LEN, &valueLen), NULL, _exit); } varDataSetLen(value, valueLen); @@ -331,6 +330,23 @@ _start: } TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit); + char info[TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE] = {0}; + if (strcasecmp(pItem->name, "dataDir") == 0) { + char* buf = &info[VARSTR_HEADER_SIZE]; + valueLen = tsnprintf(buf, TSDB_CONFIG_INFO_LEN, "level %d primary %d disabled %" PRIi8, pDiskCfg->level, + pDiskCfg->primary, pDiskCfg->disable); + } else { + valueLen = 0; + } + varDataSetLen(info, valueLen); + + pColInfo = taosArrayGet(pBlock->pDataBlock, col++); + if (pColInfo == NULL) { + code = terrno; + TAOS_CHECK_GOTO(code, NULL, _exit); + } + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, info, false), NULL, _exit); + numOfRows++; if (index > 0 && index <= exSize) { goto _start; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 24ae8382f9..8931558874 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1104,6 +1104,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { (void)strcpy(info.name, "statusInterval"); (void)snprintf(info.value, TSDB_CONFIG_VALUE_LEN, "%d", tsStatusInterval); (void)strcpy(info.scope, "server"); + // fill info.info if (taosArrayPush(rsp.variables, &info) == NULL) { code = terrno; goto _OVER; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 5afdf87afb..6689b5541b 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -953,6 +953,12 @@ static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) { goto _exit; } + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD4_LEN; + if (taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) { + goto _exit; + } + *pOutput = pBlock; _exit: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 99c03c412c..9bea3491c3 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3311,11 +3311,11 @@ static int32_t selectCommonType(SDataType* commonType, const SDataType* newType) } else { resultType = gDisplyTypes[type2][type1]; } - + if (resultType == -1) { return TSDB_CODE_SCALAR_CONVERT_ERROR; } - + if (commonType->type == newType->type) { commonType->bytes = TMAX(commonType->bytes, newType->bytes); return TSDB_CODE_SUCCESS; @@ -3328,9 +3328,9 @@ static int32_t selectCommonType(SDataType* commonType, const SDataType* newType) } else { commonType->bytes = TMAX(TMAX(commonType->bytes, newType->bytes), TYPE_BYTES[resultType]); } - + commonType->type = resultType; - + return TSDB_CODE_SUCCESS; } @@ -13127,7 +13127,7 @@ static int32_t extractShowCreateViewResultSchema(int32_t* numOfCols, SSchema** p } static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) { - *numOfCols = 3; + *numOfCols = SHOW_LOCAL_VARIABLES_RESULT_COLS; // SHOW_VARIABLES_RESULT_COLS *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); if (NULL == (*pSchema)) { return terrno; @@ -13138,13 +13138,17 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS strcpy((*pSchema)[0].name, "name"); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; - (*pSchema)[1].bytes = TSDB_CONFIG_VALUE_LEN; + (*pSchema)[1].bytes = TSDB_CONFIG_PATH_LEN; strcpy((*pSchema)[1].name, "value"); (*pSchema)[2].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[2].bytes = TSDB_CONFIG_SCOPE_LEN; strcpy((*pSchema)[2].name, "scope"); + (*pSchema)[3].type = TSDB_DATA_TYPE_BINARY; + (*pSchema)[3].bytes = TSDB_CONFIG_INFO_LEN; + strcpy((*pSchema)[3].name, "info"); + return TSDB_CODE_SUCCESS; } From 9a9f463877fb9162c0048c9cd3e1bc0f2a83ec59 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 13 Nov 2024 16:53:56 +0800 Subject: [PATCH 19/62] feat: alter TSDB_CONFIG_PATH_LEN from 128 to 4096 --- include/util/tdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 695b4159fd..4e1fb21838 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -631,7 +631,7 @@ enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 }; #define TSDB_CONFIG_VALUE_LEN 64 #define TSDB_CONFIG_SCOPE_LEN 8 #define TSDB_CONFIG_NUMBER 16 -#define TSDB_CONFIG_PATH_LEN 128 +#define TSDB_CONFIG_PATH_LEN 4096 #define TSDB_CONFIG_INFO_LEN 64 #define QUERY_ID_SIZE 20 From 1c6976ca09af0679b98cb86e5d8c1b81deb9ac24 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 13 Nov 2024 17:07:53 +0800 Subject: [PATCH 20/62] fix eliminate proj caused crash --- source/libs/planner/src/planOptimizer.c | 80 ++++++++++++++++++------- tests/system-test/2-query/union.py | 33 ++++++++++ 2 files changed, 90 insertions(+), 23 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 885faa5461..b179c58ee1 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3491,38 +3491,69 @@ static void eliminateProjPushdownProjIdx(SNodeList* pParentProjects, SNodeList* } } +static int32_t eliminateProjOptFindProjPrefixWithOrderCheck(SProjectLogicNode* pProj, SProjectLogicNode* pChild, SNodeList** pNewChildTargets, bool *orderMatch) { + int32_t code = 0; + SNode* pProjection = NULL, *pChildTarget = NULL; + *orderMatch = true; + FORBOTH(pProjection, pProj->pProjections, pChildTarget, pChild->node.pTargets) { + if (!pProjection) break; + if (0 != strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) { + *orderMatch = false; + break; + } + SNode* pNew = NULL; + code = nodesCloneNode(pChildTarget, &pNew); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(pNewChildTargets, pNew); + } + if (TSDB_CODE_SUCCESS != code && pNewChildTargets) { + nodesDestroyList(*pNewChildTargets); + *pNewChildTargets = NULL; + break; + } + } + return code; +} + +static int32_t eliminateProjOptPushTargetsToSetOpChildren(SProjectLogicNode* pSetOp) { + SNode* pChildProj = NULL; + int32_t code = 0; + bool orderMatch = false; + FOREACH(pChildProj, pSetOp->node.pChildren) { + if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChildProj)) { + SProjectLogicNode* pChildLogic = (SProjectLogicNode*)pChildProj; + SNodeList* pNewChildTargetsForChild = NULL; + code = eliminateProjOptFindProjPrefixWithOrderCheck(pSetOp, pChildLogic, &pNewChildTargetsForChild, &orderMatch); + if (TSDB_CODE_SUCCESS != code) break; + nodesDestroyList(pChildLogic->node.pTargets); + pChildLogic->node.pTargets = pNewChildTargetsForChild; + alignProjectionWithTarget((SLogicNode*)pChildLogic); + if (pChildLogic->isSetOpProj) { + code = eliminateProjOptPushTargetsToSetOpChildren(pChildLogic); + if (TSDB_CODE_SUCCESS != code) break; + } + } + } + return code; +} + static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SProjectLogicNode* pProjectNode) { SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProjectNode->node.pChildren, 0); int32_t code = 0; + bool isSetOpProj = false; + bool orderMatch = false; if (NULL == pProjectNode->node.pParent) { SNodeList* pNewChildTargets = NULL; - code = nodesMakeList(&pNewChildTargets); - if (TSDB_CODE_SUCCESS != code) { - return code; - } SNode * pProjection = NULL, *pChildTarget = NULL; - bool orderMatch = true; - bool needOrderMatch = + isSetOpProj = QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChild) && ((SProjectLogicNode*)pChild)->isSetOpProj; - if (needOrderMatch) { + if (isSetOpProj) { // For sql: select ... from (select ... union all select ...); // When eliminating the outer proj (the outer select), we have to make sure that the outer proj projections and // union all project targets have same columns in the same order. See detail in TD-30188 - FORBOTH(pProjection, pProjectNode->pProjections, pChildTarget, pChild->pTargets) { - if (!pProjection) break; - if (0 != strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) { - orderMatch = false; - break; - } - SNode* pNew = NULL; - code = nodesCloneNode(pChildTarget, &pNew); - if (TSDB_CODE_SUCCESS == code) { - code = nodesListStrictAppend(pNewChildTargets, pNew); - } - if (TSDB_CODE_SUCCESS != code) break; - } + code = eliminateProjOptFindProjPrefixWithOrderCheck(pProjectNode, (SProjectLogicNode*)pChild, &pNewChildTargets, &orderMatch); } else { FOREACH(pProjection, pProjectNode->pProjections) { FOREACH(pChildTarget, pChild->pTargets) { @@ -3530,7 +3561,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* SNode* pNew = NULL; code = nodesCloneNode(pChildTarget, &pNew); if (TSDB_CODE_SUCCESS == code) { - code = nodesListStrictAppend(pNewChildTargets, pNew); + code = nodesListMakeStrictAppend(&pNewChildTargets, pNew); } break; } @@ -3545,8 +3576,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* return code; } - if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets) && - (!needOrderMatch || (needOrderMatch && orderMatch))) { + if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets) && (!isSetOpProj || orderMatch)) { nodesDestroyList(pChild->pTargets); pChild->pTargets = pNewChildTargets; } else { @@ -3574,6 +3604,10 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* nodesDestroyNode((SNode*)pProjectNode); // if pChild is a project logic node, remove its projection which is not reference by its target. alignProjectionWithTarget(pChild); + if (isSetOpProj && orderMatch) { + // Since we have eliminated the outer proj, we need to push down the new targets to the children of the set operation. + code = eliminateProjOptPushTargetsToSetOpChildren((SProjectLogicNode*)pChild); + } } pCxt->optimized = true; return code; diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py index 2aa01f2c02..b74db0104d 100644 --- a/tests/system-test/2-query/union.py +++ b/tests/system-test/2-query/union.py @@ -369,8 +369,41 @@ class TDTestCase: ''' ) + def test_TS_5630(self): + sql = "CREATE DATABASE `ep_iot` BUFFER 256 CACHESIZE 20 CACHEMODEL 'both' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 3 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0" + tdSql.execute(sql, queryTimes=1) + tdLog.info("database ep_iot created") + sql = "CREATE STABLE `ep_iot`.`sldc_dp` (`ts` TIMESTAMP, `data_write_time` TIMESTAMP, `jz1fdgl` DOUBLE, `jz1ssfdfh` DOUBLE, `jz1fdmh` DOUBLE, `jz1gdmh` DOUBLE, `jz1qjrhl` DOUBLE, `jz1zhcydl` DOUBLE, `jz1zkby` DOUBLE, `jz1zzqyl` DOUBLE, `jz1zzqwda` DOUBLE, `jz1zzqwdb` DOUBLE, `jz1zzqll` DOUBLE, `jz1gswd` DOUBLE, `jz1gsll` DOUBLE, `jz1glxl` DOUBLE, `jz1qjrh` DOUBLE, `jz1zhrxl` DOUBLE, `jz1gmjassllfk` DOUBLE, `jz1gmjasslllj` DOUBLE, `jz1gmjbssllfk` DOUBLE, `jz1gmjbsslllj` DOUBLE, `jz1gmjcssllfk` DOUBLE, `jz1gmjcsslllj` DOUBLE, `jz1gmjdssllfk` DOUBLE, `jz1gmjdsslllj` DOUBLE, `jz1gmjessllfk` DOUBLE, `jz1gmjesslllj` DOUBLE, `jz1gmjfssllfk` DOUBLE, `jz1gmjfsslllj` DOUBLE, `jz1zrqwda` DOUBLE, `jz1zrqwdb` DOUBLE, `jz1zrzqyl` DOUBLE, `jz1mmjadl` DOUBLE, `jz1mmjbdl` DOUBLE, `jz1mmjcdl` DOUBLE, `jz1mmjddl` DOUBLE, `jz1mmjedl` DOUBLE, `jz1mmjfdl` DOUBLE, `jz1cyqckwda` DOUBLE, `jz1cyqckwdb` DOUBLE, `jz1njswd` DOUBLE, `jz1nqqxhsckawd` DOUBLE, `jz1nqqxhsckbwd` DOUBLE, `jz1nqqxhsrkawd` DOUBLE, `jz1nqqxhsrkbwd` DOUBLE, `jz1kyqackyqwdsel` DOUBLE, `jz1kyqbckyqwdsel` DOUBLE, `jz1yfjackyqwd` DOUBLE, `jz1yfjbckyqwd` DOUBLE, `jz1trkyqwd` DOUBLE, `jz1trkyqwd1` DOUBLE, `jz1trkyqwd2` DOUBLE, `jz1trkyqwd3` DOUBLE, `jz1tckjyqwd1` DOUBLE, `jz1tckjyqwd2` DOUBLE, `jz1tckyqwd1` DOUBLE, `jz1bya` DOUBLE, `jz1byb` DOUBLE, `jz1pqwda` DOUBLE, `jz1pqwdb` DOUBLE, `jz1gmjadl` DOUBLE, `jz1gmjbdl` DOUBLE, `jz1gmjcdl` DOUBLE, `jz1gmjddl` DOUBLE, `jz1gmjedl` DOUBLE, `jz1gmjfdl` DOUBLE, `jz1yfjadl` DOUBLE, `jz1yfjbdl` DOUBLE, `jz1ycfjadl` DOUBLE, `jz1ycfjbdl` DOUBLE, `jz1sfjadl` DOUBLE, `jz1sfjbdl` DOUBLE, `jz1fdjyggl` DOUBLE, `jz1fdjwggl` DOUBLE, `jz1sjzs` DOUBLE, `jz1zfl` DOUBLE, `jz1ltyl` DOUBLE, `jz1smb` DOUBLE, `jz1rll` DOUBLE, `jz1grd` DOUBLE, `jz1zjwd` DOUBLE, `jz1yl` DOUBLE, `jz1kyqckwd` DOUBLE, `jz1abmfsybrkcy` DOUBLE, `jz1bbmfsybrkcy` DOUBLE, `jz1abjcsdmfytwdzdz` DOUBLE, `jz1bbjcsdmfytwdzdz` DOUBLE, `jz2fdgl` DOUBLE, `jz2ssfdfh` DOUBLE, `jz2fdmh` DOUBLE, `jz2gdmh` DOUBLE, `jz2qjrhl` DOUBLE, `jz2zhcydl` DOUBLE, `jz2zkby` DOUBLE, `jz2zzqyl` DOUBLE, `jz2zzqwda` DOUBLE, `jz2zzqwdb` DOUBLE, `jz2zzqll` DOUBLE, `jz2gswd` DOUBLE, `jz2gsll` DOUBLE, `jz2glxl` DOUBLE, `jz2qjrh` DOUBLE, `jz2zhrxl` DOUBLE, `jz2gmjassllfk` DOUBLE, `jz2gmjasslllj` DOUBLE, `jz2gmjbssllfk` DOUBLE, `jz2gmjbsslllj` DOUBLE, `jz2gmjcssllfk` DOUBLE, `jz2gmjcsslllj` DOUBLE, `jz2gmjdssllfk` DOUBLE, `jz2gmjdsslllj` DOUBLE, `jz2gmjessllfk` DOUBLE, `jz2gmjesslllj` DOUBLE, `jz2gmjfssllfk` DOUBLE, `jz2gmjfsslllj` DOUBLE, `jz2zrqwda` DOUBLE, `jz2zrqwdb` DOUBLE, `jz2zrzqyl` DOUBLE, `jz2mmjadl` DOUBLE, `jz2mmjbdl` DOUBLE, `jz2mmjcdl` DOUBLE, `jz2mmjddl` DOUBLE, `jz2mmjedl` DOUBLE, `jz2mmjfdl` DOUBLE, `jz2cyqckwda` DOUBLE, `jz2cyqckwdb` DOUBLE, `jz2njswd` DOUBLE, `jz2nqqxhsckawd` DOUBLE, `jz2nqqxhsckbwd` DOUBLE, `jz2nqqxhsrkawd` DOUBLE, `jz2nqqxhsrkbwd` DOUBLE, `jz2kyqackyqwdsel` DOUBLE, `jz2kyqbckyqwdsel` DOUBLE, `jz2yfjackyqwd` DOUBLE, `jz2yfjbckyqwd` DOUBLE, `jz2trkyqwd` DOUBLE, `jz2trkyqwd1` DOUBLE, `jz2trkyqwd2` DOUBLE, `jz2trkyqwd3` DOUBLE, `jz2tckjyqwd1` DOUBLE, `jz2tckjyqwd2` DOUBLE, `jz2tckyqwd1` DOUBLE, `jz2bya` DOUBLE, `jz2byb` DOUBLE, `jz2pqwda` DOUBLE, `jz2pqwdb` DOUBLE, `jz2gmjadl` DOUBLE, `jz2gmjbdl` DOUBLE, `jz2gmjcdl` DOUBLE, `jz2gmjddl` DOUBLE, `jz2gmjedl` DOUBLE, `jz2gmjfdl` DOUBLE, `jz2yfjadl` DOUBLE, `jz2yfjbdl` DOUBLE, `jz2ycfjadl` DOUBLE, `jz2ycfjbdl` DOUBLE, `jz2sfjadl` DOUBLE, `jz2sfjbdl` DOUBLE, `jz2fdjyggl` DOUBLE, `jz2fdjwggl` DOUBLE, `jz2sjzs` DOUBLE, `jz2zfl` DOUBLE, `jz2ltyl` DOUBLE, `jz2smb` DOUBLE, `jz2rll` DOUBLE, `jz2grd` DOUBLE, `jz2zjwd` DOUBLE, `jz2yl` DOUBLE, `jz2kyqckwd` DOUBLE, `jz2abmfsybrkcy` DOUBLE, `jz2bbmfsybrkcy` DOUBLE, `jz2abjcsdmfytwdzdz` DOUBLE, `jz2bbjcsdmfytwdzdz` DOUBLE) TAGS (`iot_hub_id` VARCHAR(100), `device_group_code` VARCHAR(100), `device_code` VARCHAR(100))" + tdLog.info("stable ep_iot.sldc_dp created") + tdSql.execute(sql, queryTimes=1) + sql = "insert into ep_iot.sldc_dp_t1 using ep_iot.sldc_dp tags('a','a','a') values(now, now, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9,0,1);" + tdSql.execute(sql, queryTimes=1) + sql = "insert into ep_iot.sldc_dp_t1 using ep_iot.sldc_dp tags('b','b','b') values(now, now, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9,0,1);" + tdSql.execute(sql, queryTimes=1) + sql = "insert into ep_iot.sldc_dp_t1 using ep_iot.sldc_dp tags('c','c','c') values(now, now, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9,0,1);" + tdSql.execute(sql, queryTimes=1) + sql = "insert into ep_iot.sldc_dp_t1 using ep_iot.sldc_dp tags('d','d','d') values(now, now, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9,0,1);" + tdSql.execute(sql, queryTimes=1) + sql = "insert into ep_iot.sldc_dp_t1 using ep_iot.sldc_dp tags('e','e','e') values(now, now, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9, 0,1,2,3,4,5,6,7,8,9,0,1);" + tdSql.execute(sql, queryTimes=1) + sql = "select scdw_code, scdw_name, jzmc, fdgl, jzzt from ((select '01072016' as scdw_code, '盛鲁电厂' as scdw_name, '机组1' as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '01072016' as scdw_code, '盛鲁电厂' as scdw_name, '机组2' as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '00103673' as scdw_code, '鲁西电厂' as scdw_name, '机组1'as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '00103673' as scdw_code, '鲁西电厂' as scdw_name, '机组2'as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '01061584' as scdw_code, '富源热电' as scdw_name, '机组1'as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt ,last(ts) as ts from ep_iot.sldc_dp) union all ( select '01061584' as scdw_code, '富源热电' as scdw_name, '机组2'as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt ,last(ts) as ts from ep_iot.sldc_dp)) where scdw_code like '%%';" + tdSql.query(sql, queryTimes=1) + tdSql.checkCols(5) + tdSql.checkRows(6) + + sql = "select scdw_name, scdw_code, jzmc, fdgl, jzzt from ((select '01072016' as scdw_code, '盛鲁电厂' as scdw_name, '机组1' as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '01072016' as scdw_code, '盛鲁电厂' as scdw_name, '机组2' as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '00103673' as scdw_code, '鲁西电厂' as scdw_name, '机组1'as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '00103673' as scdw_code, '鲁西电厂' as scdw_name, '机组2'as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '01061584' as scdw_code, '富源热电' as scdw_name, '机组1'as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt ,last(ts) as ts from ep_iot.sldc_dp) union all ( select '01061584' as scdw_code, '富源热电' as scdw_name, '机组2'as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt ,last(ts) as ts from ep_iot.sldc_dp)) where scdw_code like '%%';" + tdSql.query(sql, queryTimes=1) + tdSql.checkCols(5) + tdSql.checkRows(6) + sql = "select scdw_name, scdw_code, jzzt from ((select '01072016' as scdw_code, '盛鲁电厂' as scdw_name, '机组1' as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '01072016' as scdw_code, '盛鲁电厂' as scdw_name, '机组2' as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '00103673' as scdw_code, '鲁西电厂' as scdw_name, '机组1'as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '00103673' as scdw_code, '鲁西电厂' as scdw_name, '机组2'as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '01061584' as scdw_code, '富源热电' as scdw_name, '机组1'as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt ,last(ts) as ts from ep_iot.sldc_dp) union all ( select '01061584' as scdw_code, '富源热电' as scdw_name, '机组2'as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt ,last(ts) as ts from ep_iot.sldc_dp)) where scdw_code like '%%';" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(6) + tdSql.checkCols(3) + tdSql.execute("drop database ep_iot") + def run(self): tdSql.prepare() + self.test_TS_5630() tdLog.printNoPrefix("==========step1:create table") self.__create_tb() From 1c79d342cc587d62cc037a35e57fb865673ffac8 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 13 Nov 2024 17:59:29 +0800 Subject: [PATCH 21/62] fix: encode SVariablesInfo --- source/common/src/tmsg.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index edf0db9954..e083176544 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5618,6 +5618,7 @@ int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) { TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->name)); TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->value)); TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->scope)); + TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->info)); return 0; } @@ -5625,6 +5626,7 @@ int32_t tDecodeSVariablesInfo(SDecoder *pDecoder, SVariablesInfo *pInfo) { TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->name)); TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->value)); TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->scope)); + TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->info)); return 0; } From 75c5209c8dea9ece4305407592cb446c43f6a064 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 13 Nov 2024 18:27:09 +0800 Subject: [PATCH 22/62] fix: SVariablesInfo compatibility --- source/common/src/tmsg.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index e083176544..bc8830505e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5618,7 +5618,6 @@ int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) { TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->name)); TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->value)); TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->scope)); - TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pInfo->info)); return 0; } @@ -5626,7 +5625,6 @@ int32_t tDecodeSVariablesInfo(SDecoder *pDecoder, SVariablesInfo *pInfo) { TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->name)); TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->value)); TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->scope)); - TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pInfo->info)); return 0; } @@ -5644,6 +5642,12 @@ int32_t tSerializeSShowVariablesRsp(void *buf, int32_t bufLen, SShowVariablesRsp SVariablesInfo *pInfo = taosArrayGet(pRsp->variables, i); TAOS_CHECK_EXIT(tEncodeSVariablesInfo(&encoder, pInfo)); } + + for (int32_t i = 0; i < varNum; ++i) { + SVariablesInfo *pInfo = taosArrayGet(pRsp->variables, i); + TAOS_CHECK_RETURN(tEncodeCStr(&encoder, pInfo->info)); + } + tEndEncode(&encoder); _exit: @@ -5677,6 +5681,13 @@ int32_t tDeserializeSShowVariablesRsp(void *buf, int32_t bufLen, SShowVariablesR TAOS_CHECK_EXIT(terrno); } } + + if (!tDecodeIsEnd(&decoder)) { + for (int32_t i = 0; i < varNum; ++i) { + SVariablesInfo *pInfo = taosArrayGet(pRsp->variables, i); + TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pInfo->info)); + } + } } tEndDecode(&decoder); From ea7c30505a8c1394a8c315defdf08b84f5eb011f Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 13 Nov 2024 19:37:22 +0800 Subject: [PATCH 23/62] fix(stream):fix bloom filter decode issue --- source/libs/executor/src/scanoperator.c | 28 +++++++++++++------------ 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 5b5d5c5d11..ecb6bd7aee 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3468,11 +3468,6 @@ void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) goto _end; } - void* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); - if (!pUpInfo) { - lino = __LINE__; - goto _end; - } SDecoder decoder = {0}; pDeCoder = &decoder; tDecoderInit(pDeCoder, buf, tlen); @@ -3481,14 +3476,21 @@ void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) goto _end; } - code = pInfo->stateStore.updateInfoDeserialize(pDeCoder, pUpInfo); - if (code == TSDB_CODE_SUCCESS) { - pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo); - pInfo->pUpdateInfo = pUpInfo; - } else { - taosMemoryFree(pUpInfo); - lino = __LINE__; - goto _end; + if (pInfo->pUpdateInfo != NULL) { + void* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo)); + if (!pUpInfo) { + lino = __LINE__; + goto _end; + } + code = pInfo->stateStore.updateInfoDeserialize(pDeCoder, pUpInfo); + if (code == TSDB_CODE_SUCCESS) { + pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo); + pInfo->pUpdateInfo = pUpInfo; + } else { + taosMemoryFree(pUpInfo); + lino = __LINE__; + goto _end; + } } if (tDecodeIsEnd(pDeCoder)) { From b64e28e6798a28778195b3b2b1799ad66fef3966 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 14 Nov 2024 09:19:58 +0800 Subject: [PATCH 24/62] docs: minor change --- .../zh/14-reference/01-components/01-taosd.md | 350 +++++++++--------- .../zh/14-reference/01-components/02-taosc.md | 141 ++++--- 2 files changed, 245 insertions(+), 246 deletions(-) diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index dc35d40b07..64ae69528b 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -27,65 +27,65 @@ taosd 命令行参数如下 ### 连接相关 |参数名称|支持版本|参数含义| -|-----------------------|-----------|-| -|firstEp | |taosd 启动时，主动连接的集群中首个 dnode 的 end point，默认值 localhost:6030| -|secondEp | |taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，无默认值| -|fqdn | |taosd 监听的服务地址，默认为所在服务器上配置的第一个 hostname| -|serverPort | |taosd 监听的端口，默认值 6030| -|compressMsgSize | |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；默认值 -1| -|shellActivityTimer | |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3| -|numOfRpcSessions | |RPC 支持的最大连接数，取值范围 100-100000，默认值 30000| -|numOfRpcThreads | |RPC 线程数目，默认值为 CPU 核数的一半| -|numOfTaskQueueThreads | |dnode 处理 RPC 消息的线程数| -|statusInterval | |dnode 与 mnode 之间的心跳间隔| -|rpcQueueMemoryAllowed | |dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值 服务器内存的 1/10 | -|resolveFQDNRetryTime | |FQDN 解析失败时的重试次数| -|timeToGetAvailableConn | |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，默认值 500000| -|maxShellConns | |允许创建的最大链接数| -|maxRetryWaitTime | |重连最大超时时间| -|shareConnLimit |3.3.4.3 之后|内部参数，一个链接可以共享的查询数目，取值范围 1-256，默认值 10| -|readTimeout |3.3.4.3 之后|内部参数，最小超时时间，取值范围 64-604800，单位为秒，默认值 900| +|-----------------------|----------|-| +|firstEp | |taosd 启动时，主动连接的集群中首个 dnode 的 end point，默认值 localhost:6030| +|secondEp | |taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，无默认值| +|fqdn | |taosd 监听的服务地址，默认为所在服务器上配置的第一个 hostname| +|serverPort | |taosd 监听的端口，默认值 6030| +|compressMsgSize | |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；默认值 -1| +|shellActivityTimer | |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3| +|numOfRpcSessions | |RPC 支持的最大连接数，取值范围 100-100000，默认值 30000| +|numOfRpcThreads | |RPC 线程数目，默认值为 CPU 核数的一半| +|numOfTaskQueueThreads | |dnode 处理 RPC 消息的线程数| +|statusInterval | |dnode 与 mnode 之间的心跳间隔| +|rpcQueueMemoryAllowed | |dnode 允许的 rpc 消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值 服务器内存的 1/10 | +|resolveFQDNRetryTime | |FQDN 解析失败时的重试次数| +|timeToGetAvailableConn | |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，默认值 500000| +|maxShellConns | |允许创建的最大链接数| +|maxRetryWaitTime | |重连最大超时时间| +|shareConnLimit |3.3.4.3 后|内部参数，一个链接可以共享的查询数目，取值范围 1-256，默认值 10| +|readTimeout |3.3.4.3 后|内部参数，最小超时时间，取值范围 64-604800，单位为秒，默认值 900| ### 监控相关 |参数名称|支持版本|参数含义| -|-----------------------|-----------|-| -|monitor | |是否收集监控数据并上报，0：关闭；1:打开；默认值 0| -|monitorFqdn | |taosKeeper 服务所在服务器的 FQDN，默认值 无| -|monitorPort | |taosKeeper 服务所监听的端口号，默认值 6043| -|monitorInterval | |监控数据库记录系统参数（CPU/内存）的时间间隔，单位是秒，取值范围 1-200000 ，默认值 30| -|monitorMaxLogs | |缓存的待上报日志条数| -|monitorComp | |是否采用压缩方式上报监控日志时| -|monitorLogProtocol | |是否打印监控日志| -|monitorForceV2 | |是否使用 V2 版本协议上报| -|telemetryReporting | |是否上传 telemetry，0：不上传，1：上传，默认值 1| -|telemetryServer | |telemetry 服务器地址| -|telemetryPort | |telemetry 服务器端口编号| -|telemetryInterval | |telemetry 上传时间间隔，单位为秒，默认 43200| -|crashReporting | |是否上传 crash 信息；0：不上传，1：上传；默认值 1| +|-----------------------|----------|-| +|monitor | |是否收集监控数据并上报，0：关闭；1:打开；默认值 0| +|monitorFqdn | |taosKeeper 服务所在服务器的 FQDN，默认值 无| +|monitorPort | |taosKeeper 服务所监听的端口号，默认值 6043| +|monitorInterval | |监控数据库记录系统参数（CPU/内存）的时间间隔，单位是秒，取值范围 1-200000 ，默认值 30| +|monitorMaxLogs | |缓存的待上报日志条数| +|monitorComp | |是否采用压缩方式上报监控日志时| +|monitorLogProtocol | |是否打印监控日志| +|monitorForceV2 | |是否使用 V2 版本协议上报| +|telemetryReporting | |是否上传 telemetry，0：不上传，1：上传，默认值 1| +|telemetryServer | |telemetry 服务器地址| +|telemetryPort | |telemetry 服务器端口编号| +|telemetryInterval | |telemetry 上传时间间隔，单位为秒，默认 43200| +|crashReporting | |是否上传 crash 信息；0：不上传，1：上传；默认值 1| ### 查询相关 |参数名称|支持版本|参数含义| -|------------------------|-----------|-| -|countAlwaysReturnValue | |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| -|tagFilterCache | |是否缓存标签过滤结果| -|maxNumOfDistinctRes | |允许返回的 distinct 结果最大行数，默认值 10 万，最大允许值 1 亿| -|queryBufferSize | |暂不生效| -|queryRspPolicy | |查询响应策略| -|filterScalarMode | |强制使用标量过滤模式，0：关闭；1：开启，默认值 0| -|queryPlannerTrace | |内部参数，查询计划是否输出详细日志| -|queryNodeChunkSize | |内部参数，查询计划的块大小| -|queryUseNodeAllocator | |内部参数，查询计划的分配方法| -|queryMaxConcurrentTables| |内部参数，查询计划的并发数目| -|queryRsmaTolerance | |内部参数，用于判定查询哪一级 rsma 数据时的容忍时间，单位为毫秒| -|enableQueryHb | |内部参数，是否发送查询心跳消息| -|pqSortMemThreshold | |内部参数，排序使用的内存阈值| +|------------------------|----------|-| +|countAlwaysReturnValue | |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| +|tagFilterCache | |是否缓存标签过滤结果| +|maxNumOfDistinctRes | |允许返回的 distinct 结果最大行数，默认值 10 万，最大允许值 1 亿| +|queryBufferSize | |暂不生效| +|queryRspPolicy | |查询响应策略| +|filterScalarMode | |强制使用标量过滤模式，0：关闭；1：开启，默认值 0| +|queryPlannerTrace | |内部参数，查询计划是否输出详细日志| +|queryNodeChunkSize | |内部参数，查询计划的块大小| +|queryUseNodeAllocator | |内部参数，查询计划的分配方法| +|queryMaxConcurrentTables| |内部参数，查询计划的并发数目| +|queryRsmaTolerance | |内部参数，用于判定查询哪一级 rsma 数据时的容忍时间，单位为毫秒| +|enableQueryHb | |内部参数，是否发送查询心跳消息| +|pqSortMemThreshold | |内部参数，排序使用的内存阈值| ### 区域相关 |参数名称|支持版本|参数含义| -|-----------------|-----------|-| -|timezone | |时区；缺省从系统中动态获取当前的时区设置| -|locale | |系统区位信息及编码格式，缺省从系统中获取| -|charset | |字符集编码，缺省从系统中获取| +|-----------------|----------|-| +|timezone | |时区；缺省从系统中动态获取当前的时区设置| +|locale | |系统区位信息及编码格式，缺省从系统中获取| +|charset | |字符集编码，缺省从系统中获取| :::info 1. 为应对多时区的数据写入和查询问题，TDengine 采用 Unix 时间戳(Unix Timestamp)来记录和存储时间戳。Unix 时间戳的特点决定了任一时刻不论在任何时区，产生的时间戳均一致。需要注意的是，Unix 时间戳是在客户端完成转换和记录。为了确保客户端其他形式的时间转换为正确的 Unix 时间戳，需要设置正确的时区。 @@ -164,147 +164,147 @@ charset 的有效值是 UTF-8。 ### 存储相关 |参数名称|支持版本|参数含义| -|--------------------|-----------|-| -|dataDir | |数据文件目录，所有的数据文件都将写入该目录，默认值 /var/lib/taos| -|tempDir | |指定所有系统运行过程中的临时文件生成的目录，默认值 /tmp| -|minimalDataDirGB | |dataDir 指定的时序数据存储目录所需要保留的最小空间，单位 GB，默认值 2| -|minimalTmpDirGB | |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，默认值 1| -|minDiskFreeSize |3.1.1.0 之后|当某块磁盘上的可用空间小于等于这个阈值时，该磁盘将不再被选择用于生成新的数据文件，单位为字节，取值范围 52428800-1073741824，默认值为 52428800；企业版参数| -|s3MigrateIntervalSec|3.3.4.3 之后|本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600；企业版参数| -|s3MigrateEnabled |3.3.4.3 之后|是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1；企业版参数| -|s3Accesskey |3.3.4.3 之后|冒号分隔的用户 SecretId:SecretKey，例如 AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E；企业版参数| -|s3Endpoint |3.3.4.3 之后|用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 保持一致，否则无法访问；企业版参数| -|s3BucketName |3.3.4.3 之后|存储桶名称，减号后面是用户注册 COS 服务的 AppId，其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔；参数值均为字符串类型，但不需要引号；例如 test0711-1309024725；企业版参数| -|s3PageCacheSize |3.3.4.3 之后|S3 page cache 缓存页数目，取值范围 4-1048576，单位为页，默认值 4096；企业版参数| -|s3UploadDelaySec |3.3.4.3 之后|data 文件持续多长时间不再变动后上传至 S3，取值范围 1-2592000 (30天），单位为秒，默认值 60；企业版参数| -|cacheLazyLoadThreshold | |内部参数，缓存的装载策略| +|--------------------|----------|-| +|dataDir | |数据文件目录，所有的数据文件都将写入该目录，默认值 /var/lib/taos| +|tempDir | |指定所有系统运行过程中的临时文件生成的目录，默认值 /tmp| +|minimalDataDirGB | |dataDir 指定的时序数据存储目录所需要保留的最小空间，单位 GB，默认值 2| +|minimalTmpDirGB | |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，默认值 1| +|minDiskFreeSize |3.1.1.0 后|当某块磁盘上的可用空间小于等于这个阈值时，该磁盘将不再被选择用于生成新的数据文件，单位为字节，取值范围 52428800-1073741824，默认值为 52428800；企业版参数| +|s3MigrateIntervalSec|3.3.4.3 后|本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600；企业版参数| +|s3MigrateEnabled |3.3.4.3 后|是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1；企业版参数| +|s3Accesskey |3.3.4.3 后|冒号分隔的用户 SecretId:SecretKey，例如 AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E；企业版参数| +|s3Endpoint |3.3.4.3 后|用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 保持一致，否则无法访问；企业版参数| +|s3BucketName |3.3.4.3 后|存储桶名称，减号后面是用户注册 COS 服务的 AppId，其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔；参数值均为字符串类型，但不需要引号；例如 test0711-1309024725；企业版参数| +|s3PageCacheSize |3.3.4.3 后|S3 page cache 缓存页数目，取值范围 4-1048576，单位为页，默认值 4096；企业版参数| +|s3UploadDelaySec |3.3.4.3 后|data 文件持续多长时间不再变动后上传至 S3，取值范围 1-2592000 (30天），单位为秒，默认值 60；企业版参数| +|cacheLazyLoadThreshold| |内部参数，缓存的装载策略| ### 集群相关 |参数名称|支持版本|参数含义| -|--------------------------|-----------|-| -|supportVnodes | |dnode 支持的最大 vnode 数目，取值范围 0-4096，默认值 CPU 核数的 2 倍 + 5| -|numOfCommitThreads | |落盘线程的最大数量，取值范围 0-1024，默认值为 4| -|numOfMnodeReadThreads | |mnode 的 Read 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| -|numOfVnodeQueryThreads | |vnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| -|numOfVnodeFetchThreads | |vnode 的 Fetch 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| -|numOfVnodeRsmaThreads | |vnode 的 Rsma 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| -|numOfQnodeQueryThreads | |qnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| -|numOfSnodeSharedThreads | |snode 的共享线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| -|numOfSnodeUniqueThreads | |snode 的独占线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| -|ratioOfVnodeStreamThreads | |流计算使用 vnode 线程的比例，取值范围 0.01-4，默认值 4| -|ttlUnit | |ttl 参数的单位，取值范围 1-31572500，单位为秒，默认值 86400| -|ttlPushInterval | |ttl 检测超时频率，取值范围 1-100000，单位为秒，默认值 10| -|ttlChangeOnWrite | |ttl 到期时间是否伴随表的修改操作改变；0：不改变，1：改变；默认值为 0| -|ttlBatchDropNum | |ttl 一批删除子表的数目，最小值为 0，默认值 10000| -|retentionSpeedLimitMB | |数据在不同级别硬盘上迁移时的速度限制，取值范围 0-1024，单位 MB，默认值 0，表示不限制| -|maxTsmaNum | |集群内可创建的TSMA个数；取值范围 0-3；默认值 3| -|tmqMaxTopicNum | |订阅最多可建立的 topic 数量；取值范围 1-10000；默认值为 20| -|tmqRowSize | |订阅数据块的最大记录条数，取值范围 1-1000000，默认值 4096| -|audit | |审计功能开关；企业版参数| -|auditInterval | |审计数据上报的时间间隔；企业版参数| -|auditCreateTable | |是否针对创建子表开启申计功能；企业版参数| -|encryptAlgorithm | |数据加密算法；企业版参数| -|encryptScope | |加密范围；企业版参数| -|enableWhiteList | |白名单功能开关；企业版参数| -|syncLogBufferMemoryAllowed| |一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值 服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 | -|syncElectInterval | |内部参数，用于同步模块调试| -|syncHeartbeatInterval | |内部参数，用于同步模块调试| -|syncHeartbeatTimeout | |内部参数，用于同步模块调试| -|syncSnapReplMaxWaitN | |内部参数，用于同步模块调试| -|syncSnapReplMaxWaitN | |内部参数，用于同步模块调试| -|arbHeartBeatIntervalSec | |内部参数，用于同步模块调试| -|arbCheckSyncIntervalSec | |内部参数，用于同步模块调试| -|arbSetAssignedTimeoutSec | |内部参数，用于同步模块调试| -|mndSdbWriteDelta | |内部参数，用于 mnode 模块调试| -|mndLogRetention | |内部参数，用于 mnode 模块调试| -|skipGrant | |内部参数，用于授权检查| -|trimVDbIntervalSec | |内部参数，用于删除过期数据| -|ttlFlushThreshold | |内部参数，ttl 定时器的频率| -|compactPullupInterval | |内部参数，数据重整定时器的频率| -|walFsyncDataSizeLimit | |内部参数，WAL 进行 FSYNC 的阈值| -|transPullupInterval | |内部参数，mnode 执行事务的重试间隔| -|mqRebalanceInterval | |内部参数，消费者再平衡的时间间隔| -|uptimeInterval | |内部参数，用于记录系统启动时间| -|timeseriesThreshold | |内部参数，用于统计用量| -|udf | |是否启动 UDF 服务；0：不启动，1：启动；默认值为 0 | -|udfdResFuncs | |内部参数，用于 UDF 结果集设置| -|udfdLdLibPath | |内部参数，表示 UDF 装载的库路径| +|--------------------------|----------|-| +|supportVnodes | |dnode 支持的最大 vnode 数目，取值范围 0-4096，默认值 CPU 核数的 2 倍 + 5| +|numOfCommitThreads | |落盘线程的最大数量，取值范围 0-1024，默认值为 4| +|numOfMnodeReadThreads | |mnode 的 Read 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| +|numOfVnodeQueryThreads | |vnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| +|numOfVnodeFetchThreads | |vnode 的 Fetch 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| +|numOfVnodeRsmaThreads | |vnode 的 Rsma 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）| +|numOfQnodeQueryThreads | |qnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）| +|numOfSnodeSharedThreads | |snode 的共享线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| +|numOfSnodeUniqueThreads | |snode 的独占线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不小于 2，不超过 4）| +|ratioOfVnodeStreamThreads | |流计算使用 vnode 线程的比例，取值范围 0.01-4，默认值 4| +|ttlUnit | |ttl 参数的单位，取值范围 1-31572500，单位为秒，默认值 86400| +|ttlPushInterval | |ttl 检测超时频率，取值范围 1-100000，单位为秒，默认值 10| +|ttlChangeOnWrite | |ttl 到期时间是否伴随表的修改操作改变；0：不改变，1：改变；默认值为 0| +|ttlBatchDropNum | |ttl 一批删除子表的数目，最小值为 0，默认值 10000| +|retentionSpeedLimitMB | |数据在不同级别硬盘上迁移时的速度限制，取值范围 0-1024，单位 MB，默认值 0，表示不限制| +|maxTsmaNum | |集群内可创建的TSMA个数；取值范围 0-3；默认值 3| +|tmqMaxTopicNum | |订阅最多可建立的 topic 数量；取值范围 1-10000；默认值为 20| +|tmqRowSize | |订阅数据块的最大记录条数，取值范围 1-1000000，默认值 4096| +|audit | |审计功能开关；企业版参数| +|auditInterval | |审计数据上报的时间间隔；企业版参数| +|auditCreateTable | |是否针对创建子表开启申计功能；企业版参数| +|encryptAlgorithm | |数据加密算法；企业版参数| +|encryptScope | |加密范围；企业版参数| +|enableWhiteList | |白名单功能开关；企业版参数| +|syncLogBufferMemoryAllowed| |一个 dnode 允许的 sync 日志缓存消息占用的内存最大值，单位 bytes，取值范围 104857600-INT64_MAX，默认值 服务器内存的 1/10，3.1.3.2/3.3.2.13 版本开始生效 | +|syncElectInterval | |内部参数，用于同步模块调试| +|syncHeartbeatInterval | |内部参数，用于同步模块调试| +|syncHeartbeatTimeout | |内部参数，用于同步模块调试| +|syncSnapReplMaxWaitN | |内部参数，用于同步模块调试| +|syncSnapReplMaxWaitN | |内部参数，用于同步模块调试| +|arbHeartBeatIntervalSec | |内部参数，用于同步模块调试| +|arbCheckSyncIntervalSec | |内部参数，用于同步模块调试| +|arbSetAssignedTimeoutSec | |内部参数，用于同步模块调试| +|mndSdbWriteDelta | |内部参数，用于 mnode 模块调试| +|mndLogRetention | |内部参数，用于 mnode 模块调试| +|skipGrant | |内部参数，用于授权检查| +|trimVDbIntervalSec | |内部参数，用于删除过期数据| +|ttlFlushThreshold | |内部参数，ttl 定时器的频率| +|compactPullupInterval | |内部参数，数据重整定时器的频率| +|walFsyncDataSizeLimit | |内部参数，WAL 进行 FSYNC 的阈值| +|transPullupInterval | |内部参数，mnode 执行事务的重试间隔| +|mqRebalanceInterval | |内部参数，消费者再平衡的时间间隔| +|uptimeInterval | |内部参数，用于记录系统启动时间| +|timeseriesThreshold | |内部参数，用于统计用量| +|udf | |是否启动 UDF 服务；0：不启动，1：启动；默认值为 0 | +|udfdResFuncs | |内部参数，用于 UDF 结果集设置| +|udfdLdLibPath | |内部参数，表示 UDF 装载的库路径| ### 流计算参数 |参数名称|支持版本|参数含义| -|-----------------------|-----------|-| -|disableStream | |流计算的启动开关| -|streamBufferSize | |控制内存中窗口状态缓存的大小，默认值为 128MB| -|streamAggCnt | |内部参数，并发进行聚合计算的数目| -|checkpointInterval | |内部参数，checkponit 同步间隔| -|concurrentCheckpoint | |内部参数，是否并发检查 checkpoint| -|maxStreamBackendCache | |内部参数，流计算使用的最大缓存| -|streamSinkDataRate | |内部参数，用于控制流计算结果的写入速度| +|-----------------------|----------|-| +|disableStream | |流计算的启动开关| +|streamBufferSize | |控制内存中窗口状态缓存的大小，默认值为 128MB| +|streamAggCnt | |内部参数，并发进行聚合计算的数目| +|checkpointInterval | |内部参数，checkponit 同步间隔| +|concurrentCheckpoint | |内部参数，是否并发检查 checkpoint| +|maxStreamBackendCache | |内部参数，流计算使用的最大缓存| +|streamSinkDataRate | |内部参数，用于控制流计算结果的写入速度| ### 日志相关 |参数名称|支持版本|参数含义| -|----------------|-----------|-| -|logDir | |日志文件目录，运行日志将写入该目录，默认值 /var/log/taos| -|minimalLogDirGB | |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，默认值 1| -|numOfLogLines | |单个日志文件允许的最大行数，默认值 10,000,000| -|asyncLog | |日志写入模式，0：同步，1：异步，默认值 1| -|logKeepDays | |日志文件的最长保存时间，单位：天，默认值 0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taosdlog.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| -|slowLogThreshold|3.3.3.0 之后|慢查询门限值，大于等于门限值认为是慢查询，单位秒，默认值 3 | -|slowLogMaxLen |3.3.3.0 之后|慢查询日志最大长度，取值范围 1-16384，默认值 4096| -|slowLogScope |3.3.3.0 之后|慢查询记录类型，取值范围 ALL/QUERY/INSERT/OTHERS/NONE，默认值 QUERY| -|slowLogExceptDb |3.3.3.0 之后|指定的数据库不上报慢查询，仅支持配置换一个数据库| -|debugFlag | |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| -|tmrDebugFlag | |定时器模块的日志开关，取值范围同上| -|uDebugFlag | |共用功能模块的日志开关，取值范围同上| -|rpcDebugFlag | |rpc 模块的日志开关，取值范围同上| -|qDebugFlag | |query 模块的日志开关，取值范围同上| -|dDebugFlag | |dnode 模块的日志开关，取值范围同上| -|vDebugFlag | |vnode 模块的日志开关，取值范围同上| -|mDebugFlag | |mnode 模块的日志开关，取值范围同上| -|azDebugFlag |3.3.4.3 之后|S3 模块的日志开关，取值范围同上| -|sDebugFlag | |sync 模块的日志开关，取值范围同上| -|tsdbDebugFlag | |tsdb 模块的日志开关，取值范围同上| -|tqDebugFlag | |tq 模块的日志开关，取值范围同上| -|fsDebugFlag | |fs 模块的日志开关，取值范围同上| -|udfDebugFlag | |udf 模块的日志开关，取值范围同上| -|smaDebugFlag | |sma 模块的日志开关，取值范围同上| -|idxDebugFlag | |index 模块的日志开关，取值范围同上| -|tdbDebugFlag | |tdb 模块的日志开关，取值范围同上| -|metaDebugFlag | |meta 模块的日志开关，取值范围同上| -|stDebugFlag | |stream 模块的日志开关，取值范围同上| -|sndDebugFlag | |snode 模块的日志开关，取值范围同上| +|----------------|----------|-| +|logDir | |日志文件目录，运行日志将写入该目录，默认值 /var/log/taos| +|minimalLogDirGB | |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，默认值 1| +|numOfLogLines | |单个日志文件允许的最大行数，默认值 10,000,000| +|asyncLog | |日志写入模式，0：同步，1：异步，默认值 1| +|logKeepDays | |日志文件的最长保存时间，单位：天，默认值 0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taosdlog.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| +|slowLogThreshold|3.3.3.0 后|慢查询门限值，大于等于门限值认为是慢查询，单位秒，默认值 3 | +|slowLogMaxLen |3.3.3.0 后|慢查询日志最大长度，取值范围 1-16384，默认值 4096| +|slowLogScope |3.3.3.0 后|慢查询记录类型，取值范围 ALL/QUERY/INSERT/OTHERS/NONE，默认值 QUERY| +|slowLogExceptDb |3.3.3.0 后|指定的数据库不上报慢查询，仅支持配置换一个数据库| +|debugFlag | |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| +|tmrDebugFlag | |定时器模块的日志开关，取值范围同上| +|uDebugFlag | |共用功能模块的日志开关，取值范围同上| +|rpcDebugFlag | |rpc 模块的日志开关，取值范围同上| +|qDebugFlag | |query 模块的日志开关，取值范围同上| +|dDebugFlag | |dnode 模块的日志开关，取值范围同上| +|vDebugFlag | |vnode 模块的日志开关，取值范围同上| +|mDebugFlag | |mnode 模块的日志开关，取值范围同上| +|azDebugFlag |3.3.4.3 后|S3 模块的日志开关，取值范围同上| +|sDebugFlag | |sync 模块的日志开关，取值范围同上| +|tsdbDebugFlag | |tsdb 模块的日志开关，取值范围同上| +|tqDebugFlag | |tq 模块的日志开关，取值范围同上| +|fsDebugFlag | |fs 模块的日志开关，取值范围同上| +|udfDebugFlag | |udf 模块的日志开关，取值范围同上| +|smaDebugFlag | |sma 模块的日志开关，取值范围同上| +|idxDebugFlag | |index 模块的日志开关，取值范围同上| +|tdbDebugFlag | |tdb 模块的日志开关，取值范围同上| +|metaDebugFlag | |meta 模块的日志开关，取值范围同上| +|stDebugFlag | |stream 模块的日志开关，取值范围同上| +|sndDebugFlag | |snode 模块的日志开关，取值范围同上| ### 调试相关 |参数名称|支持版本|参数含义| -|--------------------|-----------|-| -|enableCoreFile | |crash 时是否生成 core 文件，0：不生成，1：生成；默认值 1| -|configDir | |配置文件所在目录| -|scriptDir | |内部测试工具的脚本目录| -|assert | |断言控制开关，默认值 0| -|randErrorChance | |内部参数，用于随机失败测试| -|randErrorDivisor | |内部参数，用于随机失败测试| -|randErrorScope | |内部参数，用于随机失败测试| -|safetyCheckLevel | |内部参数，用于随机失败测试| -|experimental | |内部参数，用于一些实验特性| -|simdEnable |3.3.4.3 之后|内部参数，用于测试 SIMD 加速| -|AVX512Enable |3.3.4.3 之后|内部参数，用于测试 AVX512 加速| -|rsyncPort | |内部参数，用于调试流计算| -|snodeAddress | |内部参数，用于调试流计算| -|checkpointBackupDir | |内部参数，用于恢复 snode 数据| -|enableAuditDelete | |内部参数，用于测试审计功能| -|slowLogThresholdTest| |内部参数，用于测试慢日志| +|--------------------|----------|-| +|enableCoreFile | |crash 时是否生成 core 文件，0：不生成，1：生成；默认值 1| +|configDir | |配置文件所在目录| +|scriptDir | |内部测试工具的脚本目录| +|assert | |断言控制开关，默认值 0| +|randErrorChance | |内部参数，用于随机失败测试| +|randErrorDivisor | |内部参数，用于随机失败测试| +|randErrorScope | |内部参数，用于随机失败测试| +|safetyCheckLevel | |内部参数，用于随机失败测试| +|experimental | |内部参数，用于一些实验特性| +|simdEnable |3.3.4.3 后|内部参数，用于测试 SIMD 加速| +|AVX512Enable |3.3.4.3 后|内部参数，用于测试 AVX512 加速| +|rsyncPort | |内部参数，用于调试流计算| +|snodeAddress | |内部参数，用于调试流计算| +|checkpointBackupDir | |内部参数，用于恢复 snode 数据| +|enableAuditDelete | |内部参数，用于测试审计功能| +|slowLogThresholdTest| |内部参数，用于测试慢日志| ### 压缩参数 |参数名称|支持版本|参数含义| -|------------|-----------|-| -|fPrecision | |设置 float 类型浮点数压缩精度 ，取值范围 0.1 ~ 0.00000001 ，默认值 0.00000001 , 小于此值的浮点数尾数部分将被截断| -|dPrecision | |设置 double 类型浮点数压缩精度 , 取值范围 0.1 ~ 0.0000000000000001 ， 默认值 0.0000000000000001 ， 小于此值的浮点数尾数部分将被截取| -|lossyColumn |3.3.0.0 之前|对 float 和/或 double 类型启用 TSZ 有损压缩；取值范围 float/double/none；默认值 none，表示关闭无损压缩| -|ifAdtFse | |在启用 TSZ 有损压缩时，使用 FSE 算法替换 HUFFMAN 算法，FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法；0：关闭，1：打开；默认值为 0| -|maxRange | |内部参数，用于有损压缩设置| -|curRange | |内部参数，用于有损压缩设置| -|compressor | |内部参数，用于有损压缩设置| +|------------|----------|-| +|fPrecision | |设置 float 类型浮点数压缩精度 ，取值范围 0.1 ~ 0.00000001 ，默认值 0.00000001 , 小于此值的浮点数尾数部分将被截断| +|dPrecision | |设置 double 类型浮点数压缩精度 , 取值范围 0.1 ~ 0.0000000000000001 ， 默认值 0.0000000000000001 ， 小于此值的浮点数尾数部分将被截取| +|lossyColumn |3.3.0.0 前|对 float 和/或 double 类型启用 TSZ 有损压缩；取值范围 float/double/none；默认值 none，表示关闭无损压缩| +|ifAdtFse | |在启用 TSZ 有损压缩时，使用 FSE 算法替换 HUFFMAN 算法，FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法；0：关闭，1：打开；默认值为 0| +|maxRange | |内部参数，用于有损压缩设置| +|curRange | |内部参数，用于有损压缩设置| +|compressor | |内部参数，用于有损压缩设置| **补充说明** 1. 在 3.2.0.0 ~ 3.3.0.0（不包含）版本生效，启用该参数后不能回退到升级前的版本 diff --git a/docs/zh/14-reference/01-components/02-taosc.md b/docs/zh/14-reference/01-components/02-taosc.md index 3efaddee26..09653ae3ef 100755 --- a/docs/zh/14-reference/01-components/02-taosc.md +++ b/docs/zh/14-reference/01-components/02-taosc.md @@ -10,99 +10,98 @@ TDengine 客户端驱动提供了应用编程所需要的全部 API，并且在 ### 连接相关 |参数名称|支持版本|参数含义| -|----------------------|-----------|-| -|firstEp | |启动时，主动连接的集群中首个 dnode 的 endpoint，缺省值：hostname:6030，若无法获取该服务器的 hostname，则赋值为 localhost| -|secondEp | |启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，没有缺省值| -|compressMsgSize | |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；缺省值 -1| -|shellActivityTimer | |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3| -|numOfRpcSessions | |RPC 支持的最大连接数，取值范围 100-100000，缺省值 30000| -|numOfRpcThreads | |RPC 线程数目，默认值为 CPU 核数的一半| -|timeToGetAvailableConn| |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值 500000| -|useAdapter | |内部参数，是否使用 taosadapter，影响 CSV 文件导入| -|shareConnLimit |3.3.4.3 之后|内部参数，一个链接可以共享的查询数目，取值范围 1-256，默认值 10| -|readTimeout |3.3.4.3 之后|内部参数，最小超时时间，取值范围 64-604800，单位为秒，默认值 900| +|----------------------|----------|-| +|firstEp | |启动时，主动连接的集群中首个 dnode 的 endpoint，缺省值：hostname:6030，若无法获取该服务器的 hostname，则赋值为 localhost| +|secondEp | |启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，没有缺省值| +|compressMsgSize | |是否对 RPC 消息进行压缩；-1：所有消息都不压缩；0：所有消息都压缩；N (N>0)：只有大于 N 个字节的消息才压缩；缺省值 -1| +|shellActivityTimer | |客户端向 mnode 发送心跳的时长，单位为秒，取值范围 1-120，默认值 3| +|numOfRpcSessions | |RPC 支持的最大连接数，取值范围 100-100000，缺省值 30000| +|numOfRpcThreads | |RPC 线程数目，默认值为 CPU 核数的一半| +|timeToGetAvailableConn| |获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值 500000| +|useAdapter | |内部参数，是否使用 taosadapter，影响 CSV 文件导入| +|shareConnLimit |3.3.4.3 后|内部参数，一个链接可以共享的查询数目，取值范围 1-256，默认值 10| +|readTimeout |3.3.4.3 后|内部参数，最小超时时间，取值范围 64-604800，单位为秒，默认值 900| ### 查询相关 |参数名称|支持版本|参数含义| -|---------------------------------|-----------|-| -|countAlwaysReturnValue | |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| -|keepColumnName | |Last、First、LastRow 函数查询且未指定别名时，自动设置别名为列名（不含函数名），因此 order by 子句如果引用了该列名将自动引用该列对应的函数；1：表示自动设置别名为列名(不包含函数名)，0：表示不自动设置别名；缺省值：0| -|multiResultFunctionStarReturnTags|3.3.3.0 以后|查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列；查询普通表、子表时，不受该参数影响；0：不返回标签列，1：返回标签列；缺省值：0；该参数设置为 0 时，last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列；为 1 时，返回超级表的普通列和标签列| -| - | |指定单个客户端元数据缓存大小的最大值，单位 MB；缺省值 -1，表示无限制| -|maxTsmaCalcDelay | |查询时客户端可允许的 tsma 计算延迟，若 tsma 的计算延迟大于配置值，则该 TSMA 将不会被使用；取值范围 600s - 86400s，即 10 分钟 - 1 小时；缺省值：600 秒| -|tsmaDataDeleteMark | |TSMA 计算的历史数据中间结果保存时间，单位为毫秒；取值范围 >= 3600000，即大于等于1h；缺省值：86400000，即 1d | -|queryPolicy | |查询语句的执行策略，1：只使用 vnode，不使用 qnode；2：没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行；3：vnode 只运行扫描算子，其余算子均在 qnode 执行；缺省值：1| -|queryTableNotExistAsEmpty | |查询表不存在时是否返回空结果集；false：返回错误；true：返回空结果集；缺省值 false| -|querySmaOptimize | |sma index 的优化策略，0：表示不使用 sma index，永远从原始数据进行查询；1：表示使用 sma index，对符合的语句，直接从预计算的结果进行查询；缺省值：0| -|queryPlannerTrace | |内部参数，查询计划是否输出详细日志| -|queryNodeChunkSize | |内部参数，查询计划的块大小| -|queryUseNodeAllocator | |内部参数，查询计划的分配方法| -|queryMaxConcurrentTables | |内部参数，查询计划的并发数目| -|enableQueryHb | |内部参数，是否发送查询心跳消息| -|minSlidingTime | |内部参数，sliding 的最小允许值| -|minIntervalTime | |内部参数，interval 的最小允许值| +|---------------------------------|---------|-| +|countAlwaysReturnValue | |count/hyperloglog 函数在输入数据为空或者 NULL 的情况下是否返回值；0：返回空行，1：返回；默认值 1；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了 TSMA 时，且相应的组或窗口内数据为空或者 NULL，对应的组或窗口将不返回查询结果；注意此参数客户端和服务端值应保持一致| +|keepColumnName | |Last、First、LastRow 函数查询且未指定别名时，自动设置别名为列名（不含函数名），因此 order by 子句如果引用了该列名将自动引用该列对应的函数；1：表示自动设置别名为列名(不包含函数名)，0：表示不自动设置别名；缺省值：0| +|multiResultFunctionStarReturnTags|3.3.3.0 后|查询超级表时，last(\*)/last_row(\*)/first(\*) 是否返回标签列；查询普通表、子表时，不受该参数影响；0：不返回标签列，1：返回标签列；缺省值：0；该参数设置为 0 时，last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列；为 1 时，返回超级表的普通列和标签列| +|metaCacheMaxSize | |指定单个客户端元数据缓存大小的最大值，单位 MB；缺省值 -1，表示无限制| +|maxTsmaCalcDelay | |查询时客户端可允许的 tsma 计算延迟，若 tsma 的计算延迟大于配置值，则该 TSMA 将不会被使用；取值范围 600s - 86400s，即 10 分钟 - 1 小时；缺省值：600 秒| +|tsmaDataDeleteMark | |TSMA 计算的历史数据中间结果保存时间，单位为毫秒；取值范围 >= 3600000，即大于等于1h；缺省值：86400000，即 1d | +|queryPolicy | |查询语句的执行策略，1：只使用 vnode，不使用 qnode；2：没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行；3：vnode 只运行扫描算子，其余算子均在 qnode 执行；缺省值：1| +|queryTableNotExistAsEmpty | |查询表不存在时是否返回空结果集；false：返回错误；true：返回空结果集；缺省值 false| +|querySmaOptimize | |sma index 的优化策略，0：表示不使用 sma index，永远从原始数据进行查询；1：表示使用 sma index，对符合的语句，直接从预计算的结果进行查询；缺省值：0| +|queryPlannerTrace | |内部参数，查询计划是否输出详细日志| +|queryNodeChunkSize | |内部参数，查询计划的块大小| +|queryUseNodeAllocator | |内部参数，查询计划的分配方法| +|queryMaxConcurrentTables | |内部参数，查询计划的并发数目| +|enableQueryHb | |内部参数，是否发送查询心跳消息| +|minSlidingTime | |内部参数，sliding 的最小允许值| +|minIntervalTime | |内部参数，interval 的最小允许值| ### 写入相关 |参数名称|支持版本|参数含义| -|------------------------------|-----------|-| -|smlChildTableName | |schemaless 自定义的子表名的 key，无缺省值| -|smlAutoChildTableNameDelimiter| |schemaless tag 之间的连接符，连起来作为子表名，无缺省值| -|smlTagName | |schemaless tag 为空时默认的 tag 名字，缺省值 "_tag_null"| -|smlTsDefaultName | |schemaless 自动建表的时间列名字通过该配置设置，缺省值 "_ts"| -|smlDot2Underline | |schemaless 把超级表名中的 dot 转成下划线| -|maxInsertBatchRows | |内部参数，一批写入的最大条数| +|------------------------------|----------|-| +|smlChildTableName | |schemaless 自定义的子表名的 key，无缺省值| +|smlAutoChildTableNameDelimiter| |schemaless tag 之间的连接符，连起来作为子表名，无缺省值| +|smlTagName | |schemaless tag 为空时默认的 tag 名字，缺省值 "_tag_null"| +|smlTsDefaultName | |schemaless 自动建表的时间列名字通过该配置设置，缺省值 "_ts"| +|smlDot2Underline | |schemaless 把超级表名中的 dot 转成下划线| +|maxInsertBatchRows | |内部参数，一批写入的最大条数| ### 区域相关 |参数名称|支持版本|参数含义| -|-----------------|-----------|-| -|timezone | |时区；缺省从系统中动态获取当前的时区设置| -|locale | |系统区位信息及编码格式，缺省从系统中获取| -|charset | |字符集编码，缺省从系统中获取| +|-----------------|----------|-| +|timezone | |时区；缺省从系统中动态获取当前的时区设置| +|locale | |系统区位信息及编码格式，缺省从系统中获取| +|charset | |字符集编码，缺省从系统中获取| ### 存储相关 |参数名称|支持版本|参数含义| -|-----------------|-----------|-| -|tempDir | |指定所有运行过程中的临时文件生成的目录，Linux 平台默认值为 /tmp| -|minimalTmpDirGB | |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，缺省值：1| +|-----------------|----------|-| +|tempDir | |指定所有运行过程中的临时文件生成的目录，Linux 平台默认值为 /tmp| +|minimalTmpDirGB | |tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，缺省值：1| ### 日志相关 |参数名称|支持版本|参数含义| -|-----------------|-----------|-| -|logDir | |日志文件目录，运行日志将写入该目录，缺省值：/var/log/taos| -|minimalLogDirGB | |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，缺省值：1| -|numOfLogLines | |单个日志文件允许的最大行数，缺省值：10,000,000| -|asyncLog | |日志写入模式，0：同步，1：异步，缺省值：1| -|logKeepDays | |日志文件的最长保存时间，单位：天，缺省值：0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taoslogx.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| -|debugFlag | |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| -|tmrDebugFlag | |定时器模块的日志开关，取值范围同上| -|uDebugFlag | |共用功能模块的日志开关，取值范围同上| -|rpcDebugFlag | |rpc 模块的日志开关，取值范围同上| -|jniDebugFlag | |jni 模块的日志开关，取值范围同上| -|qDebugFlag | |query 模块的日志开关，取值范围同上| -|cDebugFlag | |客户端模块的日志开关，取值范围同上| -|simDebugFlag | |内部参数，测试工具的日志开关，取值范围同上| -|tqClientDebugFlag|3.3.4.3 之后|客户端模块的日志开关，取值范围同上| +|-----------------|----------|-| +|logDir | |日志文件目录，运行日志将写入该目录，缺省值：/var/log/taos| +|minimalLogDirGB | |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，缺省值：1| +|numOfLogLines | |单个日志文件允许的最大行数，缺省值：10,000,000| +|asyncLog | |日志写入模式，0：同步，1：异步，缺省值：1| +|logKeepDays | |日志文件的最长保存时间，单位：天，缺省值：0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taoslogx.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件| +|debugFlag | |运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）；默认值 131 或 135 （取决于不同模块）| +|tmrDebugFlag | |定时器模块的日志开关，取值范围同上| +|uDebugFlag | |共用功能模块的日志开关，取值范围同上| +|rpcDebugFlag | |rpc 模块的日志开关，取值范围同上| +|jniDebugFlag | |jni 模块的日志开关，取值范围同上| +|qDebugFlag | |query 模块的日志开关，取值范围同上| +|cDebugFlag | |客户端模块的日志开关，取值范围同上| +|simDebugFlag | |内部参数，测试工具的日志开关，取值范围同上| +|tqClientDebugFlag|3.3.4.3 后|客户端模块的日志开关，取值范围同上| ### 调试相关 |参数名称|支持版本|参数含义| |-----------------|-----------|-| -|crashReporting | |是否上传 crash 到 telemetry，0：不上传，1：上传；缺省值：1| -|enableCoreFile | |crash 时是否生成 core 文件，0：不生成，1：生成；缺省值：1| -|assert | |断言控制开关，缺省值：0| -|configDir | |配置文件所在目录| -|scriptDir | |内部参数，测试用例的目录| -|randErrorChance |3.3.3.0 之后|内部参数，用于随机失败测试| -|randErrorDivisor |3.3.3.0 之后|内部参数，用于随机失败测试| -|randErrorScope |3.3.3.0 之后|内部参数，用于随机失败测试| -|safetyCheckLevel |3.3.3.0 之后|内部参数，用于随机失败测试| -|simdEnable |3.3.4.3 之后|内部参数，用于测试 SIMD 加速| -|AVX512Enable |3.3.4.3 之后|内部参数，用于测试 AVX512 加速| +|crashReporting | |是否上传 crash 到 telemetry，0：不上传，1：上传；缺省值：1| +|enableCoreFile | |crash 时是否生成 core 文件，0：不生成，1：生成；缺省值：1| +|assert | |断言控制开关，缺省值：0| +|configDir | |配置文件所在目录| +|scriptDir | |内部参数，测试用例的目录| +|randErrorChance |3.3.3.0 后|内部参数，用于随机失败测试| +|randErrorDivisor |3.3.3.0 后|内部参数，用于随机失败测试| +|randErrorScope |3.3.3.0 后|内部参数，用于随机失败测试| +|safetyCheckLevel |3.3.3.0 后|内部参数，用于随机失败测试| +|simdEnable |3.3.4.3 后|内部参数，用于测试 SIMD 加速| +|AVX512Enable |3.3.4.3 后|内部参数，用于测试 AVX512 加速| ### SHELL 相关 |参数名称|支持版本|参数含义| -|-----------------|-----------|-| -|enableScience | |是否开启科学计数法显示浮点数；0：不开始，1：开启；缺省值：1| +|-----------------|----------|-| +|enableScience | |是否开启科学计数法显示浮点数；0：不开始，1：开启；缺省值：1| ## API From 75a7da81792d5cc51b5a84622987dc167a704563 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 14 Nov 2024 09:36:39 +0800 Subject: [PATCH 25/62] fix: ci issue --- source/common/src/tmisce.c | 2 +- tests/system-test/0-others/information_schema.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tmisce.c b/source/common/src/tmisce.c index dfba2ebf6c..8988fab56a 100644 --- a/source/common/src/tmisce.c +++ b/source/common/src/tmisce.c @@ -331,7 +331,7 @@ _start: TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit); char info[TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE] = {0}; - if (strcasecmp(pItem->name, "dataDir") == 0) { + if (strcasecmp(pItem->name, "dataDir") == 0 && pDiskCfg) { char* buf = &info[VARSTR_HEADER_SIZE]; valueLen = tsnprintf(buf, TSDB_CONFIG_INFO_LEN, "level %d primary %d disabled %" PRIi8, pDiskCfg->level, pDiskCfg->primary, pDiskCfg->disable); diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index aa548d4e59..ccf95db563 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(281, 282)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(282, 283)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(56, len(tdSql.queryResult)) From ab7b0026b36d757f07ce083d02e6e5e4b6d6d77f Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 14 Nov 2024 10:19:44 +0800 Subject: [PATCH 26/62] refine code --- source/libs/planner/src/planOptimizer.c | 43 ++++++++++++++++--------- tests/system-test/2-query/union.py | 7 +++- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index b179c58ee1..39024731ed 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -3501,15 +3501,17 @@ static int32_t eliminateProjOptFindProjPrefixWithOrderCheck(SProjectLogicNode* p *orderMatch = false; break; } - SNode* pNew = NULL; - code = nodesCloneNode(pChildTarget, &pNew); - if (TSDB_CODE_SUCCESS == code) { - code = nodesListMakeStrictAppend(pNewChildTargets, pNew); - } - if (TSDB_CODE_SUCCESS != code && pNewChildTargets) { - nodesDestroyList(*pNewChildTargets); - *pNewChildTargets = NULL; - break; + if (pNewChildTargets) { + SNode* pNew = NULL; + code = nodesCloneNode(pChildTarget, &pNew); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(pNewChildTargets, pNew); + } + if (TSDB_CODE_SUCCESS != code && pNewChildTargets) { + nodesDestroyList(*pNewChildTargets); + *pNewChildTargets = NULL; + break; + } } } return code; @@ -3543,6 +3545,8 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* int32_t code = 0; bool isSetOpProj = false; bool orderMatch = false; + bool sizeMatch = LIST_LENGTH(pProjectNode->pProjections) == LIST_LENGTH(pChild->pTargets); + bool needReplaceTargets = true; if (NULL == pProjectNode->node.pParent) { SNodeList* pNewChildTargets = NULL; @@ -3553,7 +3557,12 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* // For sql: select ... from (select ... union all select ...); // When eliminating the outer proj (the outer select), we have to make sure that the outer proj projections and // union all project targets have same columns in the same order. See detail in TD-30188 - code = eliminateProjOptFindProjPrefixWithOrderCheck(pProjectNode, (SProjectLogicNode*)pChild, &pNewChildTargets, &orderMatch); + code = eliminateProjOptFindProjPrefixWithOrderCheck(pProjectNode, (SProjectLogicNode*)pChild, + sizeMatch ? NULL : &pNewChildTargets, &orderMatch); + if (TSDB_CODE_SUCCESS == code && sizeMatch && orderMatch) { + pNewChildTargets = pChild->pTargets; + needReplaceTargets = false; + } } else { FOREACH(pProjection, pProjectNode->pProjections) { FOREACH(pChildTarget, pChild->pTargets) { @@ -3577,10 +3586,12 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* } if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets) && (!isSetOpProj || orderMatch)) { - nodesDestroyList(pChild->pTargets); - pChild->pTargets = pNewChildTargets; + if (needReplaceTargets) { + nodesDestroyList(pChild->pTargets); + pChild->pTargets = pNewChildTargets; + } } else { - nodesDestroyList(pNewChildTargets); + if (needReplaceTargets) nodesDestroyList(pNewChildTargets); OPTIMIZE_FLAG_SET_MASK(pProjectNode->node.optimizedFlag, OPTIMIZE_FLAG_ELIMINATE_PROJ); pCxt->optimized = true; return TSDB_CODE_SUCCESS; @@ -3603,10 +3614,10 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* NODES_CLEAR_LIST(pProjectNode->node.pChildren); nodesDestroyNode((SNode*)pProjectNode); // if pChild is a project logic node, remove its projection which is not reference by its target. - alignProjectionWithTarget(pChild); - if (isSetOpProj && orderMatch) { + if (needReplaceTargets) { + alignProjectionWithTarget(pChild); // Since we have eliminated the outer proj, we need to push down the new targets to the children of the set operation. - code = eliminateProjOptPushTargetsToSetOpChildren((SProjectLogicNode*)pChild); + if (isSetOpProj && orderMatch && !sizeMatch) code = eliminateProjOptPushTargetsToSetOpChildren((SProjectLogicNode*)pChild); } } pCxt->optimized = true; diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py index b74db0104d..5104489592 100644 --- a/tests/system-test/2-query/union.py +++ b/tests/system-test/2-query/union.py @@ -399,7 +399,12 @@ class TDTestCase: tdSql.query(sql, queryTimes=1) tdSql.checkRows(6) tdSql.checkCols(3) - tdSql.execute("drop database ep_iot") + + sql = "select scdw_code, scdw_name, jzmc, fdgl, jzzt,ts from ((select '01072016' as scdw_code, '盛鲁电厂' as scdw_name, '机组1' as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '01072016' as scdw_code, '盛鲁电厂' as scdw_name, '机组2' as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '00103673' as scdw_code, '鲁西电厂' as scdw_name, '机组1'as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '00103673' as scdw_code, '鲁西电厂' as scdw_name, '机组2'as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt, last(ts) as ts from ep_iot.sldc_dp) union all ( select '01061584' as scdw_code, '富源热电' as scdw_name, '机组1'as jzmc, last(jz1fdjyggl) as fdgl, '填报' as jzzt ,last(ts) as ts from ep_iot.sldc_dp) union all ( select '01061584' as scdw_code, '富源热电' as scdw_name, '机组2'as jzmc, last(jz2fdjyggl) as fdgl, '填报' as jzzt ,last(ts) as ts from ep_iot.sldc_dp)) where scdw_code like '%%';" + tdSql.query(sql, queryTimes=1) + tdSql.checkCols(6) + tdSql.checkRows(6) + ##tdSql.execute("drop database ep_iot") def run(self): tdSql.prepare() From 336429e14904e60ce05188c3b82988363c272da5 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 14 Nov 2024 16:01:28 +0800 Subject: [PATCH 27/62] fix/TD-32863-add-creating-hash-fix-review --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index a81208c567..2ee607949d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -63,14 +63,18 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { int32_t numOfVnodes = 0; SVnodeObj **ppVnodes = NULL; - code = vmGetAllVnodeListFromHashWithCreating(pMgmt, &numOfVnodes, &ppVnodes); + code = taosThreadMutexLock(&pMgmt->mutex); if (code != 0) { return code; } - code = taosThreadMutexLock(&pMgmt->mutex); + code = vmGetAllVnodeListFromHashWithCreating(pMgmt, &numOfVnodes, &ppVnodes); if (code != 0) { - goto _OVER; + int32_t r = taosThreadMutexUnlock(&pMgmt->mutex); + if (r != 0) { + dError("vgId:%d, failed to unlock mutex since %s", vgId, tstrerror(r)); + } + return code; } for (int32_t v = 0; v < numOfVnodes; v++) { From bc7d46a5101bf3848341b5310f9b18ce9b210b44 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 14 Nov 2024 16:57:04 +0800 Subject: [PATCH 28/62] Update index.md --- docs/zh/28-releases/03-notes/index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/zh/28-releases/03-notes/index.md b/docs/zh/28-releases/03-notes/index.md index 3268c088b2..d1a48ab9a8 100644 --- a/docs/zh/28-releases/03-notes/index.md +++ b/docs/zh/28-releases/03-notes/index.md @@ -3,6 +3,7 @@ title: 版本说明 sidebar_label: 版本说明 description: 各版本版本说明 --- -[3.3.4.3](./3.3.4.3) + +[3.3.4.3](./3.3.4.3) [3.3.3.0](./3.3.3.0) [3.3.2.0](./3.3.2.0) From 6a8d1e3cf2ce4b4e541cc4181cad4cf0054d3f9c Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Thu, 14 Nov 2024 17:41:56 +0800 Subject: [PATCH 29/62] docs:fix a typo 03-table.md --- docs/zh/14-reference/03-taos-sql/03-table.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-taos-sql/03-table.md b/docs/zh/14-reference/03-taos-sql/03-table.md index 81ad60e3d2..40e2802fcd 100644 --- a/docs/zh/14-reference/03-taos-sql/03-table.md +++ b/docs/zh/14-reference/03-taos-sql/03-table.md @@ -227,7 +227,7 @@ DROP TABLE [IF EXISTS] [db_name.]tb_name [, [IF EXISTS] [db_name.]tb_name] ... 如下 SQL 语句可以列出当前数据库中的所有表名。 ```sql -SHOW TABLES [LIKE tb_name_wildchar]; +SHOW TABLES [LIKE tb_name_wildcard]; ``` ### 显示表创建语句 From 37021cb6f6d0aa1d69120c66bafd8492df4795b9 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Thu, 14 Nov 2024 17:56:22 +0800 Subject: [PATCH 30/62] docs: Optimize description of JDBC subscription.md --- docs/zh/07-develop/07-tmq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md index a91a764c67..8b272bdfff 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.md @@ -16,7 +16,7 @@ TDengine 提供了类似于消息队列产品的数据订阅和消费接口。 **注意** 在 TDengine 连接器实现中，对于订阅查询，有以下限制。 -- 查询语句限制：订阅查询只能使用 select 语句，不支持其他类型的SQL，如 insert、update 或 delete 等。 +- 查询语句限制：订阅查询只能使用 select 语句，并不支持其他类型的SQL，如订阅库，订阅超级表（非 select 方式），insert、update 或 delete 等。 - 原始始数据查询：订阅查询只能查询原始数据，而不能查询聚合或计算结果。 - 时间顺序限制：订阅查询只能按照时间正序查询数据。 From d7cee30914100d070ebbf82891071d6ed8b75cd0 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Thu, 14 Nov 2024 19:42:52 +0800 Subject: [PATCH 31/62] Update 3.3.4.3.md --- docs/zh/28-releases/03-notes/3.3.4.3.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/zh/28-releases/03-notes/3.3.4.3.md b/docs/zh/28-releases/03-notes/3.3.4.3.md index 9ded536872..8ffd5802ed 100644 --- a/docs/zh/28-releases/03-notes/3.3.4.3.md +++ b/docs/zh/28-releases/03-notes/3.3.4.3.md @@ -3,6 +3,12 @@ title: 3.3.4.3 版本说明 sidebar_label: 3.3.4.3 description: 3.3.4.3 版本说明 --- + +### 行为变更及兼容性 +1. 多副本流计算中必须使用 snode +1. 增加了流计算的兼容性保证机制，避免后续函数变更产生新的兼容性问题，但之前版本的流计算必须重建，具体参见 https://docs.taosdata.com/advanced/stream/#流计算升级故障恢复 +1. 调整 case when 语句结果类型的判断方法 + ### 新特性 1. 新功能：流计算的 TWA 函数支持时间驱动的结果推送模式 1. 新功能：流计算的 Interp 函数支持时间驱动的结果推送模式 From 4a09cb7138b36007e974dcb0725fd818d2a1e62f Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 15 Nov 2024 08:05:50 +0800 Subject: [PATCH 32/62] fix/TD-32863-add-creating-hash-fix-case --- tests/system-test/0-others/multilevel_createdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/multilevel_createdb.py b/tests/system-test/0-others/multilevel_createdb.py index a4fd94f4c1..70131a760b 100644 --- a/tests/system-test/0-others/multilevel_createdb.py +++ b/tests/system-test/0-others/multilevel_createdb.py @@ -68,7 +68,7 @@ class TDTestCase: checkFiles(r'/mnt/data1/*/*',1) checkFiles(r'/mnt/data2/*/*',0) - tdSql.execute('create database nws vgroups 20 buffer 3000 stt_trigger 1 wal_level 1 wal_retention_period 0') + tdSql.execute('create database nws vgroups 20 stt_trigger 1 wal_level 1 wal_retention_period 0') checkFiles(r'/mnt/data1/vnode/*/wal',5) checkFiles(r'/mnt/data2/vnode/*/wal',5) From a854597de14c05b8fcf172f2484bcc01f69e84b5 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Fri, 15 Nov 2024 13:48:57 +0800 Subject: [PATCH 33/62] docs:Update keep description in 02-database.md --- docs/zh/14-reference/03-taos-sql/02-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index 91b39976a1..0d9ab019c0 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -63,7 +63,7 @@ database_option: { - DURATION：数据文件存储数据的时间跨度。可以使用加单位的表示形式，如 DURATION 100h、DURATION 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。不加时间单位时默认单位为天，如 DURATION 50 表示 50 天。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 -- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 +- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据从而释放存储空间。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 - STT_TRIGGER：表示落盘文件触发文件合并的个数。开源版本固定为 1，企业版本可设置范围为 1 到 16。对于少表高频写入场景，此参数建议使用默认配置；而对于多表低频写入场景，此参数建议配置较大的值。 - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 - 0：表示可以创建多张超级表。 From 07a11a4ea58498c1f0befa7b7b778128412f68d5 Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Fri, 15 Nov 2024 13:49:26 +0800 Subject: [PATCH 34/62] test:add stream test cases --- tests/system-test/8-stream/checkpoint_info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/8-stream/checkpoint_info.py b/tests/system-test/8-stream/checkpoint_info.py index 00e5d1f688..522017a702 100644 --- a/tests/system-test/8-stream/checkpoint_info.py +++ b/tests/system-test/8-stream/checkpoint_info.py @@ -84,6 +84,7 @@ class TDTestCase: def restart_stream(self): tdLog.debug("========restart stream========") + time.sleep(10) for i in range(5): tdSql.execute("pause stream s1") time.sleep(2) @@ -136,4 +137,4 @@ class TDTestCase: tdSql.close() tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) From 0c45c80d71b55feff88a56af68f28c2c7d8a361d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 15 Nov 2024 14:26:53 +0800 Subject: [PATCH 35/62] enh: add dangle child table filter in recovery mode --- source/dnode/vnode/src/meta/metaOpen.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 659ba3f777..9a5bea33e3 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -324,7 +324,11 @@ static int32_t metaGenerateNewMeta(SMeta **ppMeta) { SMetaEntry me = {0}; tDecoderInit(&dc, value, valueSize); if (metaDecodeEntry(&dc, &me) == 0) { - if (metaHandleEntry(pNewMeta, &me) != 0) { + if (me.type == TSDB_CHILD_TABLE && + tdbTbGet(pMeta->pUidIdx, &me.ctbEntry.suid, sizeof(me.ctbEntry.suid), NULL, NULL) != 0) { + metaError("vgId:%d failed to get super table uid:%" PRId64 " for child table uid:%" PRId64, + TD_VID(pVnode), me.ctbEntry.suid, uid); + } else if (metaHandleEntry(pNewMeta, &me) != 0) { metaError("vgId:%d failed to handle entry, uid:%" PRId64, TD_VID(pVnode), uid); } } From 2f978a11c2721ea7bfe7a3e7e958a2d4e67baeac Mon Sep 17 00:00:00 2001 From: happyguoxy Date: Fri, 15 Nov 2024 15:01:34 +0800 Subject: [PATCH 36/62] test:add stream test cases --- tests/system-test/8-stream/checkpoint_info2.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/system-test/8-stream/checkpoint_info2.py b/tests/system-test/8-stream/checkpoint_info2.py index 7f2e8b0672..3dc57477f7 100644 --- a/tests/system-test/8-stream/checkpoint_info2.py +++ b/tests/system-test/8-stream/checkpoint_info2.py @@ -33,17 +33,21 @@ class TDTestCase: info_path = os.path.join(root, f'checkpoint{checkpointid}', 'info') if os.path.exists(info_path): if task_id in info_path: + tdLog.info(f"info file found in {info_path}") return info_path else: continue else: + tdLog.info(f"info file not found in {info_path}") return None + else: + tdLog.info(f"no checkpoint{checkpointid} in {dirpath}") def get_dnode_info(self): ''' get a dict from vnode to dnode ''' self.vnode_dict = {} - sql = 'select dnode_id, vgroup_id from information_schema.ins_vnodes' + sql = 'select dnode_id, vgroup_id from information_schema.ins_vnodes where status = "leader"' result = tdSql.getResult(sql) for (dnode,vnode) in result: self.vnode_dict[vnode] = dnode @@ -68,15 +72,18 @@ class TDTestCase: while(True): if(self.check_vnodestate()): break + self.get_dnode_info() sql = 'select task_id, node_id, checkpoint_id, checkpoint_ver from information_schema.ins_stream_tasks where `level` = "source" or `level` = "agg" and node_type == "vnode"' for task_id, vnode, checkpoint_id, checkpoint_ver in tdSql.getResult(sql): dirpath = f"{cluster.dnodes[self.vnode_dict[vnode]-1].dataDir}/vnode/vnode{vnode}/" info_path = self.find_checkpoint_info_file(dirpath, checkpoint_id, task_id) if info_path is None: + tdLog.info(f"info path: {dirpath} is null") return False with open(info_path, 'r') as f: info_id, info_ver = f.read().split() if int(info_id) != int(checkpoint_id) or int(info_ver) != int(checkpoint_ver): + tdLog.info(f"infoId: {info_id}, checkpointId: {checkpoint_id}, infoVer: {info_ver}, checkpointVer: {checkpoint_ver}") return False return True @@ -131,4 +138,4 @@ class TDTestCase: tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) From 9cc2aec9d66d27aa5a9d1873291f9be797d064b1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Nov 2024 17:12:01 +0800 Subject: [PATCH 37/62] fix(stream): set correct value and open inputQ for stream if reset checkpoint. --- source/dnode/mnode/impl/src/mndStream.c | 7 ++++++- source/libs/stream/src/streamCheckpoint.c | 15 +++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 81db427afd..6336cd6e49 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2434,7 +2434,12 @@ static void doAddReportStreamTask(SArray *pList, int64_t reportChkptId, const SC mDebug("s-task:0x%x expired checkpoint-report msg in checkpoint-report list update from %" PRId64 "->%" PRId64, pReport->taskId, p->checkpointId, pReport->checkpointId); - memcpy(p, pReport, sizeof(STaskChkptInfo)); + // update the checkpoint report info + p->checkpointId = pReport->checkpointId; + p->ts = pReport->checkpointTs; + p->version = pReport->checkpointVer; + p->transId = pReport->transId; + p->dropHTask = pReport->dropHTask; } else { mWarn("taskId:0x%x already in checkpoint-report list", pReport->taskId); } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 7724d1c5ff..df13b9a585 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -255,8 +255,7 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check "the interrupted checkpoint", id, vgId, pBlock->srcTaskId); - streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); - return code; + return TSDB_CODE_STREAM_INVLD_CHKPT; } if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) { @@ -264,14 +263,14 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64 " discard", id, vgId, pActiveInfo->activeId, checkpointId); - return code; + return TSDB_CODE_STREAM_INVLD_CHKPT; } else { // checkpointId == pActiveInfo->activeId if (pActiveInfo->allUpstreamTriggerRecv == 1) { stDebug( "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, " "checkpointId:%" PRId64 " transId:%d", id, vgId, checkpointId, transId); - return code; + return TSDB_CODE_STREAM_INVLD_CHKPT; } if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { @@ -286,14 +285,14 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64 ", prev recvTs:%" PRId64 " discard", pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); - return code; + return TSDB_CODE_STREAM_INVLD_CHKPT; } } } } } - return 0; + return TSDB_CODE_SUCCESS; } int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { @@ -317,6 +316,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock code = doCheckBeforeHandleChkptTrigger(pTask, checkpointId, pBlock, transId); streamMutexUnlock(&pTask->lock); if (code) { + if (taskLevel != TASK_LEVEL__SOURCE) { // the checkpoint-trigger is discard, open the inputQ for upstream tasks + streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); + } + streamFreeQitem((SStreamQueueItem*)pBlock); return code; } From 628808c9a4bbfe83d8efff189b284442de1ed066 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Nov 2024 17:16:47 +0800 Subject: [PATCH 38/62] fix(stream): fix dead lock. --- source/libs/stream/src/streamDispatch.c | 2 ++ source/libs/stream/src/streamStartTask.c | 1 + 2 files changed, 3 insertions(+) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index e0fa199199..5807240f5e 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1170,6 +1170,7 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { if (taosArrayGetSize(pTask->upstreamInfo.pList) != num) { stError("s-task:%s invalid number of sent readyMsg:%d to upstream:%d", id, num, (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList)); + streamMutexUnlock(&pActiveInfo->lock); return TSDB_CODE_STREAM_INTERNAL_ERROR; } @@ -1412,6 +1413,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa if (size > 0) { STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, 0); if (pReady == NULL) { + streamMutexUnlock(&pActiveInfo->lock); return terrno; } diff --git a/source/libs/stream/src/streamStartTask.c b/source/libs/stream/src/streamStartTask.c index ed12687e41..9c16ff036e 100644 --- a/source/libs/stream/src/streamStartTask.c +++ b/source/libs/stream/src/streamStartTask.c @@ -433,6 +433,7 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { // send hb msg to mnode before closing all tasks. int32_t code = streamMetaSendMsgBeforeCloseTasks(pMeta, &pTaskList); if (code != TSDB_CODE_SUCCESS) { + streamMetaRUnLock(pMeta); return code; } From f304a92229ab4e4cc072165cdb8a5632a87682f6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Nov 2024 17:21:20 +0800 Subject: [PATCH 39/62] refactor: add new error code. --- include/util/taoserror.h | 1 + source/util/src/terror.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 9a8b39b84c..b563c186c3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -1010,6 +1010,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_STREAM_CONFLICT_EVENT TAOS_DEF_ERROR_CODE(0, 0x4106) #define TSDB_CODE_STREAM_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x4107) #define TSDB_CODE_STREAM_INPUTQ_FULL TAOS_DEF_ERROR_CODE(0, 0x4108) +#define TSDB_CODE_STREAM_INVLD_CHKPT TAOS_DEF_ERROR_CODE(0, 0x4109) // TDLite #define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x5100) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ce209cc718..ce754e8795 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -852,7 +852,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_IVLD_STATUS, "Invalid task status TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_CONFLICT_EVENT, "Stream conflict event") TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INTERNAL_ERROR, "Stream internal error") TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_NOT_LEADER, "Stream task not on leader vnode") -TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INPUTQ_FULL, "Task input queue is full") +TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INPUTQ_FULL, "Task input queue is full") +TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_INVLD_CHKPT, "Invalid checkpoint trigger msg") // TDLite TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS, "Invalid TDLite open flags") From b1981d309fefe53bd85a6bf1e5b034127fb2e372 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 15 Nov 2024 17:41:14 +0800 Subject: [PATCH 40/62] fix(stream): return error code. --- source/libs/stream/src/streamCheckpoint.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index df13b9a585..6c4c285ded 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -222,14 +222,14 @@ static int32_t doCheckBeforeHandleChkptTrigger(SStreamTask* pTask, int64_t check stError("s-task:%s vgId:%d current checkpointId:%" PRId64 " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); - return code; + return TSDB_CODE_STREAM_TASK_NOT_EXIST; } if (pActiveInfo->failedId >= checkpointId) { stError("s-task:%s vgId:%d checkpointId:%" PRId64 " transId:%d, has been marked failed, failedId:%" PRId64 " discard the checkpoint-trigger block", id, vgId, checkpointId, transId, pActiveInfo->failedId); - return code; + return TSDB_CODE_STREAM_TASK_NOT_EXIST; } if (pTask->chkInfo.checkpointId == checkpointId) { From 09fe252a31831e05707a139172b546b0ab1a1468 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 15 Nov 2024 17:52:06 +0800 Subject: [PATCH 41/62] remove duplicate group by cols --- include/libs/nodes/nodes.h | 1 + source/libs/nodes/src/nodesEqualFuncs.c | 9 +++++- source/libs/nodes/src/nodesUtilFuncs.c | 34 ++++++++++++++++++++ source/libs/parser/src/parTranslater.c | 2 -- source/libs/planner/src/planLogicCreater.c | 7 ++-- tests/system-test/2-query/group_partition.py | 13 ++++++++ 6 files changed, 61 insertions(+), 5 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 72dd3ef3e0..6384c536ce 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -174,6 +174,7 @@ char* nodesGetNameFromColumnNode(SNode* pNode); int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots); void nodesSortList(SNodeList** pList, int32_t (*)(SNode* pNode1, SNode* pNode2)); void destroyFuncParam(void* pFuncStruct); +int32_t nodesListDeduplicate(SNodeList** pList); #ifdef __cplusplus } diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 241da85267..891843761a 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -153,6 +153,12 @@ static bool caseWhenNodeEqual(const SCaseWhenNode* a, const SCaseWhenNode* b) { return true; } +static bool groupingSetNodeEqual(const SGroupingSetNode* a, const SGroupingSetNode* b) { + COMPARE_SCALAR_FIELD(groupingSetType); + COMPARE_NODE_LIST_FIELD(pParameterList); + return true; +} + bool nodesEqualNode(const SNode* a, const SNode* b) { if (a == b) { return true; @@ -181,10 +187,11 @@ bool nodesEqualNode(const SNode* a, const SNode* b) { return whenThenNodeEqual((const SWhenThenNode*)a, (const SWhenThenNode*)b); case QUERY_NODE_CASE_WHEN: return caseWhenNodeEqual((const SCaseWhenNode*)a, (const SCaseWhenNode*)b); + case QUERY_NODE_GROUPING_SET: + return groupingSetNodeEqual((const SGroupingSetNode*)a, (const SGroupingSetNode*)b); case QUERY_NODE_REAL_TABLE: case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_JOIN_TABLE: - case QUERY_NODE_GROUPING_SET: case QUERY_NODE_ORDER_BY_EXPR: case QUERY_NODE_LIMIT: return false; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index a9d0aa2924..4677b5d5f0 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2948,3 +2948,37 @@ void nodesSortList(SNodeList** pList, int32_t (*comp)(SNode* pNode1, SNode* pNod inSize *= 2; } } + +static SNode* nodesListFindNode(SNodeList* pList, SNode* pNode) { + SNode* pFound = NULL; + FOREACH(pFound, pList) { + if (nodesEqualNode(pFound, pNode)) { + break; + } + } + return pFound; +} + +int32_t nodesListDeduplicate(SNodeList** ppList) { + if (!ppList || LIST_LENGTH(*ppList) == 0) return TSDB_CODE_SUCCESS; + SNodeList* pTmp = NULL; + int32_t code = nodesMakeList(&pTmp); + if (TSDB_CODE_SUCCESS == code) { + SNode* pNode = NULL; + FOREACH(pNode, *ppList) { + SNode* pFound = nodesListFindNode(pTmp, pNode); + if (NULL == pFound) { + code = nodesCloneNode(pNode, &pFound); + if (TSDB_CODE_SUCCESS == code) code = nodesListStrictAppend(pTmp, pFound); + if (TSDB_CODE_SUCCESS != code) break; + } + } + } + if (TSDB_CODE_SUCCESS == code) { + nodesDestroyList(*ppList); + *ppList = pTmp; + } else { + nodesDestroyList(pTmp); + } + return code; +} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9bea3491c3..b77c349a78 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5531,7 +5531,6 @@ static int32_t translateGroupByList(STranslateContext* pCxt, SSelectStmt* pSelec SReplaceGroupByAliasCxt cxt = { .pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList}; nodesRewriteExprsPostOrder(pSelect->pGroupByList, translateGroupPartitionByImpl, &cxt); - return pCxt->errCode; } @@ -5543,7 +5542,6 @@ static int32_t translatePartitionByList(STranslateContext* pCxt, SSelectStmt* pS SReplaceGroupByAliasCxt cxt = { .pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList}; nodesRewriteExprsPostOrder(pSelect->pPartitionByList, translateGroupPartitionByImpl, &cxt); - return pCxt->errCode; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 34c83acee8..f4c4926ca1 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -838,8 +838,11 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } if (NULL != pSelect->pGroupByList) { - pAgg->pGroupKeys = NULL; - code = nodesCloneList(pSelect->pGroupByList, &pAgg->pGroupKeys); + code = nodesListDeduplicate(&pSelect->pGroupByList); + if (TSDB_CODE_SUCCESS == code) { + pAgg->pGroupKeys = NULL; + code = nodesCloneList(pSelect->pGroupByList, &pAgg->pGroupKeys); + } } // rewrite the expression in subsequent clauses diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 7ee528841c..d48df16231 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -437,6 +437,18 @@ class TDTestCase: tdSql.checkRows(10) tdSql.query(f"select const_col from (select 1 as const_col, count(c1) from {self.dbname}.{self.stable} t group by c1) partition by 1") tdSql.checkRows(10) + + def test_TD_32883(self): + sql = "select avg(c1), t9 from stb group by t9,t9, tbname" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(5) + sql = "select avg(c1), t10 from stb group by t10,t10, tbname" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(5) + sql = "select avg(c1), t10 from stb partition by t10,t10, tbname" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(5) + def run(self): tdSql.prepare() self.prepare_db() @@ -470,6 +482,7 @@ class TDTestCase: self.test_event_window(nonempty_tb_num) self.test_TS5567() + self.test_TD_32883() ## test old version before changed # self.test_groupby('group', 0, 0) From 2be0ca15d8653b6b593c38b9bcc90f1befa84690 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 15 Nov 2024 18:10:44 +0800 Subject: [PATCH 42/62] fix group_partition test --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 4 ++-- source/libs/parser/src/parCalcConst.c | 2 ++ tests/system-test/2-query/group_partition.py | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index f30f7eb310..cc896dfa80 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -6115,7 +6115,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { TSDB_CHECK_CODE(code, lino, _end); } - goto _end; + return code; } } @@ -6142,7 +6142,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { acquired = false; TSDB_CHECK_CODE(code, lino, _end); } - goto _end; + return code; } if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) { diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index e757ec8b24..778c34ff02 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -326,6 +326,7 @@ static int32_t calcConstProjections(SCalcConstContext* pCxt, SSelectStmt* pSelec static int32_t calcConstGroupBy(SCalcConstContext* pCxt, SSelectStmt* pSelect) { int32_t code = calcConstList(pSelect->pGroupByList); +#if 0 if (TSDB_CODE_SUCCESS == code) { SNode* pNode = NULL; FOREACH(pNode, pSelect->pGroupByList) { @@ -338,6 +339,7 @@ static int32_t calcConstGroupBy(SCalcConstContext* pCxt, SSelectStmt* pSelect) { } NODES_DESTORY_LIST(pSelect->pGroupByList); } +#endif return code; } diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index d48df16231..b77cf3f8b7 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -422,19 +422,19 @@ class TDTestCase: def test_TS5567(self): tdSql.query(f"select const_col from (select 1 as const_col from {self.dbname}.{self.stable}) t group by const_col") - tdSql.checkRows(50) + tdSql.checkRows(1) tdSql.query(f"select const_col from (select 1 as const_col from {self.dbname}.{self.stable}) t partition by const_col") tdSql.checkRows(50) tdSql.query(f"select const_col from (select 1 as const_col, count(c1) from {self.dbname}.{self.stable} t group by c1) group by const_col") - tdSql.checkRows(10) + tdSql.checkRows(1) tdSql.query(f"select const_col from (select 1 as const_col, count(c1) from {self.dbname}.{self.stable} t group by c1) partition by const_col") tdSql.checkRows(10) tdSql.query(f"select const_col as c_c from (select 1 as const_col from {self.dbname}.{self.stable}) t group by c_c") - tdSql.checkRows(50) + tdSql.checkRows(1) tdSql.query(f"select const_col as c_c from (select 1 as const_col from {self.dbname}.{self.stable}) t partition by c_c") tdSql.checkRows(50) tdSql.query(f"select const_col from (select 1 as const_col, count(c1) from {self.dbname}.{self.stable} t group by c1) group by 1") - tdSql.checkRows(10) + tdSql.checkRows(1) tdSql.query(f"select const_col from (select 1 as const_col, count(c1) from {self.dbname}.{self.stable} t group by c1) partition by 1") tdSql.checkRows(10) From 55239964a979cd44f5ca965b4aa92a9e72b81a0d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 16 Nov 2024 01:56:33 +0800 Subject: [PATCH 43/62] refactor: injection error. --- source/libs/stream/src/streamCheckpoint.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 6c4c285ded..2280b7f06f 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -362,6 +362,10 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock } } +#if 0 + taosMsleep(20*1000); +#endif + if (taskLevel == TASK_LEVEL__SOURCE) { int8_t type = pTask->outputInfo.type; pActiveInfo->allUpstreamTriggerRecv = 1; From b589f9089c3956ddb354ee5666036b6bc06b6112 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 18 Nov 2024 09:33:43 +0800 Subject: [PATCH 44/62] add test case --- source/libs/executor/src/scanoperator.c | 1 - source/libs/parser/src/parTranslater.c | 2 ++ tests/system-test/2-query/group_partition.py | 9 ++++++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ecb6bd7aee..eac4f87b14 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3043,7 +3043,6 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } if (code) { - blockDataFreeRes((SSDataBlock*)pBlock); QUERY_CHECK_CODE(code, lino, _end); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b77c349a78..9bea3491c3 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5531,6 +5531,7 @@ static int32_t translateGroupByList(STranslateContext* pCxt, SSelectStmt* pSelec SReplaceGroupByAliasCxt cxt = { .pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList}; nodesRewriteExprsPostOrder(pSelect->pGroupByList, translateGroupPartitionByImpl, &cxt); + return pCxt->errCode; } @@ -5542,6 +5543,7 @@ static int32_t translatePartitionByList(STranslateContext* pCxt, SSelectStmt* pS SReplaceGroupByAliasCxt cxt = { .pTranslateCxt = pCxt, .pProjectionList = pSelect->pProjectionList}; nodesRewriteExprsPostOrder(pSelect->pPartitionByList, translateGroupPartitionByImpl, &cxt); + return pCxt->errCode; } diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index b77cf3f8b7..74f5e86267 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -439,13 +439,16 @@ class TDTestCase: tdSql.checkRows(10) def test_TD_32883(self): - sql = "select avg(c1), t9 from stb group by t9,t9, tbname" + sql = "select avg(c1), t9 from db.stb group by t9,t9, tbname" tdSql.query(sql, queryTimes=1) tdSql.checkRows(5) - sql = "select avg(c1), t10 from stb group by t10,t10, tbname" + sql = "select avg(c1), t10 from db.stb group by t10,t10, tbname" tdSql.query(sql, queryTimes=1) tdSql.checkRows(5) - sql = "select avg(c1), t10 from stb partition by t10,t10, tbname" + sql = "select avg(c1), t10 from db.stb partition by t10,t10, tbname" + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(5) + sql = "select avg(c1), concat(t9,t10) from db.stb group by concat(t9,t10), concat(t9,t10),tbname" tdSql.query(sql, queryTimes=1) tdSql.checkRows(5) From 6d8a5b892ee5abe10b94b9d10a05d95babcd200a Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 18 Nov 2024 09:37:20 +0800 Subject: [PATCH 45/62] rename doc 28-index.md ->28-tsma.md --- docs/zh/14-reference/03-taos-sql/{28-index.md => 28-tsma.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/zh/14-reference/03-taos-sql/{28-index.md => 28-tsma.md} (100%) diff --git a/docs/zh/14-reference/03-taos-sql/28-index.md b/docs/zh/14-reference/03-taos-sql/28-tsma.md similarity index 100% rename from docs/zh/14-reference/03-taos-sql/28-index.md rename to docs/zh/14-reference/03-taos-sql/28-tsma.md From c6657653c68e5049f64247d38acfeafccb7c97a6 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:42:15 +0800 Subject: [PATCH 46/62] docs/Update 03-stream.md --- docs/zh/06-advanced/03-stream.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/06-advanced/03-stream.md b/docs/zh/06-advanced/03-stream.md index c47831dde3..c26924561c 100644 --- a/docs/zh/06-advanced/03-stream.md +++ b/docs/zh/06-advanced/03-stream.md @@ -124,7 +124,7 @@ create stream if not exists count_history_s fill_history 1 into count_history as 窗口关闭是由事件时间决定的，如事件流中断、或持续延迟，此时事件时间无法更新，可能导致无法得到最新的计算结果。 -因此，流计算提供了以事件时间结合处理时间计算的 MAX_DELAY 触发模式。MAX_DELAY 模式在窗口关闭时会立即触发计算。此外，当数据写入后，计算触发的时间超过 max delay 指定的时间，则立即触发计算。 +因此，流计算提供了以事件时间结合处理时间计算的 MAX_DELAY 触发模式：MAX_DELAY 模式在窗口关闭时会立即触发计算，它的单位可以自行指定，具体单位：a（毫秒）、s（秒）、m（分）、h（小时）、d（天）、w（周）。此外，当数据写入后，计算触发的时间超过 MAX_DELAY 指定的时间，则立即触发计算。 ### 流计算的窗口关闭 @@ -259,4 +259,4 @@ flush database test1; 5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0 -6.启动 taosd \ No newline at end of file +6.启动 taosd From c954d0ac7addd1e8f395c8a0d9f006c97a0e2b59 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 18 Nov 2024 11:20:32 +0800 Subject: [PATCH 47/62] fix:skip begin&end snapshot while wal level = 0. --- source/libs/wal/src/walWrite.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 1a9652b3bb..66ead2fd26 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -376,6 +376,10 @@ static FORCE_INLINE int32_t walCheckAndRoll(SWal *pWal) { int32_t walBeginSnapshot(SWal *pWal, int64_t ver, int64_t logRetention) { int32_t code = 0; + if (pWal->cfg.level == TAOS_WAL_SKIP) { + TAOS_RETURN(TSDB_CODE_SUCCESS); + } + if (logRetention < 0) { TAOS_RETURN(TSDB_CODE_FAILED); } @@ -404,6 +408,10 @@ _exit: int32_t walEndSnapshot(SWal *pWal) { int32_t code = 0, lino = 0; + if (pWal->cfg.level == TAOS_WAL_SKIP) { + TAOS_RETURN(TSDB_CODE_SUCCESS); + } + TAOS_UNUSED(taosThreadRwlockWrlock(&pWal->mutex)); int64_t ver = pWal->vers.verInSnapshotting; From 7348cbe5ef1394e10fc0784c8570582d06d2367b Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 18 Nov 2024 12:31:38 +0800 Subject: [PATCH 48/62] fix/TD-32914-free-vnode-after-lock --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 2ee607949d..17a99c1806 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -578,16 +578,13 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { } void vmRemoveFromCreatingHash(SVnodeMgmt *pMgmt, int32_t vgId) { - (void)taosThreadRwlockWrlock(&pMgmt->lock); SVnodeObj *pOld = NULL; + + (void)taosThreadRwlockWrlock(&pMgmt->lock); int32_t r = taosHashGetDup(pMgmt->creatingHash, &vgId, sizeof(int32_t), (void *)&pOld); if (r != 0) { dError("vgId:%d, failed to get vnode from creating Hash", vgId); } - if (pOld) { - dTrace("vgId:%d, free vnode pOld:%p", vgId, &pOld); - vmFreeVnodeObj(&pOld); - } dTrace("vgId:%d, remove from creating Hash", vgId); r = taosHashRemove(pMgmt->creatingHash, &vgId, sizeof(int32_t)); if (r != 0) { @@ -595,6 +592,11 @@ void vmRemoveFromCreatingHash(SVnodeMgmt *pMgmt, int32_t vgId) { } (void)taosThreadRwlockUnlock(&pMgmt->lock); + if (pOld) { + dTrace("vgId:%d, free vnode pOld:%p", vgId, &pOld); + vmFreeVnodeObj(&pOld); + } + _OVER: if (r != 0) { dError("vgId:%d, failed to remove vnode from creatingHash since %s", vgId, tstrerror(r)); From f0c182988cd23c3dc8561559a351b553e0fd6024 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Mon, 18 Nov 2024 13:49:03 +0800 Subject: [PATCH 49/62] Add test case for begin&end snapshot while wal_level =0 . --- source/libs/wal/test/walMetaTest.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 3e6fab116f..a958ad74e0 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -510,4 +510,27 @@ TEST_F(WalSkipLevel, restart) { TearDown(); SetUp(); +} + +TEST_F(WalSkipLevel, roll) { + int code; + int i; + for (i = 0; i < 100; i++) { + code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + code = walCommit(pWal, i); + } + walBeginSnapshot(pWal, i - 1, 0); + walEndSnapshot(pWal); + code = walAppendLog(pWal, 5, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_NE(code, 0); + for (; i < 200; i++) { + code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); + ASSERT_EQ(code, 0); + code = walCommit(pWal, i); + } + code = walBeginSnapshot(pWal, i - 1, 0); + ASSERT_EQ(code, 0); + code = walEndSnapshot(pWal); + ASSERT_EQ(code, 0); } \ No newline at end of file From 6c18289de0f0a85e48f37abba6f3c6bebe660f10 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 13 Nov 2024 17:01:50 +0800 Subject: [PATCH 50/62] fix tsbs perf issue --- include/libs/catalog/catalog.h | 1 + include/util/tbuffer.inc | 18 +++++- source/dnode/vnode/src/tsdb/tsdbDataFileRW.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 11 +++- source/libs/catalog/inc/catalogInt.h | 1 + source/libs/catalog/src/ctgAsync.c | 32 +++++----- source/libs/function/src/builtinsimpl.c | 61 ++++++++++---------- source/libs/parser/inc/parUtil.h | 1 + source/libs/parser/src/parAstParser.c | 3 +- source/libs/parser/src/parUtil.c | 1 + 10 files changed, 80 insertions(+), 51 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index df3f87973f..7c6f02513e 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -102,6 +102,7 @@ typedef struct SCatalogReq { bool svrVerRequired; bool forceUpdate; bool cloned; + bool forceFetchViewMeta; } SCatalogReq; typedef struct SMetaRes { diff --git a/include/util/tbuffer.inc b/include/util/tbuffer.inc index 39090fb7fa..633517ca58 100644 --- a/include/util/tbuffer.inc +++ b/include/util/tbuffer.inc @@ -186,11 +186,25 @@ static int32_t tBufferGetI16(SBufferReader *reader, int16_t *value) { } static int32_t tBufferGetI32(SBufferReader *reader, int32_t *value) { - return tBufferGet(reader, sizeof(*value), value); + if (reader->offset + sizeof(int32_t) > reader->buffer->size) { + return TSDB_CODE_OUT_OF_RANGE; + } + if (value) { + *value = *(int32_t*)BR_PTR(reader); + } + reader->offset += sizeof(int32_t); + return 0; } static int32_t tBufferGetI64(SBufferReader *reader, int64_t *value) { - return tBufferGet(reader, sizeof(*value), value); + if (reader->offset + sizeof(int64_t) > reader->buffer->size) { + return TSDB_CODE_OUT_OF_RANGE; + } + if (value) { + *value = *(int64_t*)BR_PTR(reader); + } + reader->offset += sizeof(int64_t); + return 0; } static int32_t tBufferGetU8(SBufferReader *reader, uint8_t *value) { return tBufferGet(reader, sizeof(*value), value); } diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index 720ba68414..f51ffe0c83 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -972,7 +972,7 @@ static int32_t tsdbDataFileWriteBrinRecord(SDataFileWriter *writer, const SBrinR break; } - if ((writer->brinBlock->numOfRecords) >= writer->config->maxRow) { + if ((writer->brinBlock->numOfRecords) >= 256) { TAOS_CHECK_GOTO(tsdbDataFileWriteBrinBlock(writer), &lino, _exit); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index f30f7eb310..2b1be747dd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -836,6 +836,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead pList = &pReader->status.uidList; int32_t i = 0; + int32_t j = 0; while (i < TARRAY2_SIZE(pBlkArray)) { pBrinBlk = &pBlkArray->data[i]; if (pBrinBlk->maxTbid.suid < pReader->info.suid) { @@ -851,7 +852,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead (pBrinBlk->minTbid.suid <= pReader->info.suid) && (pBrinBlk->maxTbid.suid >= pReader->info.suid), code, lino, _end, TSDB_CODE_INTERNAL_ERROR); - if (pBrinBlk->maxTbid.suid == pReader->info.suid && pBrinBlk->maxTbid.uid < pList->tableUidList[0]) { + if (pBrinBlk->maxTbid.suid == pReader->info.suid && pBrinBlk->maxTbid.uid < pList->tableUidList[j]) { i += 1; continue; } @@ -864,6 +865,14 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead TSDB_CHECK_NULL(p1, code, lino, _end, terrno); i += 1; + if (pBrinBlk->maxTbid.suid == pReader->info.suid) { + while (j < numOfTables && pList->tableUidList[j] < pBrinBlk->maxTbid.uid) { + j++; + } + if (j >= numOfTables) { + break; + } + } } et2 = taosGetTimestampUs(); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index e757163ba8..bfc5ca9142 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -271,6 +271,7 @@ typedef struct SCtgViewsCtx { SArray* pNames; SArray* pResList; SArray* pFetchs; + bool forceFetch; } SCtgViewsCtx; typedef enum { diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index c1dcdf2741..9bfb4102aa 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -20,6 +20,11 @@ #include "tref.h" #include "trpc.h" +typedef struct SCtgViewTaskParam { + bool forceFetch; + SArray* pTableReqs; +} SCtgViewTaskParam; + void ctgIsTaskDone(SCtgJob* pJob, CTG_TASK_TYPE type, bool* done) { SCtgTask* pTask = NULL; @@ -500,7 +505,7 @@ int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) { int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgTask task = {0}; - + SCtgViewTaskParam* p = param; task.type = CTG_TASK_GET_VIEW; task.taskId = taskIdx; task.pJob = pJob; @@ -511,7 +516,8 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { } SCtgViewsCtx* ctx = task.taskCtx; - ctx->pNames = param; + ctx->pNames = p->pTableReqs; + ctx->forceFetch = p->forceFetch; ctx->pResList = taosArrayInit(pJob->viewNum, sizeof(SMetaRes)); if (NULL == ctx->pResList) { qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, @@ -849,13 +855,12 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg); int32_t tbTagNum = (int32_t)taosArrayGetSize(pReq->pTableTag); int32_t viewNum = (int32_t)ctgGetTablesReqNum(pReq->pView); - int32_t tbTsmaNum = (int32_t)taosArrayGetSize(pReq->pTableTSMAs); + int32_t tbTsmaNum = tsQuerySmaOptimize ? (int32_t)taosArrayGetSize(pReq->pTableTSMAs) : 0; int32_t tsmaNum = (int32_t)taosArrayGetSize(pReq->pTSMAs); int32_t tbNameNum = (int32_t)ctgGetTablesReqNum(pReq->pTableName); int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum + viewNum + tbTsmaNum + tbNameNum; - *job = taosMemoryCalloc(1, sizeof(SCtgJob)); if (NULL == *job) { ctgError("failed to calloc, size:%d,QID:0x%" PRIx64, (int32_t)sizeof(SCtgJob), pConn->requestId); @@ -1014,7 +1019,8 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const } if (viewNum > 0) { - CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_VIEW, pReq->pView, NULL)); + SCtgViewTaskParam param = {.forceFetch = pReq->forceFetchViewMeta, .pTableReqs = pReq->pView}; + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_VIEW, ¶m, NULL)); } if (tbTsmaNum > 0) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TSMA, pReq->pTableTSMAs, NULL)); @@ -3712,16 +3718,14 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { bool tbMetaDone = false; SName* pName = NULL; - /* - ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone); - if (tbMetaDone) { - CTG_ERR_RET(ctgBuildViewNullRes(pTask, pCtx)); - TSWAP(pTask->res, pCtx->pResList); + ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone); + if (tbMetaDone && !pCtx->forceFetch) { + CTG_ERR_RET(ctgBuildViewNullRes(pTask, pCtx)); + TSWAP(pTask->res, pCtx->pResList); - CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); - return TSDB_CODE_SUCCESS; - } - */ + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } int32_t dbNum = taosArrayGetSize(pCtx->pNames); int32_t fetchIdx = 0; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c2e2e9c17c..1b755e0f38 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3037,61 +3037,58 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { TSKEY startKey = getRowPTs(pInput->pPTS, 0); TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); -#if 0 - int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - - // the optimized version only valid if all tuples in one block are monotonious increasing or descreasing. - // this assumption is NOT always works if project operator exists in downstream. - if (blockDataOrder == TSDB_ORDER_ASC) { + if (pCtx->order == TSDB_ORDER_ASC && !pCtx->hasPrimaryKey) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveLastrow(pCtx, data, i, cts, pInfo); + int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo); + if (code != TSDB_CODE_SUCCESS) return code; } break; } - } else { // descending order + } else if (!pCtx->hasPrimaryKey && pCtx->order == TSDB_ORDER_DESC) { + // the optimized version only valid if all tuples in one block are monotonious increasing or descreasing. + // this assumption is NOT always works if project operator exists in downstream. for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); numOfElems++; if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { - doSaveLastrow(pCtx, data, i, cts, pInfo); + int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo); + if (code != TSDB_CODE_SUCCESS) return code; } break; } - } -#else + } else { + int64_t* pts = (int64_t*)pInput->pPTS->pData; + int from = -1; + int32_t i = -1; + while (funcInputGetNextRowIndex(pInput, from, false, &i, &from)) { + bool isNull = colDataIsNull(pInputCol, pInput->numOfRows, i, NULL); + char* data = isNull ? NULL : colDataGetData(pInputCol, i); + TSKEY cts = pts[i]; - int64_t* pts = (int64_t*)pInput->pPTS->pData; - int from = -1; - int32_t i = -1; - while (funcInputGetNextRowIndex(pInput, from, false, &i, &from)) { - bool isNull = colDataIsNull(pInputCol, pInput->numOfRows, i, NULL); - char* data = isNull ? NULL : colDataGetData(pInputCol, i); - TSKEY cts = pts[i]; - - numOfElems++; - char* pkData = NULL; - if (pCtx->hasPrimaryKey) { - pkData = colDataGetData(pkCol, i); - } - if (pResInfo->numOfRes == 0 || pInfo->ts < cts || - (pInfo->ts == pts[i] && pkCompareFn && pkCompareFn(pkData, pInfo->pkData) < 0)) { - int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; + numOfElems++; + char* pkData = NULL; + if (pCtx->hasPrimaryKey) { + pkData = colDataGetData(pkCol, i); + } + if (pResInfo->numOfRes == 0 || pInfo->ts < cts || + (pInfo->ts == pts[i] && pkCompareFn && pkCompareFn(pkData, pInfo->pkData) < 0)) { + int32_t code = doSaveLastrow(pCtx, data, i, cts, pInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + pResInfo->numOfRes = 1; } - pResInfo->numOfRes = 1; } - } -#endif + } SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 857c7604a9..7298b04eb0 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -115,6 +115,7 @@ typedef struct SParseMetaCache { SHashObj* pTableName; // key is tbFUid, elements is STableMeta*(append with tbName) SArray* pDnodes; // element is SEpSet bool dnodeRequired; + bool forceFetchViewMeta; } SParseMetaCache; int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...); diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index eecc04658b..b78e10768f 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -810,7 +810,7 @@ static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowC if (TSDB_CODE_SUCCESS == code) { code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache); } - + pCxt->pMetaCache->forceFetchViewMeta = true; return code; } @@ -888,6 +888,7 @@ static int32_t collectMetaKeyFromCreateViewStmt(SCollectMetaKeyCxt* pCxt, SCreat static int32_t collectMetaKeyFromDropViewStmt(SCollectMetaKeyCxt* pCxt, SDropViewStmt* pStmt) { int32_t code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName, AUTH_TYPE_ALTER, pCxt->pMetaCache); + pCxt->pMetaCache->forceFetchViewMeta = true; return code; } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index e35eea9e72..44e44982a3 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -817,6 +817,7 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog } #endif pCatalogReq->dNodeRequired = pMetaCache->dnodeRequired; + pCatalogReq->forceFetchViewMeta = pMetaCache->forceFetchViewMeta; return code; } From a6ab76b02e97186f91226d958fdcacea1a8c2856 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 14 Nov 2024 10:50:27 +0800 Subject: [PATCH 51/62] fix lastrow function tests --- source/libs/function/src/builtinsimpl.c | 6 ++++-- source/libs/planner/src/planOptimizer.c | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1b755e0f38..83227dea9e 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3039,7 +3039,8 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { if (pCtx->order == TSDB_ORDER_ASC && !pCtx->hasPrimaryKey) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { - char* data = colDataGetData(pInputCol, i); + bool isNull = colDataIsNull(pInputCol, pInput->numOfRows, i, NULL); + char* data = isNull ? NULL : colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); numOfElems++; @@ -3054,7 +3055,8 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { // the optimized version only valid if all tuples in one block are monotonious increasing or descreasing. // this assumption is NOT always works if project operator exists in downstream. for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - char* data = colDataGetData(pInputCol, i); + bool isNull = colDataIsNull(pInputCol, pInput->numOfRows, i, NULL); + char* data = isNull ? NULL : colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); numOfElems++; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 39024731ed..a1809ff137 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -204,6 +204,7 @@ static void optSetParentOrder(SLogicNode* pNode, EOrder order, SLogicNode* pNode // case QUERY_NODE_LOGIC_PLAN_WINDOW: case QUERY_NODE_LOGIC_PLAN_AGG: case QUERY_NODE_LOGIC_PLAN_SORT: + case QUERY_NODE_LOGIC_PLAN_FILL: if (pNode == pNodeForcePropagate) { pNode->outputTsOrder = order; break; From 2d9b78216b10896f5bdc7137df581f48791fe212 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Mon, 18 Nov 2024 14:52:21 +0800 Subject: [PATCH 52/62] Create pull_request_template.md --- .github/pull_request_template.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000..7d877987ac --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,5 @@ +# Pull Request Checklist + +- [ ] Is the user manual updated? +- [ ] Are the test cases passed and automated? +- [ ] Is there no significant decrease in test coverage? From f5c66cec19d01f74d0595c59b0d9dc0fa1522806 Mon Sep 17 00:00:00 2001 From: Jinqing Kuang Date: Mon, 18 Nov 2024 15:04:40 +0800 Subject: [PATCH 53/62] fix(query)[TD-32861]. fix that WHERE condition not work with INTERP function Previously, a statement with INTERP would force underlying TABLE SCAN operator to scan all data within the RANGE clause, causing the time range constraint in the WHERE condition to be ignored. This fix ensures that the TABLE SCAN respects both the RANGE clause and WHERE condition, improving query accuracy and performance. --- include/common/tcommon.h | 1 + source/dnode/vnode/src/tsdb/tsdbRead2.c | 19 +- source/libs/executor/src/timesliceoperator.c | 47 +- tests/army/query/function/ans/interp.csv | 649 +++++++++++++++++++ tests/army/query/function/in/interp.in | 50 ++ tests/army/query/function/test_interp.py | 11 + 6 files changed, 766 insertions(+), 11 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index ea764e6760..111c9dde3c 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -250,6 +250,7 @@ typedef struct SQueryTableDataCond { int32_t type; // data block load type: bool skipRollup; STimeWindow twindows; + STimeWindow extTwindows[2]; int64_t startVersion; int64_t endVersion; bool notLoadData; // response the actual data, not only the rows in the attribute of info.row of ssdatablock diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index f30f7eb310..ae1488f11b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1134,7 +1134,12 @@ static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInf *pInfo = NULL; size_t num = TARRAY_SIZE(pBlockIter->blockList); - TSDB_CHECK_CONDITION(num != 0, code, lino, _end, TSDB_CODE_INVALID_PARA); + if (num == 0) { + // Some callers would attempt to call this function. Filter out certain normal cases and return directly to avoid + // generating excessive unnecessary error logs. + TSDB_CHECK_CONDITION(num == pBlockIter->numOfBlocks, code, lino, _end, TSDB_CODE_INVALID_PARA); + return TSDB_CODE_INVALID_PARA; + } *pInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index); TSDB_CHECK_NULL(*pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA); @@ -5530,12 +5535,10 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi // update the SQueryTableDataCond to create inner reader int32_t order = pCond->order; if (order == TSDB_ORDER_ASC) { - pCond->twindows.ekey = window.skey - 1; - pCond->twindows.skey = INT64_MIN; + pCond->twindows = pCond->extTwindows[0]; pCond->order = TSDB_ORDER_DESC; } else { - pCond->twindows.skey = window.ekey + 1; - pCond->twindows.ekey = INT64_MAX; + pCond->twindows = pCond->extTwindows[1]; pCond->order = TSDB_ORDER_ASC; } @@ -5544,11 +5547,9 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi TSDB_CHECK_CODE(code, lino, _end); if (order == TSDB_ORDER_ASC) { - pCond->twindows.skey = window.ekey + 1; - pCond->twindows.ekey = INT64_MAX; + pCond->twindows = pCond->extTwindows[1]; } else { - pCond->twindows.skey = INT64_MIN; - pCond->twindows.ekey = window.ekey - 1; + pCond->twindows = pCond->extTwindows[0]; } pCond->order = order; diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index f77aa8f34a..50deba932f 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -1131,6 +1131,47 @@ static int32_t extractPkColumnFromFuncs(SNodeList* pFuncs, bool* pHasPk, SColumn return TSDB_CODE_SUCCESS; } +/** + * @brief Determine the actual time range for reading data based on the RANGE clause and the WHERE conditions. + * @param[in] cond The range specified by WHERE condition. + * @param[in] range The range specified by RANGE clause. + * @param[out] twindow The range to be read in DESC order, and only one record is needed. + * @param[out] extTwindow The external range to read for only one record, which is used for FILL clause. + * @note `cond` and `twindow` may be the same address. + */ +static int32_t getQueryExtWindow(const STimeWindow* cond, const STimeWindow* range, STimeWindow* twindow, + STimeWindow* extTwindows) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + STimeWindow tempWindow; + + if (cond->skey > cond->ekey || range->skey > range->ekey) { + *twindow = extTwindows[0] = extTwindows[1] = TSWINDOW_DESC_INITIALIZER; + return code; + } + + if (range->ekey < cond->skey) { + extTwindows[1] = *cond; + *twindow = extTwindows[0] = TSWINDOW_DESC_INITIALIZER; + return code; + } + + if (cond->ekey < range->skey) { + extTwindows[0] = *cond; + *twindow = extTwindows[1] = TSWINDOW_DESC_INITIALIZER; + return code; + } + + // Only scan data in the time range intersecion. + extTwindows[0] = extTwindows[1] = *cond; + twindow->skey = TMAX(cond->skey, range->skey); + twindow->ekey = TMIN(cond->ekey, range->ekey); + extTwindows[0].ekey = twindow->skey - 1; + extTwindows[1].skey = twindow->ekey + 1; + + return code; +} + int32_t createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { QRY_PARAM_CHECK(pOptrInfo); @@ -1206,8 +1247,10 @@ int32_t createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyN if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info; - pScanInfo->base.cond.twindows = pInfo->win; - pScanInfo->base.cond.type = TIMEWINDOW_RANGE_EXTERNAL; + SQueryTableDataCond *cond = &pScanInfo->base.cond; + cond->type = TIMEWINDOW_RANGE_EXTERNAL; + code = getQueryExtWindow(&cond->twindows, &pInfo->win, &cond->twindows, cond->extTwindows); + QUERY_CHECK_CODE(code, lino, _error); } setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, diff --git a/tests/army/query/function/ans/interp.csv b/tests/army/query/function/ans/interp.csv index e1ba236aa1..3eaccd887a 100644 --- a/tests/army/query/function/ans/interp.csv +++ b/tests/army/query/function/ans/interp.csv @@ -366,3 +366,652 @@ taos> select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _i 2020-02-01 00:00:16.000 | td32727 | 10 | 10 | true | 1 | 2020-02-01 00:00:16.000 | td32727 | 15 | 15 | true | 1 | +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-02 00:00:00' and '2020-01-01 00:00:00' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(null); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-02 00:00:00' and '2020-01-01 00:00:00' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(value, 1); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-02 00:00:00' and '2020-01-01 00:00:00' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(prev); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-02 00:00:00' and '2020-01-01 00:00:00' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(next); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-02 00:00:00' and '2020-01-01 00:00:00' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(linear); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-02 00:00:00' range('2020-01-01 00:00:30', '2020-01-01 00:00:00') every(1s) fill(null); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-02 00:00:00' range('2020-01-01 00:00:30', '2020-01-01 00:00:00') every(1s) fill(value, 1); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-02 00:00:00' range('2020-01-01 00:00:30', '2020-01-01 00:00:00') every(1s) fill(prev); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-02 00:00:00' range('2020-01-01 00:00:30', '2020-01-01 00:00:00') every(1s) fill(next); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-02 00:00:00' range('2020-01-01 00:00:30', '2020-01-01 00:00:00') every(1s) fill(linear); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:20' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(null); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:20' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(value, 1); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:20' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(prev); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:20' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(next); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:20' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(linear); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(null); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:21.000 | false | 21 | + 2020-01-01 00:00:22.000 | true | NULL | + 2020-01-01 00:00:23.000 | true | NULL | + 2020-01-01 00:00:24.000 | true | NULL | + 2020-01-01 00:00:25.000 | true | NULL | + 2020-01-01 00:00:26.000 | true | NULL | + 2020-01-01 00:00:27.000 | true | NULL | + 2020-01-01 00:00:28.000 | true | NULL | + 2020-01-01 00:00:29.000 | true | NULL | + 2020-01-01 00:00:30.000 | true | NULL | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(value, 1); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:21.000 | false | 21 | + 2020-01-01 00:00:22.000 | true | 1 | + 2020-01-01 00:00:23.000 | true | 1 | + 2020-01-01 00:00:24.000 | true | 1 | + 2020-01-01 00:00:25.000 | true | 1 | + 2020-01-01 00:00:26.000 | true | 1 | + 2020-01-01 00:00:27.000 | true | 1 | + 2020-01-01 00:00:28.000 | true | 1 | + 2020-01-01 00:00:29.000 | true | 1 | + 2020-01-01 00:00:30.000 | true | 1 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(prev); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:21.000 | false | 21 | + 2020-01-01 00:00:22.000 | true | 21 | + 2020-01-01 00:00:23.000 | true | 21 | + 2020-01-01 00:00:24.000 | true | 21 | + 2020-01-01 00:00:25.000 | true | 21 | + 2020-01-01 00:00:26.000 | true | 21 | + 2020-01-01 00:00:27.000 | true | 21 | + 2020-01-01 00:00:28.000 | true | 21 | + 2020-01-01 00:00:29.000 | true | 21 | + 2020-01-01 00:00:30.000 | true | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(next); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(linear); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:15', '2020-01-01 00:00:30') every(1s) fill(null); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | NULL | + 2020-01-01 00:00:17.000 | true | NULL | + 2020-01-01 00:00:18.000 | true | NULL | + 2020-01-01 00:00:19.000 | true | NULL | + 2020-01-01 00:00:20.000 | true | NULL | + 2020-01-01 00:00:21.000 | false | 21 | + 2020-01-01 00:00:22.000 | true | NULL | + 2020-01-01 00:00:23.000 | true | NULL | + 2020-01-01 00:00:24.000 | true | NULL | + 2020-01-01 00:00:25.000 | true | NULL | + 2020-01-01 00:00:26.000 | true | NULL | + 2020-01-01 00:00:27.000 | true | NULL | + 2020-01-01 00:00:28.000 | true | NULL | + 2020-01-01 00:00:29.000 | true | NULL | + 2020-01-01 00:00:30.000 | true | NULL | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:15', '2020-01-01 00:00:30') every(1s) fill(value, 1); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 1 | + 2020-01-01 00:00:17.000 | true | 1 | + 2020-01-01 00:00:18.000 | true | 1 | + 2020-01-01 00:00:19.000 | true | 1 | + 2020-01-01 00:00:20.000 | true | 1 | + 2020-01-01 00:00:21.000 | false | 21 | + 2020-01-01 00:00:22.000 | true | 1 | + 2020-01-01 00:00:23.000 | true | 1 | + 2020-01-01 00:00:24.000 | true | 1 | + 2020-01-01 00:00:25.000 | true | 1 | + 2020-01-01 00:00:26.000 | true | 1 | + 2020-01-01 00:00:27.000 | true | 1 | + 2020-01-01 00:00:28.000 | true | 1 | + 2020-01-01 00:00:29.000 | true | 1 | + 2020-01-01 00:00:30.000 | true | 1 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:15', '2020-01-01 00:00:30') every(1s) fill(prev); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 15 | + 2020-01-01 00:00:17.000 | true | 15 | + 2020-01-01 00:00:18.000 | true | 15 | + 2020-01-01 00:00:19.000 | true | 15 | + 2020-01-01 00:00:20.000 | true | 15 | + 2020-01-01 00:00:21.000 | false | 21 | + 2020-01-01 00:00:22.000 | true | 21 | + 2020-01-01 00:00:23.000 | true | 21 | + 2020-01-01 00:00:24.000 | true | 21 | + 2020-01-01 00:00:25.000 | true | 21 | + 2020-01-01 00:00:26.000 | true | 21 | + 2020-01-01 00:00:27.000 | true | 21 | + 2020-01-01 00:00:28.000 | true | 21 | + 2020-01-01 00:00:29.000 | true | 21 | + 2020-01-01 00:00:30.000 | true | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:15', '2020-01-01 00:00:30') every(1s) fill(next); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 21 | + 2020-01-01 00:00:17.000 | true | 21 | + 2020-01-01 00:00:18.000 | true | 21 | + 2020-01-01 00:00:19.000 | true | 21 | + 2020-01-01 00:00:20.000 | true | 21 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:15', '2020-01-01 00:00:30') every(1s) fill(linear); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 16 | + 2020-01-01 00:00:17.000 | true | 17 | + 2020-01-01 00:00:18.000 | true | 18 | + 2020-01-01 00:00:19.000 | true | 19 | + 2020-01-01 00:00:20.000 | true | 20 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(null); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | false | 0 | + 2020-01-01 00:00:01.000 | false | 1 | + 2020-01-01 00:00:02.000 | true | NULL | + 2020-01-01 00:00:03.000 | false | 3 | + 2020-01-01 00:00:04.000 | true | NULL | + 2020-01-01 00:00:05.000 | true | NULL | + 2020-01-01 00:00:06.000 | false | 6 | + 2020-01-01 00:00:07.000 | true | NULL | + 2020-01-01 00:00:08.000 | true | NULL | + 2020-01-01 00:00:09.000 | true | NULL | + 2020-01-01 00:00:10.000 | false | 10 | + 2020-01-01 00:00:11.000 | true | NULL | + 2020-01-01 00:00:12.000 | true | NULL | + 2020-01-01 00:00:13.000 | true | NULL | + 2020-01-01 00:00:14.000 | true | NULL | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | NULL | + 2020-01-01 00:00:17.000 | true | NULL | + 2020-01-01 00:00:18.000 | true | NULL | + 2020-01-01 00:00:19.000 | true | NULL | + 2020-01-01 00:00:20.000 | true | NULL | + 2020-01-01 00:00:21.000 | false | 21 | + 2020-01-01 00:00:22.000 | true | NULL | + 2020-01-01 00:00:23.000 | true | NULL | + 2020-01-01 00:00:24.000 | true | NULL | + 2020-01-01 00:00:25.000 | true | NULL | + 2020-01-01 00:00:26.000 | true | NULL | + 2020-01-01 00:00:27.000 | true | NULL | + 2020-01-01 00:00:28.000 | true | NULL | + 2020-01-01 00:00:29.000 | true | NULL | + 2020-01-01 00:00:30.000 | true | NULL | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(value, 1); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | false | 0 | + 2020-01-01 00:00:01.000 | false | 1 | + 2020-01-01 00:00:02.000 | true | 1 | + 2020-01-01 00:00:03.000 | false | 3 | + 2020-01-01 00:00:04.000 | true | 1 | + 2020-01-01 00:00:05.000 | true | 1 | + 2020-01-01 00:00:06.000 | false | 6 | + 2020-01-01 00:00:07.000 | true | 1 | + 2020-01-01 00:00:08.000 | true | 1 | + 2020-01-01 00:00:09.000 | true | 1 | + 2020-01-01 00:00:10.000 | false | 10 | + 2020-01-01 00:00:11.000 | true | 1 | + 2020-01-01 00:00:12.000 | true | 1 | + 2020-01-01 00:00:13.000 | true | 1 | + 2020-01-01 00:00:14.000 | true | 1 | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 1 | + 2020-01-01 00:00:17.000 | true | 1 | + 2020-01-01 00:00:18.000 | true | 1 | + 2020-01-01 00:00:19.000 | true | 1 | + 2020-01-01 00:00:20.000 | true | 1 | + 2020-01-01 00:00:21.000 | false | 21 | + 2020-01-01 00:00:22.000 | true | 1 | + 2020-01-01 00:00:23.000 | true | 1 | + 2020-01-01 00:00:24.000 | true | 1 | + 2020-01-01 00:00:25.000 | true | 1 | + 2020-01-01 00:00:26.000 | true | 1 | + 2020-01-01 00:00:27.000 | true | 1 | + 2020-01-01 00:00:28.000 | true | 1 | + 2020-01-01 00:00:29.000 | true | 1 | + 2020-01-01 00:00:30.000 | true | 1 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(prev); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | false | 0 | + 2020-01-01 00:00:01.000 | false | 1 | + 2020-01-01 00:00:02.000 | true | 1 | + 2020-01-01 00:00:03.000 | false | 3 | + 2020-01-01 00:00:04.000 | true | 3 | + 2020-01-01 00:00:05.000 | true | 3 | + 2020-01-01 00:00:06.000 | false | 6 | + 2020-01-01 00:00:07.000 | true | 6 | + 2020-01-01 00:00:08.000 | true | 6 | + 2020-01-01 00:00:09.000 | true | 6 | + 2020-01-01 00:00:10.000 | false | 10 | + 2020-01-01 00:00:11.000 | true | 10 | + 2020-01-01 00:00:12.000 | true | 10 | + 2020-01-01 00:00:13.000 | true | 10 | + 2020-01-01 00:00:14.000 | true | 10 | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 15 | + 2020-01-01 00:00:17.000 | true | 15 | + 2020-01-01 00:00:18.000 | true | 15 | + 2020-01-01 00:00:19.000 | true | 15 | + 2020-01-01 00:00:20.000 | true | 15 | + 2020-01-01 00:00:21.000 | false | 21 | + 2020-01-01 00:00:22.000 | true | 21 | + 2020-01-01 00:00:23.000 | true | 21 | + 2020-01-01 00:00:24.000 | true | 21 | + 2020-01-01 00:00:25.000 | true | 21 | + 2020-01-01 00:00:26.000 | true | 21 | + 2020-01-01 00:00:27.000 | true | 21 | + 2020-01-01 00:00:28.000 | true | 21 | + 2020-01-01 00:00:29.000 | true | 21 | + 2020-01-01 00:00:30.000 | true | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(next); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | false | 0 | + 2020-01-01 00:00:01.000 | false | 1 | + 2020-01-01 00:00:02.000 | true | 3 | + 2020-01-01 00:00:03.000 | false | 3 | + 2020-01-01 00:00:04.000 | true | 6 | + 2020-01-01 00:00:05.000 | true | 6 | + 2020-01-01 00:00:06.000 | false | 6 | + 2020-01-01 00:00:07.000 | true | 10 | + 2020-01-01 00:00:08.000 | true | 10 | + 2020-01-01 00:00:09.000 | true | 10 | + 2020-01-01 00:00:10.000 | false | 10 | + 2020-01-01 00:00:11.000 | true | 15 | + 2020-01-01 00:00:12.000 | true | 15 | + 2020-01-01 00:00:13.000 | true | 15 | + 2020-01-01 00:00:14.000 | true | 15 | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 21 | + 2020-01-01 00:00:17.000 | true | 21 | + 2020-01-01 00:00:18.000 | true | 21 | + 2020-01-01 00:00:19.000 | true | 21 | + 2020-01-01 00:00:20.000 | true | 21 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(linear); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | false | 0 | + 2020-01-01 00:00:01.000 | false | 1 | + 2020-01-01 00:00:02.000 | true | 2 | + 2020-01-01 00:00:03.000 | false | 3 | + 2020-01-01 00:00:04.000 | true | 4 | + 2020-01-01 00:00:05.000 | true | 5 | + 2020-01-01 00:00:06.000 | false | 6 | + 2020-01-01 00:00:07.000 | true | 7 | + 2020-01-01 00:00:08.000 | true | 8 | + 2020-01-01 00:00:09.000 | true | 9 | + 2020-01-01 00:00:10.000 | false | 10 | + 2020-01-01 00:00:11.000 | true | 11 | + 2020-01-01 00:00:12.000 | true | 12 | + 2020-01-01 00:00:13.000 | true | 13 | + 2020-01-01 00:00:14.000 | true | 14 | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 16 | + 2020-01-01 00:00:17.000 | true | 17 | + 2020-01-01 00:00:18.000 | true | 18 | + 2020-01-01 00:00:19.000 | true | 19 | + 2020-01-01 00:00:20.000 | true | 20 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(null); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | false | 0 | + 2020-01-01 00:00:01.000 | false | 1 | + 2020-01-01 00:00:02.000 | true | NULL | + 2020-01-01 00:00:03.000 | false | 3 | + 2020-01-01 00:00:04.000 | true | NULL | + 2020-01-01 00:00:05.000 | true | NULL | + 2020-01-01 00:00:06.000 | false | 6 | + 2020-01-01 00:00:07.000 | true | NULL | + 2020-01-01 00:00:08.000 | true | NULL | + 2020-01-01 00:00:09.000 | true | NULL | + 2020-01-01 00:00:10.000 | false | 10 | + 2020-01-01 00:00:11.000 | true | NULL | + 2020-01-01 00:00:12.000 | true | NULL | + 2020-01-01 00:00:13.000 | true | NULL | + 2020-01-01 00:00:14.000 | true | NULL | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | NULL | + 2020-01-01 00:00:17.000 | true | NULL | + 2020-01-01 00:00:18.000 | true | NULL | + 2020-01-01 00:00:19.000 | true | NULL | + 2020-01-01 00:00:20.000 | true | NULL | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(value, 1); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | false | 0 | + 2020-01-01 00:00:01.000 | false | 1 | + 2020-01-01 00:00:02.000 | true | 1 | + 2020-01-01 00:00:03.000 | false | 3 | + 2020-01-01 00:00:04.000 | true | 1 | + 2020-01-01 00:00:05.000 | true | 1 | + 2020-01-01 00:00:06.000 | false | 6 | + 2020-01-01 00:00:07.000 | true | 1 | + 2020-01-01 00:00:08.000 | true | 1 | + 2020-01-01 00:00:09.000 | true | 1 | + 2020-01-01 00:00:10.000 | false | 10 | + 2020-01-01 00:00:11.000 | true | 1 | + 2020-01-01 00:00:12.000 | true | 1 | + 2020-01-01 00:00:13.000 | true | 1 | + 2020-01-01 00:00:14.000 | true | 1 | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 1 | + 2020-01-01 00:00:17.000 | true | 1 | + 2020-01-01 00:00:18.000 | true | 1 | + 2020-01-01 00:00:19.000 | true | 1 | + 2020-01-01 00:00:20.000 | true | 1 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(prev); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | false | 0 | + 2020-01-01 00:00:01.000 | false | 1 | + 2020-01-01 00:00:02.000 | true | 1 | + 2020-01-01 00:00:03.000 | false | 3 | + 2020-01-01 00:00:04.000 | true | 3 | + 2020-01-01 00:00:05.000 | true | 3 | + 2020-01-01 00:00:06.000 | false | 6 | + 2020-01-01 00:00:07.000 | true | 6 | + 2020-01-01 00:00:08.000 | true | 6 | + 2020-01-01 00:00:09.000 | true | 6 | + 2020-01-01 00:00:10.000 | false | 10 | + 2020-01-01 00:00:11.000 | true | 10 | + 2020-01-01 00:00:12.000 | true | 10 | + 2020-01-01 00:00:13.000 | true | 10 | + 2020-01-01 00:00:14.000 | true | 10 | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 15 | + 2020-01-01 00:00:17.000 | true | 15 | + 2020-01-01 00:00:18.000 | true | 15 | + 2020-01-01 00:00:19.000 | true | 15 | + 2020-01-01 00:00:20.000 | true | 15 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(next); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | false | 0 | + 2020-01-01 00:00:01.000 | false | 1 | + 2020-01-01 00:00:02.000 | true | 3 | + 2020-01-01 00:00:03.000 | false | 3 | + 2020-01-01 00:00:04.000 | true | 6 | + 2020-01-01 00:00:05.000 | true | 6 | + 2020-01-01 00:00:06.000 | false | 6 | + 2020-01-01 00:00:07.000 | true | 10 | + 2020-01-01 00:00:08.000 | true | 10 | + 2020-01-01 00:00:09.000 | true | 10 | + 2020-01-01 00:00:10.000 | false | 10 | + 2020-01-01 00:00:11.000 | true | 15 | + 2020-01-01 00:00:12.000 | true | 15 | + 2020-01-01 00:00:13.000 | true | 15 | + 2020-01-01 00:00:14.000 | true | 15 | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 21 | + 2020-01-01 00:00:17.000 | true | 21 | + 2020-01-01 00:00:18.000 | true | 21 | + 2020-01-01 00:00:19.000 | true | 21 | + 2020-01-01 00:00:20.000 | true | 21 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(linear); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | false | 0 | + 2020-01-01 00:00:01.000 | false | 1 | + 2020-01-01 00:00:02.000 | true | 2 | + 2020-01-01 00:00:03.000 | false | 3 | + 2020-01-01 00:00:04.000 | true | 4 | + 2020-01-01 00:00:05.000 | true | 5 | + 2020-01-01 00:00:06.000 | false | 6 | + 2020-01-01 00:00:07.000 | true | 7 | + 2020-01-01 00:00:08.000 | true | 8 | + 2020-01-01 00:00:09.000 | true | 9 | + 2020-01-01 00:00:10.000 | false | 10 | + 2020-01-01 00:00:11.000 | true | 11 | + 2020-01-01 00:00:12.000 | true | 12 | + 2020-01-01 00:00:13.000 | true | 13 | + 2020-01-01 00:00:14.000 | true | 14 | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 16 | + 2020-01-01 00:00:17.000 | true | 17 | + 2020-01-01 00:00:18.000 | true | 18 | + 2020-01-01 00:00:19.000 | true | 19 | + 2020-01-01 00:00:20.000 | true | 20 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:15' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(null); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | true | NULL | + 2020-01-01 00:00:01.000 | true | NULL | + 2020-01-01 00:00:02.000 | true | NULL | + 2020-01-01 00:00:03.000 | true | NULL | + 2020-01-01 00:00:04.000 | true | NULL | + 2020-01-01 00:00:05.000 | true | NULL | + 2020-01-01 00:00:06.000 | true | NULL | + 2020-01-01 00:00:07.000 | true | NULL | + 2020-01-01 00:00:08.000 | true | NULL | + 2020-01-01 00:00:09.000 | true | NULL | + 2020-01-01 00:00:10.000 | true | NULL | + 2020-01-01 00:00:11.000 | true | NULL | + 2020-01-01 00:00:12.000 | true | NULL | + 2020-01-01 00:00:13.000 | true | NULL | + 2020-01-01 00:00:14.000 | true | NULL | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | NULL | + 2020-01-01 00:00:17.000 | true | NULL | + 2020-01-01 00:00:18.000 | true | NULL | + 2020-01-01 00:00:19.000 | true | NULL | + 2020-01-01 00:00:20.000 | true | NULL | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:15' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(value, 1); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | true | 1 | + 2020-01-01 00:00:01.000 | true | 1 | + 2020-01-01 00:00:02.000 | true | 1 | + 2020-01-01 00:00:03.000 | true | 1 | + 2020-01-01 00:00:04.000 | true | 1 | + 2020-01-01 00:00:05.000 | true | 1 | + 2020-01-01 00:00:06.000 | true | 1 | + 2020-01-01 00:00:07.000 | true | 1 | + 2020-01-01 00:00:08.000 | true | 1 | + 2020-01-01 00:00:09.000 | true | 1 | + 2020-01-01 00:00:10.000 | true | 1 | + 2020-01-01 00:00:11.000 | true | 1 | + 2020-01-01 00:00:12.000 | true | 1 | + 2020-01-01 00:00:13.000 | true | 1 | + 2020-01-01 00:00:14.000 | true | 1 | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 1 | + 2020-01-01 00:00:17.000 | true | 1 | + 2020-01-01 00:00:18.000 | true | 1 | + 2020-01-01 00:00:19.000 | true | 1 | + 2020-01-01 00:00:20.000 | true | 1 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:15' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(prev); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 15 | + 2020-01-01 00:00:17.000 | true | 15 | + 2020-01-01 00:00:18.000 | true | 15 | + 2020-01-01 00:00:19.000 | true | 15 | + 2020-01-01 00:00:20.000 | true | 15 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:15' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(next); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | true | 15 | + 2020-01-01 00:00:01.000 | true | 15 | + 2020-01-01 00:00:02.000 | true | 15 | + 2020-01-01 00:00:03.000 | true | 15 | + 2020-01-01 00:00:04.000 | true | 15 | + 2020-01-01 00:00:05.000 | true | 15 | + 2020-01-01 00:00:06.000 | true | 15 | + 2020-01-01 00:00:07.000 | true | 15 | + 2020-01-01 00:00:08.000 | true | 15 | + 2020-01-01 00:00:09.000 | true | 15 | + 2020-01-01 00:00:10.000 | true | 15 | + 2020-01-01 00:00:11.000 | true | 15 | + 2020-01-01 00:00:12.000 | true | 15 | + 2020-01-01 00:00:13.000 | true | 15 | + 2020-01-01 00:00:14.000 | true | 15 | + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 21 | + 2020-01-01 00:00:17.000 | true | 21 | + 2020-01-01 00:00:18.000 | true | 21 | + 2020-01-01 00:00:19.000 | true | 21 | + 2020-01-01 00:00:20.000 | true | 21 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:15' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(linear); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:15.000 | false | 15 | + 2020-01-01 00:00:16.000 | true | 16 | + 2020-01-01 00:00:17.000 | true | 17 | + 2020-01-01 00:00:18.000 | true | 18 | + 2020-01-01 00:00:19.000 | true | 19 | + 2020-01-01 00:00:20.000 | true | 20 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:21' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(null); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | true | NULL | + 2020-01-01 00:00:01.000 | true | NULL | + 2020-01-01 00:00:02.000 | true | NULL | + 2020-01-01 00:00:03.000 | true | NULL | + 2020-01-01 00:00:04.000 | true | NULL | + 2020-01-01 00:00:05.000 | true | NULL | + 2020-01-01 00:00:06.000 | true | NULL | + 2020-01-01 00:00:07.000 | true | NULL | + 2020-01-01 00:00:08.000 | true | NULL | + 2020-01-01 00:00:09.000 | true | NULL | + 2020-01-01 00:00:10.000 | true | NULL | + 2020-01-01 00:00:11.000 | true | NULL | + 2020-01-01 00:00:12.000 | true | NULL | + 2020-01-01 00:00:13.000 | true | NULL | + 2020-01-01 00:00:14.000 | true | NULL | + 2020-01-01 00:00:15.000 | true | NULL | + 2020-01-01 00:00:16.000 | true | NULL | + 2020-01-01 00:00:17.000 | true | NULL | + 2020-01-01 00:00:18.000 | true | NULL | + 2020-01-01 00:00:19.000 | true | NULL | + 2020-01-01 00:00:20.000 | true | NULL | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:21' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(value, 1); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | true | 1 | + 2020-01-01 00:00:01.000 | true | 1 | + 2020-01-01 00:00:02.000 | true | 1 | + 2020-01-01 00:00:03.000 | true | 1 | + 2020-01-01 00:00:04.000 | true | 1 | + 2020-01-01 00:00:05.000 | true | 1 | + 2020-01-01 00:00:06.000 | true | 1 | + 2020-01-01 00:00:07.000 | true | 1 | + 2020-01-01 00:00:08.000 | true | 1 | + 2020-01-01 00:00:09.000 | true | 1 | + 2020-01-01 00:00:10.000 | true | 1 | + 2020-01-01 00:00:11.000 | true | 1 | + 2020-01-01 00:00:12.000 | true | 1 | + 2020-01-01 00:00:13.000 | true | 1 | + 2020-01-01 00:00:14.000 | true | 1 | + 2020-01-01 00:00:15.000 | true | 1 | + 2020-01-01 00:00:16.000 | true | 1 | + 2020-01-01 00:00:17.000 | true | 1 | + 2020-01-01 00:00:18.000 | true | 1 | + 2020-01-01 00:00:19.000 | true | 1 | + 2020-01-01 00:00:20.000 | true | 1 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:21' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(prev); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:21' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(next); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:00.000 | true | 21 | + 2020-01-01 00:00:01.000 | true | 21 | + 2020-01-01 00:00:02.000 | true | 21 | + 2020-01-01 00:00:03.000 | true | 21 | + 2020-01-01 00:00:04.000 | true | 21 | + 2020-01-01 00:00:05.000 | true | 21 | + 2020-01-01 00:00:06.000 | true | 21 | + 2020-01-01 00:00:07.000 | true | 21 | + 2020-01-01 00:00:08.000 | true | 21 | + 2020-01-01 00:00:09.000 | true | 21 | + 2020-01-01 00:00:10.000 | true | 21 | + 2020-01-01 00:00:11.000 | true | 21 | + 2020-01-01 00:00:12.000 | true | 21 | + 2020-01-01 00:00:13.000 | true | 21 | + 2020-01-01 00:00:14.000 | true | 21 | + 2020-01-01 00:00:15.000 | true | 21 | + 2020-01-01 00:00:16.000 | true | 21 | + 2020-01-01 00:00:17.000 | true | 21 | + 2020-01-01 00:00:18.000 | true | 21 | + 2020-01-01 00:00:19.000 | true | 21 | + 2020-01-01 00:00:20.000 | true | 21 | + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:21' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(linear); + _irowts | _isfilled | interp(c1) | +==================================================== + 2020-01-01 00:00:21.000 | false | 21 | + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:22' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(null); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:22' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(value, 1); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:22' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(prev); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:22' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(next); + +taos> select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:22' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(linear); + diff --git a/tests/army/query/function/in/interp.in b/tests/army/query/function/in/interp.in index 4825ab46b1..97a9936b8d 100644 --- a/tests/army/query/function/in/interp.in +++ b/tests/army/query/function/in/interp.in @@ -13,3 +13,53 @@ select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfille select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (prev) order by irowts, c2, c3; select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (linear) order by irowts, c2, c3; select _irowts as irowts ,tbname as table_name, c2 as c_c2, c3 as c_c3, _isfilled as isfilled , interp(c1) as intp_c1 from test.td32727 partition by tbname,c2,c3 range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill (value, 1) order by irowts, c2, c3; +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-02 00:00:00' and '2020-01-01 00:00:00' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(null); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-02 00:00:00' and '2020-01-01 00:00:00' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(value, 1); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-02 00:00:00' and '2020-01-01 00:00:00' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(prev); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-02 00:00:00' and '2020-01-01 00:00:00' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(next); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-02 00:00:00' and '2020-01-01 00:00:00' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(linear); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-02 00:00:00' range('2020-01-01 00:00:30', '2020-01-01 00:00:00') every(1s) fill(null); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-02 00:00:00' range('2020-01-01 00:00:30', '2020-01-01 00:00:00') every(1s) fill(value, 1); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-02 00:00:00' range('2020-01-01 00:00:30', '2020-01-01 00:00:00') every(1s) fill(prev); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-02 00:00:00' range('2020-01-01 00:00:30', '2020-01-01 00:00:00') every(1s) fill(next); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-02 00:00:00' range('2020-01-01 00:00:30', '2020-01-01 00:00:00') every(1s) fill(linear); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:20' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(null); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:20' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(value, 1); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:20' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(prev); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:20' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(next); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:20' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(linear); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(null); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(value, 1); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(prev); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(next); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:21', '2020-01-01 00:00:30') every(1s) fill(linear); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:15', '2020-01-01 00:00:30') every(1s) fill(null); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:15', '2020-01-01 00:00:30') every(1s) fill(value, 1); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:15', '2020-01-01 00:00:30') every(1s) fill(prev); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:15', '2020-01-01 00:00:30') every(1s) fill(next); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:15', '2020-01-01 00:00:30') every(1s) fill(linear); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(null); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(value, 1); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(prev); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(next); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:21' range('2020-01-01 00:00:00', '2020-01-01 00:00:30') every(1s) fill(linear); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(null); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(value, 1); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(prev); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(next); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:00' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(linear); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:15' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(null); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:15' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(value, 1); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:15' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(prev); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:15' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(next); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:15' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(linear); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:21' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(null); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:21' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(value, 1); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:21' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(prev); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:21' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(next); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:21' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(linear); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:22' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(null); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:22' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(value, 1); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:22' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(prev); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:22' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(next); +select _irowts, _isfilled, interp(c1) from test.td32861 where ts between '2020-01-01 00:00:22' and '2020-01-01 00:00:30' range('2020-01-01 00:00:00', '2020-01-01 00:00:21') every(1s) fill(linear); diff --git a/tests/army/query/function/test_interp.py b/tests/army/query/function/test_interp.py index f903e7be73..106ef1e58e 100644 --- a/tests/army/query/function/test_interp.py +++ b/tests/army/query/function/test_interp.py @@ -38,6 +38,7 @@ class TDTestCase(TBase): (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10), c9 tinyint unsigned, c10 smallint unsigned, c11 int unsigned, c12 bigint unsigned) ''' ) + tdSql.execute("create table if not exists test.td32861(ts timestamp, c1 int);") tdLog.printNoPrefix("==========step2:insert data") @@ -45,6 +46,16 @@ class TDTestCase(TBase): tdSql.execute(f"insert into test.td32727 values ('2020-02-01 00:00:10', 10, 10, 10, 10, 10.0, 10.0, true, 'varchar', 'nchar', 10, 10, 10, 10)") tdSql.execute(f"insert into test.td32727 values ('2020-02-01 00:00:15', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar', 15, 15, 15, 15)") + tdSql.execute( + """insert into test.td32861 values + ('2020-01-01 00:00:00', 0), + ('2020-01-01 00:00:01', 1), + ('2020-01-01 00:00:03', 3), + ('2020-01-01 00:00:06', 6), + ('2020-01-01 00:00:10', 10), + ('2020-01-01 00:00:15', 15), + ('2020-01-01 00:00:21', 21);""" + ) def test_normal_query_new(self, testCase): # read sql from .sql file and execute From cb44efd3297dc74d3a805d2296b1840174a34635 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 18 Nov 2024 15:22:11 +0800 Subject: [PATCH 54/62] fix comments --- source/libs/nodes/src/nodesUtilFuncs.c | 11 ++++++++++- source/libs/parser/src/parCalcConst.c | 7 ++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 4677b5d5f0..30cc552761 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2960,7 +2960,16 @@ static SNode* nodesListFindNode(SNodeList* pList, SNode* pNode) { } int32_t nodesListDeduplicate(SNodeList** ppList) { - if (!ppList || LIST_LENGTH(*ppList) == 0) return TSDB_CODE_SUCCESS; + if (!ppList || LIST_LENGTH(*ppList) <= 1) return TSDB_CODE_SUCCESS; + if (LIST_LENGTH(*ppList) == 2) { + SNode* pNode1 = nodesListGetNode(*ppList, 0); + SNode* pNode2 = nodesListGetNode(*ppList, 1); + if (nodesEqualNode(pNode1, pNode2)) { + SListCell* pCell = nodesListGetCell(*ppList, 1); + (void)nodesListErase(*ppList, pCell); + } + return TSDB_CODE_SUCCESS; + } SNodeList* pTmp = NULL; int32_t code = nodesMakeList(&pTmp); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 778c34ff02..9702c2e760 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -326,7 +326,6 @@ static int32_t calcConstProjections(SCalcConstContext* pCxt, SSelectStmt* pSelec static int32_t calcConstGroupBy(SCalcConstContext* pCxt, SSelectStmt* pSelect) { int32_t code = calcConstList(pSelect->pGroupByList); -#if 0 if (TSDB_CODE_SUCCESS == code) { SNode* pNode = NULL; FOREACH(pNode, pSelect->pGroupByList) { @@ -337,9 +336,11 @@ static int32_t calcConstGroupBy(SCalcConstContext* pCxt, SSelectStmt* pSelect) { } } } - NODES_DESTORY_LIST(pSelect->pGroupByList); + FOREACH(pNode, pSelect->pGroupByList) { + if (!cell->pPrev) continue; + ERASE_NODE(pSelect->pGroupByList); + } } -#endif return code; } From 31f5055f21bf60cd447a4f95e59291ea7190c497 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 18 Nov 2024 19:06:46 +0800 Subject: [PATCH 55/62] fix/TS-5639-check-mnode-leader --- source/dnode/mnode/impl/src/mndMain.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 6c30193ea7..9dd43225b1 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -53,7 +53,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) { if (pMnode->stopped) { code = TSDB_CODE_APP_IS_STOPPING; } else if (!mndIsLeader(pMnode)) { - code = -1; + code = 1; } else { #if 1 (void)atomic_add_fetch_32(&pMnode->rpcRef, 1); @@ -1002,8 +1002,12 @@ int64_t mndGenerateUid(const char *name, int32_t len) { int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) { - int32_t code = 0; - TAOS_CHECK_RETURN(mndAcquireRpc(pMnode)); + int32_t code = mndAcquireRpc(pMnode); + if (code < 0) { + TAOS_RETURN(code); + } else if (code == 1) { + TAOS_RETURN(TSDB_CODE_SUCCESS); + } SSdb *pSdb = pMnode->pSdb; int64_t ms = taosGetTimestampMs(); From 7f92585b93bf917ed4313ce335751fa55e2e4162 Mon Sep 17 00:00:00 2001 From: menshibin Date: Mon, 18 Nov 2024 20:09:37 +0800 Subject: [PATCH 56/62] add test tzlocal dependent --- docs/examples/node/package.json | 10 ++++++++-- tests/requirements.txt | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/examples/node/package.json b/docs/examples/node/package.json index 14303c8f37..d77c96fbb3 100644 --- a/docs/examples/node/package.json +++ b/docs/examples/node/package.json @@ -4,6 +4,12 @@ "main": "index.js", "license": "MIT", "dependencies": { - "@tdengine/websocket": "^3.1.1" - } + "@tdengine/websocket": "^3.1.2" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "description": "" } diff --git a/tests/requirements.txt b/tests/requirements.txt index c6dd044c86..a036c2b3d0 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,4 +9,5 @@ requests pexpect faker pyopenssl -hyperloglog \ No newline at end of file +hyperloglog +tzlocal \ No newline at end of file From b3eb84c0047dc5d960092f11f97482ff9443d3e6 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 19 Nov 2024 18:30:27 +0800 Subject: [PATCH 57/62] fix: fetch_row callback error code --- source/client/src/clientImpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8a0b1ddaab..94d06166f2 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -3032,13 +3032,13 @@ static void fetchCallback(void* pResult, void* param, int32_t code) { if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; taosMemoryFreeClear(pResultInfo->pData); - pRequest->body.fetchFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, 0); + pRequest->body.fetchFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, code); return; } if (pRequest->code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pResultInfo->pData); - pRequest->body.fetchFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, 0); + pRequest->body.fetchFp(((SSyncQueryParam*)pRequest->body.interParam)->userParam, pRequest, pRequest->code); return; } From 15bccef3920d7c375dc9a612dbb808e411617b38 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 20 Nov 2024 11:02:17 +0800 Subject: [PATCH 58/62] docs: describe KEEP_TIME_OFFSET --- docs/en/14-reference/12-config/index.md | 4 ++-- docs/zh/14-reference/03-taos-sql/02-database.md | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 77d183a5ef..63aa6ed447 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -773,7 +773,7 @@ lossyColumns float|double 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double ``` -### ifAdtFse +### ifAdtFse | Attribute | Description | | -------- | -------------------------------- | @@ -898,4 +898,4 @@ lossyColumns float|double | 53 | udf | Yes | Yes | | | 54 | enableCoreFile | Yes | Yes | | | 55 | ttlChangeOnWrite | No | Yes | | -| 56 | keepTimeOffset | Yes | Yes(discarded since 3.2.0.0) | | +| 56 | keepTimeOffset | Yes | Yes(discarded since 3.2.0.0) | see "KEEP_TIME_OFFSET" | diff --git a/docs/zh/14-reference/03-taos-sql/02-database.md b/docs/zh/14-reference/03-taos-sql/02-database.md index 0d9ab019c0..25d7e54823 100644 --- a/docs/zh/14-reference/03-taos-sql/02-database.md +++ b/docs/zh/14-reference/03-taos-sql/02-database.md @@ -8,10 +8,10 @@ description: "创建、删除数据库，查看、修改数据库参数" ```sql CREATE DATABASE [IF NOT EXISTS] db_name [database_options] - + database_options: database_option ... - + database_option: { VGROUPS value | PRECISION {'ms' | 'us' | 'ns'} @@ -26,6 +26,7 @@ database_option: { | MAXROWS value | MINROWS value | KEEP value + | KEEP_TIME_OFFSET value | STT_TRIGGER value | SINGLE_STABLE {0 | 1} | TABLE_PREFIX value @@ -43,7 +44,7 @@ database_option: { - VGROUPS：数据库中初始 vgroup 的数目。 - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 -- REPLICA：表示数据库副本数，取值为 1、2 或 3，默认为 1; 2 仅在企业版 3.3.0.0 及以后版本中可用。在集群中使用，副本数必须小于或等于 DNODE 的数目。且使用时存在以下限制： +- REPLICA：表示数据库副本数，取值为 1、2 或 3，默认为 1; 2 仅在企业版 3.3.0.0 及以后版本中可用。在集群中使用，副本数必须小于或等于 DNODE 的数目。且使用时存在以下限制： - 暂不支持对双副本数据库相关 Vgroup 进行 SPLITE VGROUP 或 REDISTRIBUTE VGROUP 操作 - 单副本数据库可变更为双副本数据库，但不支持从双副本变更为其它副本数，也不支持从三副本变更为双副本 - BUFFER: 一个 VNODE 写入内存池大小，单位为 MB，默认为 256，最小为 3，最大为 16384。 @@ -64,6 +65,7 @@ database_option: { - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 - KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据从而释放存储空间。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 \<= keep 1 \<= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 +- KEEP_TIME_OFFSET：自 3.2.0.0 版本生效。删除或迁移保存时间超过 KEEP 值的数据的延迟执行时间，默认值为 0 (小时)。在数据文件保存时间超过 KEEP 后，删除或迁移操作不会立即执行，而会额外等待本参数指定的时间间隔，以实现与业务高峰期错开的目的。 - STT_TRIGGER：表示落盘文件触发文件合并的个数。开源版本固定为 1，企业版本可设置范围为 1 到 16。对于少表高频写入场景，此参数建议使用默认配置；而对于多表低频写入场景，此参数建议配置较大的值。 - SINGLE_STABLE：表示此数据库中是否只可以创建一个超级表，用于超级表列非常多的情况。 - 0：表示可以创建多张超级表。 From 9314dfb80cf26cfbecb4668994fff9a008fc519d Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 20 Nov 2024 14:29:27 +0800 Subject: [PATCH 59/62] fix not setting step in tsdbread --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 4 ++-- tests/system-test/2-query/partition_by_col.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 441d073666..527486270c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4812,7 +4812,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn pBlockData = &pReader->status.fileBlockData; asc = ASCENDING_TRAVERSE(pReader->info.order); pVerRange = &pReader->info.verRange; - ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; + step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; *state = CHECK_FILEBLOCK_QUIT; code = loadNeighborIfOverlap(pFBlock, pScanInfo, pReader, &loadNeighbor); @@ -6169,7 +6169,7 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { TSDB_CHECK_CODE(code, lino, _end); } - goto _end; + return code; } } diff --git a/tests/system-test/2-query/partition_by_col.py b/tests/system-test/2-query/partition_by_col.py index ef88e88cbd..da7fe78124 100644 --- a/tests/system-test/2-query/partition_by_col.py +++ b/tests/system-test/2-query/partition_by_col.py @@ -313,7 +313,21 @@ class TDTestCase: order_by_list = 'ts,c1,c2,c3,c4,c5,c6,c7,c8,c9,t1,t2,t3,t4,t5,t6' self.prepare_and_query_and_compare(sqls, order_by_list, compare_what=COMPARE_LEN) + + def test_tsdb_read(self): + tdSql.execute('delete from t0') + tdSql.execute('flush database test') + for i in range(0, 4096): + tdSql.execute(f"insert into test.t0 values({1537146000000 + i}, 1,1,1,1,1,1,1,'a','1')") + tdSql.execute("flush database test") + tdSql.execute(f"insert into t0 values({1537146000000 + 4095}, 1,1,1,1,1,1,1,'a','1')") + for i in range(4095, 4096*2 + 100): + tdSql.execute(f"insert into test.t0 values({1537146000000 + i}, 1,1,1,1,1,1,1,'a','1')") + tdSql.execute("flush database test") + time.sleep(5) + tdSql.query('select first(ts), last(ts) from t0', queryTimes=1) + tdSql.checkRows(1) def run(self): self.prepareTestEnv() @@ -323,6 +337,8 @@ class TDTestCase: self.test_sort_for_partition_res() self.test_sort_for_partition_interval() self.test_sort_for_partition_no_agg_limit() + self.test_tsdb_read() + def stop(self): tdSql.close() From 3a4ed77b170702c4f6fdfdd4b45fa596656e6017 Mon Sep 17 00:00:00 2001 From: menshibin Date: Wed, 20 Nov 2024 16:35:27 +0800 Subject: [PATCH 60/62] upgrade python version --- Jenkinsfile2 | 2 +- docs/en/14-reference/05-connectors/30-python.mdx | 6 ++++++ docs/en/14-reference/05-connectors/35-node.mdx | 2 ++ docs/zh/14-reference/05-connector/30-python.mdx | 2 ++ docs/zh/14-reference/05-connector/35-node.mdx | 1 + tests/ci/Dockerfile | 2 +- tests/docs-examples-test/python.sh | 2 +- tests/parallel_test/run_case.sh | 4 ++-- 8 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 7ba9b4a933..910388fa8c 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -355,7 +355,7 @@ def pre_test_build_win() { bat ''' cd %WIN_COMMUNITY_ROOT%/tests/ci pip3 install taospy==2.7.16 - pip3 install taos-ws-py==0.3.3 + pip3 install taos-ws-py==0.3.5 xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ''' return 1 diff --git a/docs/en/14-reference/05-connectors/30-python.mdx b/docs/en/14-reference/05-connectors/30-python.mdx index 7263a3caa6..4f17261b33 100644 --- a/docs/en/14-reference/05-connectors/30-python.mdx +++ b/docs/en/14-reference/05-connectors/30-python.mdx @@ -41,12 +41,18 @@ We recommend using the latest version of `taospy`, regardless of the version of |Python Client Library Version|major changes| |:-------------------:|:----:| +|2.7.16|add subscription configuration (session.timeout.ms, max.poll.interval.ms)| +|2.7.15|added support for VARBINARY and GEOMETRY types| +|2.7.14|fix known issues| +|2.7.13|add TMQ synchronous submission offset interface| |2.7.12|1. added support for `varbinary` type (STMT does not yet support)
2. improved query performance (thanks to contributor [hadrianl](https://github.com/taosdata/taos-connector-python/pull/209))| |2.7.9|support for getting assignment and seek function on subscription| |2.7.8|add `execute_many` method| |Python Websocket Connection Version|major changes| |:----------------------------:|:-----:| +|0.3.5|1. added support for VARBINARY and GEOMETRY types
2. Fix known issues| +|0.3.2|1. optimize WebSocket SQL query and insertion performance
2. Fix known issues
3. Modify the readme and document| |0.2.9|bugs fixes| |0.2.5|1. support for getting assignment and seek function on subscription
2. support schemaless
3. support STMT| |0.2.4|support `unsubscribe` on subscription| diff --git a/docs/en/14-reference/05-connectors/35-node.mdx b/docs/en/14-reference/05-connectors/35-node.mdx index 476f9bab71..2aeef7af1e 100644 --- a/docs/en/14-reference/05-connectors/35-node.mdx +++ b/docs/en/14-reference/05-connectors/35-node.mdx @@ -27,6 +27,8 @@ Node.js client library needs to be run with Node.js 14 or higher version. | Node.js connector version | major changes | TDengine 版本 | | :-----------------------: | :------------------: | :----------------:| +| 3.1.2 | Optimized the data protocol and parsing, resulting in a significant improvement in performance | 3.2.0.0 or later | +| 3.1.1 | Optimized data transmission performance | 3.2.0.0 or later | | 3.1.0 | new version, supports websocket | 3.2.0.0 or later | ## Supported features diff --git a/docs/zh/14-reference/05-connector/30-python.mdx b/docs/zh/14-reference/05-connector/30-python.mdx index 8436c30249..3991477635 100644 --- a/docs/zh/14-reference/05-connector/30-python.mdx +++ b/docs/zh/14-reference/05-connector/30-python.mdx @@ -41,6 +41,7 @@ Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-con |Python Connector 版本|主要变化| |:-------------------:|:----:| +|2.7.16|新增订阅配置 (session.timeout.ms, max.poll.interval.ms)| |2.7.15|新增 VARBINARY 和 GEOMETRY 类型支持| |2.7.14|修复已知问题| |2.7.13|新增 tmq 同步提交 offset 接口| @@ -50,6 +51,7 @@ Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-con |Python WebSocket Connector 版本|主要变化| |:----------------------------:|:-----:| +|0.3.5|新增 VARBINARY 和 GEOMETRY 类型支持，修复已知问题| |0.3.2|优化 WebSocket sql 查询和插入性能，修改 readme 和 文档，修复已知问题| |0.2.9|已知问题修复| |0.2.5|1. 数据订阅支持获取消费进度和重置消费进度
2. 支持 schemaless
3. 支持 STMT| diff --git a/docs/zh/14-reference/05-connector/35-node.mdx b/docs/zh/14-reference/05-connector/35-node.mdx index d9512eae78..df2abfab3d 100644 --- a/docs/zh/14-reference/05-connector/35-node.mdx +++ b/docs/zh/14-reference/05-connector/35-node.mdx @@ -26,6 +26,7 @@ Node.js 连接器目前仅支持 WebSocket 连接器， 其通过 taosAdapter | Node.js 连接器 版本 | 主要变化 | TDengine 版本 | | :------------------: | :----------------------: | :----------------: | +| 3.1.2 | 对数据协议和解析进行了优化，性能得到大幅提升| 3.3.2.0 及更高版本 | | 3.1.1 | 优化了数据传输性能 | 3.3.2.0 及更高版本 | | 3.1.0 | 新版本发布，支持 WebSocket 连接 | 3.2.0.0 及更高版本 | diff --git a/tests/ci/Dockerfile b/tests/ci/Dockerfile index d3d574b484..1caa6fea9e 100644 --- a/tests/ci/Dockerfile +++ b/tests/ci/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get install -y locales psmisc sudo tree libgeos-dev libgflags2.2 libgfl RUN sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen && locale-gen RUN pip3 config set global.index-url http://admin:123456@192.168.0.212:3141/admin/dev/+simple/ RUN pip3 config set global.trusted-host 192.168.0.212 -RUN pip3 install taospy==2.7.16 taos-ws-py==0.3.3 pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog +RUN pip3 install taospy==2.7.16 taos-ws-py==0.3.5 pandas psutil fabric2 requests faker simplejson toml pexpect tzlocal distro decorator loguru hyperloglog ENV LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8 RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/' diff --git a/tests/docs-examples-test/python.sh b/tests/docs-examples-test/python.sh index 6a25683b58..3a9812637c 100644 --- a/tests/docs-examples-test/python.sh +++ b/tests/docs-examples-test/python.sh @@ -130,7 +130,7 @@ pip3 install kafka-python python3 kafka_example_consumer.py # 21 -pip3 install taos-ws-py==0.3.3 +pip3 install taos-ws-py==0.3.5 python3 conn_websocket_pandas.py # 22 diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index 5b0d34fc0a..a78d0aa4a4 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -76,9 +76,9 @@ ulimit -c unlimited md5sum /usr/lib/libtaos.so.1 md5sum /home/TDinternal/debug/build/lib/libtaos.so -#get python connector and update: taospy 2.7.16 taos-ws-py 0.3.3 +#get python connector and update: taospy 2.7.16 taos-ws-py 0.3.5 pip3 install taospy==2.7.16 -pip3 install taos-ws-py==0.3.3 +pip3 install taos-ws-py==0.3.5 $TIMEOUT_CMD $cmd RET=$? echo "cmd exit code: $RET" From da51741e66dc7992ac3d3edfe03500a4a5581c05 Mon Sep 17 00:00:00 2001 From: huohong <346479823@qq.com> Date: Wed, 20 Nov 2024 17:39:25 +0800 Subject: [PATCH 61/62] test:update 3.3.4.3 release notes in office web --- docs/en/28-releases/01-tdengine.md | 4 + docs/zh/28-releases/01-tdengine.md | 4 + docs/zh/28-releases/03-notes/3.3.4.3.md | 123 +++++++++++------------- docs/zh/28-releases/03-notes/index.md | 1 + 4 files changed, 66 insertions(+), 66 deletions(-) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index b24931b166..161efe0d61 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -24,6 +24,10 @@ import Release from "/components/ReleaseV3"; +## 3.3.4.3 + + + ## 3.3.3.0 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index cf9d7b6878..d285cb36ae 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -28,6 +28,10 @@ import Release from "/components/ReleaseV3"; +## 3.3.4.3 + + + ## 3.3.3.0 diff --git a/docs/zh/28-releases/03-notes/3.3.4.3.md b/docs/zh/28-releases/03-notes/3.3.4.3.md index 8ffd5802ed..364732b478 100644 --- a/docs/zh/28-releases/03-notes/3.3.4.3.md +++ b/docs/zh/28-releases/03-notes/3.3.4.3.md @@ -1,69 +1,60 @@ ---- -title: 3.3.4.3 版本说明 -sidebar_label: 3.3.4.3 -description: 3.3.4.3 版本说明 ---- +## 新特性 +* 新功能：流计算的 TWA 函数支持时间驱动的结果推送模式 +* 新功能：流计算的 Interp 函数支持时间驱动的结果推送模式 +* 优化：顺序执行 compact 和 split vgroup操作时的日志错误提示 +* 新功能：支持微软对象存储 -### 行为变更及兼容性 -1. 多副本流计算中必须使用 snode -1. 增加了流计算的兼容性保证机制，避免后续函数变更产生新的兼容性问题，但之前版本的流计算必须重建，具体参见 https://docs.taosdata.com/advanced/stream/#流计算升级故障恢复 -1. 调整 case when 语句结果类型的判断方法 - -### 新特性 -1. 新功能：流计算的 TWA 函数支持时间驱动的结果推送模式 -1. 新功能：流计算的 Interp 函数支持时间驱动的结果推送模式 -1. 新功能：支持微软对象存储 +## 优化 +* 优化：提升并发大查询时节点之间互相拉数据的效率 +* 优化：支持使用 AVX2 和 AVX512 对 double 、timestamp 和 bigint 类型进行解码优化 +* 优化：调整 case when 语句的结果类型判断方法 +* 优化：提升查询 “select ... from ... where ts in (...)” 的数据扫描速度 +* 优化：增加了流计算的兼容性保证机制，避免后续函数变更产生新的兼容性问题，之前版本的流计算必须重建 +* 优化：提升 taosX 在交叉写入场景下的数据同步性能 +* 优化：支持关闭整数/浮点数类型的编码 +* 优化：多副本流计算中必须使用 snode +* 优化：客户端生成唯一 ID 标识每一个查询任务，避免重复 ID 导致的内存损坏 +* 优化：加快数据库的创建时间 +* 优化：修改 s3MigrateEnabled 默认值为0 +* 优化：支持在审计数据库中记录删除操作 +* 优化：支持在指定的 dnode 中创建数据库 [企业版] +* 优化：调整删除超级表数据列时的报错信息 -### 优化 -1. 优化：提升并发大查询时节点之间互相拉数据的效率 -1. 优化：支持使用 AVX2 和 AVX512 对 double 、timestamp 和 bigint 类型进行解码优化 -1. 优化：调整 case when 语句的结果类型判断方法 -1. 优化：顺序执行 compact 和 split vgroup操作时的日志错误提示 -1. 优化：提升查询 “select ... from ... where ts in (...)” 的数据扫描速度 -1. 优化：增加了流计算的兼容性保证机制，避免后续函数变更产生新的兼容性问题，之前版本的流计算必须重建 -1. 优化：提升 taosX 在交叉写入场景下的数据同步性能 -1. 优化：支持关闭整数/浮点数类型的编码 -1. 优化：多副本流计算中必须使用 snode -1. 优化：客户端生成唯一 ID 标识每一个查询任务，避免重复 ID 导致的内存损坏 -1. 优化：加快数据库的创建时间 -1. 优化：修改 s3MigrateEnabled 默认值为0 -1. 优化：支持在审计数据库中记录删除操作 -1. 优化：支持在指定的 dnode 中创建数据库 [企业版] -1. 优化：调整删除超级表数据列时的报错信息 +## 修复 +* 修复：last_row 查询性能在 3.3.3.0 中大幅下降的问题 +* 修复：WAL 条目不完整时 taosd 无法启动的问题 +* 修复: partition by 常量时查询结果错误的问题 +* 修复：标量函数包含 _wstart 且填充方式为 prev 时计算结果错误 +* 修复：Windows 平台下的时区设置问题 +* 修复：空数据库进行 compact 操作时，事务无法结束【企业版】 +* 修复：事务冲突的逻辑错误 +* 修复：管理节点某些错误会导致事务无法停止 +* 修复：管理节点某些错误会导致事务无法停止 +* 修复：dnode 数据清空后 taosc 重试错误的问题 +* 修复：Data Compact 被异常终止后，中间文件未被清理 +* 修复：新增列后，Kafka 连接器的 earliest 模式消费不到新列数据 +* 修复：interp 函数在 fill(prev) 时行为不正确 +* 修复：TSMA 在高频元数据操作时异常停止的问题 +* 修复：show create stable 语句执行结果的标签显示错误 +* 修复：Percentile 函数在大数据量查询时会崩溃。 +* 修复：partition by 和 having 联合使用时的语法错误问题 +* 修复：interp 在 partition by tbname,c1 时 tbname 为空的问题 +* 修复：通过 stmt 写入非法布尔数值时 taosd 可能 crash +* 修复：Explorer OPC-UA 表名模板说明 +* 修复：库符号 version 与使用相同符号的库冲突的问题 https://github.com/taosdata/TDengine/issues/25920 +* 修复：在 windows 平台下 JDBC 驱动的句柄数持续升高问题 +* 修复：3.3.3.1 升级至 3.3.4.0 偶现的启动失败问题 +* 修复：Windows 平台重复增删表的内存泄漏 +* 修复：无法限制并发拉起 checkpoint 数量导致流计算消耗资源过多 +* 修复：并发查询时的 too many session 问题 +* 修复：Windows 平台下 taos shell 在慢查询场景中崩溃的问题 +* 修复：当打开 dnode日志时，加密数据库无法恢复的问题 +* 修复：由于 mnode 同步超时，进而导致 taosd 无法启动的问题 +* 修复：由于在快照同步过程中整理文件组数据的速度过慢，从而导致 Vnode（虚拟节点）无法恢复的问题 +* 修复：通过行协议向字符串类型的字段中写入带转义符的数据时，taosd 会崩溃 +* 修复：Error Code 逻辑处理错误导致的元数据文件损坏 +* 修复：查询语句中包含多个 “not” 条件语句嵌套时，未设置标量模式导致查询错误 +* 修复：vnode 统计信息上报超时导致的 dnode offline 问题 +* 修复：在不支持 avx 指令集的服务器上，taosd 启动失败问题 +* 修复：taosX 数据迁移容错处理 0x09xx 错误码 -### 修复 -1. 修复：last_row 查询性能在 3.3.3.0 中大幅下降的问题 -1. 修复：WAL 条目不完整时 taosd 无法启动的问题 -1. 修复: partition by 常量时查询结果错误的问题 -1. 修复：标量函数包含 _wstart 且填充方式为 prev 时计算结果错误 -1. 修复：Windows 平台下的时区设置问题 -1. 修复：空数据库进行 compact 操作时，事务无法结束【企业版】 -1. 修复：事务冲突的逻辑错误 -1. 修复：管理节点某些错误会导致事务无法停止 -1. 修复：管理节点某些错误会导致事务无法停止 -1. 修复：dnode 数据清空后 taosc 重试错误的问题 -1. 修复：Data Compact 被异常终止后，中间文件未被清理 -1. 修复：新增列后，Kafka 连接器的 earliest 模式消费不到新列数据 -1. 修复：interp 函数在 fill(prev) 时行为不正确 -1. 修复：TSMA 在高频元数据操作时异常停止的问题 -1. 修复：show create stable 语句执行结果的标签显示错误 -1. 修复：Percentile 函数在大数据量查询时会崩溃。 -1. 修复：partition by 和 having 联合使用时的语法错误问题 -1. 修复：interp 在 partition by tbname,c1 时 tbname 为空的问题 -1. 修复：通过 stmt 写入非法布尔数值时 taosd 可能 crash -1. 修复：库符号 version 与使用相同符号的库冲突的问题 -1. 修复：在 windows 平台下 JDBC 驱动的句柄数持续升高问题 -1. 修复：3.3.3.1 升级至 3.3.4.0 偶现的启动失败问题 -1. 修复：Windows 平台重复增删表的内存泄漏 -1. 修复：无法限制并发拉起 checkpoint 数量导致流计算消耗资源过多 -1. 修复：并发查询时的 too many session 问题 -1. 修复：Windows 平台下 taos shell 在慢查询场景中崩溃的问题 -1. 修复：当打开 dnode日志时，加密数据库无法恢复的问题 -1. 修复：由于 mnode 同步超时，进而导致 taosd 无法启动的问题 -1. 修复：由于在快照同步过程中整理文件组数据的速度过慢，从而导致 Vnode（虚拟节点）无法恢复的问题 -1. 修复：通过行协议向字符串类型的字段中写入带转义符的数据时，taosd 会崩溃 -1. 修复：Error Code 逻辑处理错误导致的元数据文件损坏 -1. 修复：查询语句中包含多个 “not” 条件语句嵌套时，未设置标量模式导致查询错误 -1. 修复：vnode 统计信息上报超时导致的 dnode offline 问题 -1. 修复：在不支持 avx 指令集的服务器上，taosd 启动失败问题 -1. 修复：taosX 数据迁移容错处理 0x09xx 错误码 diff --git a/docs/zh/28-releases/03-notes/index.md b/docs/zh/28-releases/03-notes/index.md index d1a48ab9a8..1cdd3239e7 100644 --- a/docs/zh/28-releases/03-notes/index.md +++ b/docs/zh/28-releases/03-notes/index.md @@ -4,6 +4,7 @@ sidebar_label: 版本说明 description: 各版本版本说明 --- +[3.3.4.3](./3.3.4.3) [3.3.4.3](./3.3.4.3) [3.3.3.0](./3.3.3.0) [3.3.2.0](./3.3.2.0) From 5ebb44903e944ec799b7e7c4886d78bed80cf462 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 21 Nov 2024 10:30:18 +0800 Subject: [PATCH 62/62] Revert "Update release note for 3.3.4.3 in office web" --- docs/en/28-releases/01-tdengine.md | 4 - docs/zh/28-releases/01-tdengine.md | 4 - docs/zh/28-releases/03-notes/3.3.4.3.md | 123 +++++++++++++----------- docs/zh/28-releases/03-notes/index.md | 1 - 4 files changed, 66 insertions(+), 66 deletions(-) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index 161efe0d61..b24931b166 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -24,10 +24,6 @@ import Release from "/components/ReleaseV3"; -## 3.3.4.3 - - - ## 3.3.3.0 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index d285cb36ae..cf9d7b6878 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -28,10 +28,6 @@ import Release from "/components/ReleaseV3"; -## 3.3.4.3 - - - ## 3.3.3.0 diff --git a/docs/zh/28-releases/03-notes/3.3.4.3.md b/docs/zh/28-releases/03-notes/3.3.4.3.md index 364732b478..8ffd5802ed 100644 --- a/docs/zh/28-releases/03-notes/3.3.4.3.md +++ b/docs/zh/28-releases/03-notes/3.3.4.3.md @@ -1,60 +1,69 @@ -## 新特性 -* 新功能：流计算的 TWA 函数支持时间驱动的结果推送模式 -* 新功能：流计算的 Interp 函数支持时间驱动的结果推送模式 -* 优化：顺序执行 compact 和 split vgroup操作时的日志错误提示 -* 新功能：支持微软对象存储 +--- +title: 3.3.4.3 版本说明 +sidebar_label: 3.3.4.3 +description: 3.3.4.3 版本说明 +--- -## 优化 -* 优化：提升并发大查询时节点之间互相拉数据的效率 -* 优化：支持使用 AVX2 和 AVX512 对 double 、timestamp 和 bigint 类型进行解码优化 -* 优化：调整 case when 语句的结果类型判断方法 -* 优化：提升查询 “select ... from ... where ts in (...)” 的数据扫描速度 -* 优化：增加了流计算的兼容性保证机制，避免后续函数变更产生新的兼容性问题，之前版本的流计算必须重建 -* 优化：提升 taosX 在交叉写入场景下的数据同步性能 -* 优化：支持关闭整数/浮点数类型的编码 -* 优化：多副本流计算中必须使用 snode -* 优化：客户端生成唯一 ID 标识每一个查询任务，避免重复 ID 导致的内存损坏 -* 优化：加快数据库的创建时间 -* 优化：修改 s3MigrateEnabled 默认值为0 -* 优化：支持在审计数据库中记录删除操作 -* 优化：支持在指定的 dnode 中创建数据库 [企业版] -* 优化：调整删除超级表数据列时的报错信息 +### 行为变更及兼容性 +1. 多副本流计算中必须使用 snode +1. 增加了流计算的兼容性保证机制，避免后续函数变更产生新的兼容性问题，但之前版本的流计算必须重建，具体参见 https://docs.taosdata.com/advanced/stream/#流计算升级故障恢复 +1. 调整 case when 语句结果类型的判断方法 + +### 新特性 +1. 新功能：流计算的 TWA 函数支持时间驱动的结果推送模式 +1. 新功能：流计算的 Interp 函数支持时间驱动的结果推送模式 +1. 新功能：支持微软对象存储 -## 修复 -* 修复：last_row 查询性能在 3.3.3.0 中大幅下降的问题 -* 修复：WAL 条目不完整时 taosd 无法启动的问题 -* 修复: partition by 常量时查询结果错误的问题 -* 修复：标量函数包含 _wstart 且填充方式为 prev 时计算结果错误 -* 修复：Windows 平台下的时区设置问题 -* 修复：空数据库进行 compact 操作时，事务无法结束【企业版】 -* 修复：事务冲突的逻辑错误 -* 修复：管理节点某些错误会导致事务无法停止 -* 修复：管理节点某些错误会导致事务无法停止 -* 修复：dnode 数据清空后 taosc 重试错误的问题 -* 修复：Data Compact 被异常终止后，中间文件未被清理 -* 修复：新增列后，Kafka 连接器的 earliest 模式消费不到新列数据 -* 修复：interp 函数在 fill(prev) 时行为不正确 -* 修复：TSMA 在高频元数据操作时异常停止的问题 -* 修复：show create stable 语句执行结果的标签显示错误 -* 修复：Percentile 函数在大数据量查询时会崩溃。 -* 修复：partition by 和 having 联合使用时的语法错误问题 -* 修复：interp 在 partition by tbname,c1 时 tbname 为空的问题 -* 修复：通过 stmt 写入非法布尔数值时 taosd 可能 crash -* 修复：Explorer OPC-UA 表名模板说明 -* 修复：库符号 version 与使用相同符号的库冲突的问题 https://github.com/taosdata/TDengine/issues/25920 -* 修复：在 windows 平台下 JDBC 驱动的句柄数持续升高问题 -* 修复：3.3.3.1 升级至 3.3.4.0 偶现的启动失败问题 -* 修复：Windows 平台重复增删表的内存泄漏 -* 修复：无法限制并发拉起 checkpoint 数量导致流计算消耗资源过多 -* 修复：并发查询时的 too many session 问题 -* 修复：Windows 平台下 taos shell 在慢查询场景中崩溃的问题 -* 修复：当打开 dnode日志时，加密数据库无法恢复的问题 -* 修复：由于 mnode 同步超时，进而导致 taosd 无法启动的问题 -* 修复：由于在快照同步过程中整理文件组数据的速度过慢，从而导致 Vnode（虚拟节点）无法恢复的问题 -* 修复：通过行协议向字符串类型的字段中写入带转义符的数据时，taosd 会崩溃 -* 修复：Error Code 逻辑处理错误导致的元数据文件损坏 -* 修复：查询语句中包含多个 “not” 条件语句嵌套时，未设置标量模式导致查询错误 -* 修复：vnode 统计信息上报超时导致的 dnode offline 问题 -* 修复：在不支持 avx 指令集的服务器上，taosd 启动失败问题 -* 修复：taosX 数据迁移容错处理 0x09xx 错误码 +### 优化 +1. 优化：提升并发大查询时节点之间互相拉数据的效率 +1. 优化：支持使用 AVX2 和 AVX512 对 double 、timestamp 和 bigint 类型进行解码优化 +1. 优化：调整 case when 语句的结果类型判断方法 +1. 优化：顺序执行 compact 和 split vgroup操作时的日志错误提示 +1. 优化：提升查询 “select ... from ... where ts in (...)” 的数据扫描速度 +1. 优化：增加了流计算的兼容性保证机制，避免后续函数变更产生新的兼容性问题，之前版本的流计算必须重建 +1. 优化：提升 taosX 在交叉写入场景下的数据同步性能 +1. 优化：支持关闭整数/浮点数类型的编码 +1. 优化：多副本流计算中必须使用 snode +1. 优化：客户端生成唯一 ID 标识每一个查询任务，避免重复 ID 导致的内存损坏 +1. 优化：加快数据库的创建时间 +1. 优化：修改 s3MigrateEnabled 默认值为0 +1. 优化：支持在审计数据库中记录删除操作 +1. 优化：支持在指定的 dnode 中创建数据库 [企业版] +1. 优化：调整删除超级表数据列时的报错信息 +### 修复 +1. 修复：last_row 查询性能在 3.3.3.0 中大幅下降的问题 +1. 修复：WAL 条目不完整时 taosd 无法启动的问题 +1. 修复: partition by 常量时查询结果错误的问题 +1. 修复：标量函数包含 _wstart 且填充方式为 prev 时计算结果错误 +1. 修复：Windows 平台下的时区设置问题 +1. 修复：空数据库进行 compact 操作时，事务无法结束【企业版】 +1. 修复：事务冲突的逻辑错误 +1. 修复：管理节点某些错误会导致事务无法停止 +1. 修复：管理节点某些错误会导致事务无法停止 +1. 修复：dnode 数据清空后 taosc 重试错误的问题 +1. 修复：Data Compact 被异常终止后，中间文件未被清理 +1. 修复：新增列后，Kafka 连接器的 earliest 模式消费不到新列数据 +1. 修复：interp 函数在 fill(prev) 时行为不正确 +1. 修复：TSMA 在高频元数据操作时异常停止的问题 +1. 修复：show create stable 语句执行结果的标签显示错误 +1. 修复：Percentile 函数在大数据量查询时会崩溃。 +1. 修复：partition by 和 having 联合使用时的语法错误问题 +1. 修复：interp 在 partition by tbname,c1 时 tbname 为空的问题 +1. 修复：通过 stmt 写入非法布尔数值时 taosd 可能 crash +1. 修复：库符号 version 与使用相同符号的库冲突的问题 +1. 修复：在 windows 平台下 JDBC 驱动的句柄数持续升高问题 +1. 修复：3.3.3.1 升级至 3.3.4.0 偶现的启动失败问题 +1. 修复：Windows 平台重复增删表的内存泄漏 +1. 修复：无法限制并发拉起 checkpoint 数量导致流计算消耗资源过多 +1. 修复：并发查询时的 too many session 问题 +1. 修复：Windows 平台下 taos shell 在慢查询场景中崩溃的问题 +1. 修复：当打开 dnode日志时，加密数据库无法恢复的问题 +1. 修复：由于 mnode 同步超时，进而导致 taosd 无法启动的问题 +1. 修复：由于在快照同步过程中整理文件组数据的速度过慢，从而导致 Vnode（虚拟节点）无法恢复的问题 +1. 修复：通过行协议向字符串类型的字段中写入带转义符的数据时，taosd 会崩溃 +1. 修复：Error Code 逻辑处理错误导致的元数据文件损坏 +1. 修复：查询语句中包含多个 “not” 条件语句嵌套时，未设置标量模式导致查询错误 +1. 修复：vnode 统计信息上报超时导致的 dnode offline 问题 +1. 修复：在不支持 avx 指令集的服务器上，taosd 启动失败问题 +1. 修复：taosX 数据迁移容错处理 0x09xx 错误码 diff --git a/docs/zh/28-releases/03-notes/index.md b/docs/zh/28-releases/03-notes/index.md index 1cdd3239e7..d1a48ab9a8 100644 --- a/docs/zh/28-releases/03-notes/index.md +++ b/docs/zh/28-releases/03-notes/index.md @@ -4,7 +4,6 @@ sidebar_label: 版本说明 description: 各版本版本说明 --- -[3.3.4.3](./3.3.4.3) [3.3.4.3](./3.3.4.3) [3.3.3.0](./3.3.3.0) [3.3.2.0](./3.3.2.0)