From af949927d88ab6728daadfe6befd9b666fc1688f Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Fri, 16 Jun 2023 13:40:01 +0800 Subject: [PATCH 001/100] docs: use english website for english docs --- docs/en/28-releases/01-tdengine.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index a9336697f2..6ea6c5d215 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -6,7 +6,7 @@ description: This document provides download links for all released versions of TDengine 3.x installation packages can be downloaded at the following links: -For TDengine 2.x installation packages by version, please visit [here](https://www.taosdata.com/all-downloads). +For TDengine 2.x installation packages by version, please visit [here](https://tdengine.com/downloads/historical/). import Release from "/components/ReleaseV3"; From 9785874bb2e6ae2f77bc9f2129a47cae0e07a5c8 Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Fri, 16 Jun 2023 13:40:44 +0800 Subject: [PATCH 002/100] docs: align get started page changes --- docs/en/05-get-started/index.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docs/en/05-get-started/index.md b/docs/en/05-get-started/index.md index 66573a89cd..cc3b4826dd 100644 --- a/docs/en/05-get-started/index.md +++ b/docs/en/05-get-started/index.md @@ -21,17 +21,6 @@ import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; ``` -## Study TDengine Knowledge Map - -The TDengine Knowledge Map covers the various knowledge points of TDengine, revealing the invocation relationships and data flow between various conceptual entities. Learning and understanding the TDengine Knowledge Map will help you quickly master the TDengine knowledge system. - -
-
- -
Diagram 1. TDengine Knowledge Map
-
-
- ## Join TDengine Community From 8fb4334f005ce647d81da2bd698c3939ab63b3f5 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 7 Jul 2023 10:31:11 +0800 Subject: [PATCH 003/100] test:temp commite --- tests/system-test/0-others/compatibility.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 22e319fdaf..5067275471 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -203,7 +203,6 @@ class TDTestCase: tdsql.execute("insert into db.`ct4` using db.stb1 TAGS(4) values(now(),14);") tdsql.query("select * from db.ct4") tdsql.checkData(0,1,14) - print(1) tdsql=tdCom.newTdSql() tdsql.query("describe information_schema.ins_databases;") qRows=tdsql.queryRows From f5e3e574ae462248df77104907024e584f2c5ece Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 7 Jul 2023 16:18:54 +0800 Subject: [PATCH 004/100] test:add testcase of rolling upgdade --- tests/pytest/util/dnodes.py | 2 + .../system-test/6-cluster/5dnode3mnodeRoll.py | 323 ++++++++++++++++++ 2 files changed, 325 insertions(+) create mode 100644 tests/system-test/6-cluster/5dnode3mnodeRoll.py diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 543433b4ea..89e3df81b9 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -545,6 +545,8 @@ class TDDnode: def stoptaosd(self): + tdLog.debug("start to stop taosd on dnode: %d "% (self.index)) + # print(self.asan,self.running,self.remoteIP,self.valgrind) if self.asan: stopCmd = "%s -s stop -n dnode%d" % (self.execPath, self.index) tdLog.info("execute script: " + stopCmd) diff --git a/tests/system-test/6-cluster/5dnode3mnodeRoll.py b/tests/system-test/6-cluster/5dnode3mnodeRoll.py new file mode 100644 index 0000000000..1b36dfef44 --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeRoll.py @@ -0,0 +1,323 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +from util.common import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck +from pathlib import Path +from taos.tmq import Consumer + + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +BASEVERSION = "3.0.5.0" + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.replicaVar = int(replicaVar) + + def checkProcessPid(self,processName): + i=0 + while i<60: + print(f"wait stop {processName}") + processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1] + print(f"times:{i},{processName}-pid:{processPid}") + if(processPid == ""): + break + i += 1 + sleep(1) + else: + print(f'this processName is not stoped in 60s') + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def getCfgPath(self): + buildPath = self.getBuildPath() + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + cfgPath = buildPath + "/../sim/dnode1/cfg/" + else: + cfgPath = buildPath + "/../sim/dnode1/cfg/" + + return cfgPath + + def installTaosd(self,bPath,cPath): + # os.system(f"rmtaos && mkdir -p {self.getBuildPath()}/build/lib/temp && mv {self.getBuildPath()}/build/lib/libtaos.so* {self.getBuildPath()}/build/lib/temp/ ") + # os.system(f" mv {bPath}/build {bPath}/build_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so {self.getBuildPath()}/build/lib/libtaos.so_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so.1 {self.getBuildPath()}/build/lib/libtaos.so.1_bak ") + + packagePath = "/usr/local/src/" + dataPath = cPath + "/../data/" + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" + packageTPath = packageName.split("-Linux-")[0] + my_file = Path(f"{packagePath}/{packageName}") + if not my_file.exists(): + print(f"{packageName} is not exists") + tdLog.info(f"cd {packagePath} && wget https://www.tdengine.com/assets-download/3.0/{packageName}") + os.system(f"cd {packagePath} && wget https://www.tdengine.com/assets-download/3.0/{packageName}") + else: + print(f"{packageName} has been exists") + os.system(f" cd {packagePath} && tar xvf {packageName} && cd {packageTPath} && ./install.sh -e no " ) + # tdDnodes.stop(1) + # print(f"start taosd: rm -rf {dataPath}/* && nohup taosd -c {cPath} & ") + # os.system(f"rm -rf {dataPath}/* && nohup taosd -c {cPath} & " ) + # sleep(5) + + + def buildTaosd(self,bPath): + # os.system(f"mv {bPath}/build_bak {bPath}/build ") + os.system(f" cd {bPath} ") + + def is_list_same_as_ordered_list(self,unordered_list, ordered_list): + sorted_list = sorted(unordered_list) + return sorted_list == ordered_list + + def insertAllData(self,cPath): + tableNumbers=100 + recordNumbers1=100 + recordNumbers2=1000 + tdLog.info(f"insertAllData") + tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -c {cPath} -n {recordNumbers1} -a 3 -y -k '-1' -z 5 ") + os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -c {cPath} -n {recordNumbers1} -a 3 -y -k '-1' -z 5 ") + # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") + # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') + # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') + print(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/compa4096.json ") + + os.system(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/compa4096.json ") + os.system('LD_LIBRARY_PATH=/usr/lib taos -s "alter database test WAL_RETENTION_PERIOD 1000" ') + os.system('LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists tmq_test_topic as select current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ') + os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show topics;" ') + tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql") + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db0_0', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'stbNumbers': 2, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 200, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + "rowsPerTbl": 1000, + "batchNum": 5000 + } + hostname = socket.gethostname() + dnodeNumbers=int(dnodeNumbers) + + tdLog.info("first check dnode and mnode") + tdSql=tdCom.newTdSql() + tdSql.query("select * from information_schema.ins_dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + + tdLog.printNoPrefix(f"==========step1:prepare cluster of {dnodeNumbers} dnodes whith old version-{BASEVERSION} ") + + scriptsPath = os.path.dirname(os.path.realpath(__file__)) + distro_id = distro.id() + if distro_id == "alpine": + tdLog.info(f"alpine skip Roll test") + return True + if platform.system().lower() == 'windows': + tdLog.info(f"Windows skip Roll test") + return True + + tdLog.info("====step1.1:stop all taosd and clear data dir,then start all old taosd ====") + + bPath = self.getBuildPath() + cPath = self.getCfgPath() + tdDnodes=cluster.dnodes + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + self.installTaosd(bPath,cPath) + for i in range(dnodeNumbers): + dnode_cfgPath = tdDnodes[i].cfgDir + dnode_dataPath = tdDnodes[i].dataDir + os.system(f"rm -rf {dnode_dataPath}/* && nohup taosd -c {dnode_cfgPath} & ") + + tdLog.info("====step1.2: create dnode on cluster ====") + + for i in range(1,dnodeNumbers): + dnode_id = tdDnodes[i].cfgDict["fqdn"] + ":" + tdDnodes[i].cfgDict["serverPort"] + os.system(f" LD_LIBRARY_PATH=/usr/lib taos -s 'create dnode \"{dnode_id}\" ' ") + + os.system(" LD_LIBRARY_PATH=/usr/lib taos -s 'show dnodes' ") + sleep(5) + tdLog.info("====step1.3: insert data, includes time data, tmq and stream ====") + tableNumbers=100 + recordNumbers1=100 + recordNumbers2=1000 + + dbname = "test" + stb = f"{dbname}.meters" + # os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ") + threads=[] + threads.append(threading.Thread(target=self.insertAllData, args=(cPath,))) + for tr in threads: + tr.start() + sleep(10) + tdLog.printNoPrefix("==========step2:start to rolling upgdade ") + for i in range(dnodeNumbers): + tdDnodes[i].running = 1 + tdDnodes[i].stoptaosd() + sleep(2) + tdDnodes[i].starttaosd() + + for tr in threads: + tr.join() + + tdsql=tdCom.newTdSql() + print(tdsql) + tdsql.query("select * from information_schema.ins_dnodes;") + tdLog.info(tdsql.queryResult) + tdsql.checkData(2,1,'%s:6230'%self.host) + tdSql=tdCom.newTdSql() + clusterComCheck.checkDnodes(dnodeNumbers) + + tdsql.query(f"SELECT SERVER_VERSION();") + nowServerVersion=tdsql.queryResult[0][0] + tdLog.info(f"New server version is {nowServerVersion}") + tdsql.query(f"SELECT CLIENT_VERSION();") + nowClientVersion=tdsql.queryResult[0][0] + tdLog.info(f"New client version is {nowClientVersion}") + + tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") + tdsql.query(f"select count(*) from {stb}") + tdsql.checkData(0,0,tableNumbers*recordNumbers1) + tdsql.query(f"select count(*) from db4096.stb0") + tdsql.checkData(0,0,50000) + + # tdsql.query("show streams;") + # tdsql.checkRows(2) + tdsql.query("select *,tbname from d0.almlog where mcid='m0103';") + tdsql.checkRows(6) + expectList = [0,3003,20031,20032,20033,30031] + resultList = [] + for i in range(6): + resultList.append(tdsql.queryResult[i][3]) + print(resultList) + if self.is_list_same_as_ordered_list(resultList,expectList): + print("The unordered list is the same as the ordered list.") + else: + tdlog.error("The unordered list is not the same as the ordered list.") + tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);") + tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);") + + conn = taos.connect() + + consumer = Consumer( + { + "group.id": "tg75", + "client.id": "124", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "enable.auto.commit": "true", + "experimental.snapshot.enable": "true", + } + ) + consumer.subscribe(["tmq_test_topic"]) + + while True: + res = consumer.poll(10) + if not res: + break + err = res.error() + if err is not None: + raise err + val = res.value() + + for block in val: + print(block.fetchall()) + tdsql.query("show topics;") + tdsql.checkRows(1) + + + # #check mnode status + # tdLog.info("check mnode status") + # clusterComCheck.checkMnodeStatus(mnodeNums) + + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=3,mnodeNums=3,restartNumbers=2,stopRole='dnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 47945704212442e6083d76c892b337e70703f6da Mon Sep 17 00:00:00 2001 From: huolibo Date: Tue, 11 Jul 2023 11:49:05 +0800 Subject: [PATCH 005/100] docs(driver): java seek desc --- docs/en/14-reference/03-connector/04-java.mdx | 6 ++++-- docs/zh/08-connector/14-java.mdx | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index b68aeda94c..69bbd287ed 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -36,8 +36,8 @@ REST connection supports all platforms that can run Java. | taos-jdbcdriver version | major changes | TDengine version | | :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | :--------------: | -| 3.2.4 | Subscription add the enable.auto.commit parameter and the unsubscribe() method in the WebSocket connection | 3.0.5.0 or later | -| 3.2.3 | Fixed resultSet data parsing failure in some cases | 3.0.5.0 or later | +| 3.2.4 | Subscription add the enable.auto.commit parameter and the unsubscribe() method in the WebSocket connection | - | +| 3.2.3 | Fixed resultSet data parsing failure in some cases | - | | 3.2.2 | Subscription add seek function | 3.0.5.0 or later | | 3.2.1 | JDBC REST connection supports schemaless/prepareStatement over WebSocket | 3.0.3.0 or later | | 3.2.0 | This version has been deprecated | - | @@ -1019,11 +1019,13 @@ while(true) { #### Assignment subscription Offset ```java +// get offset long position(TopicPartition partition) throws SQLException; Map position(String topic) throws SQLException; Map beginningOffsets(String topic) throws SQLException; Map endOffsets(String topic) throws SQLException; +// Overrides the fetch offsets that the consumer will use on the next poll(timeout). void seek(TopicPartition partition, long offset) throws SQLException; ``` diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 96f8991eea..5dcdd61a5f 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -1022,11 +1022,13 @@ while(true) { #### 指定订阅 Offset ```java +// 获取 offset long position(TopicPartition partition) throws SQLException; Map position(String topic) throws SQLException; Map beginningOffsets(String topic) throws SQLException; Map endOffsets(String topic) throws SQLException; +// 指定下一次 poll 中使用的 offset void seek(TopicPartition partition, long offset) throws SQLException; ``` From cb62d4cb97a6b1be8e5c4dcf72a6d9c0af677105 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 11 Jul 2023 14:49:53 +0800 Subject: [PATCH 006/100] fix:set firset version to reqOffset of response --- include/util/taoserror.h | 3 ++ source/client/src/clientTmq.c | 14 ++--- source/common/src/tmsg.c | 61 +--------------------- source/dnode/mnode/impl/inc/mndDef.h | 11 ++-- source/dnode/mnode/impl/src/mndConsumer.c | 16 +++--- source/dnode/mnode/impl/src/mndSubscribe.c | 6 +-- source/dnode/vnode/src/tq/tqUtil.c | 17 ++++-- source/util/src/terror.c | 3 ++ 8 files changed, 43 insertions(+), 88 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ffeabc5684..8b10e4217c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -772,6 +772,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x4004) #define TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x4005) #define TSDB_CODE_TMQ_SNAPSHOT_ERROR TAOS_DEF_ERROR_CODE(0, 0x4006) +#define TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x4007) +#define TSDB_CODE_TMQ_INVALID_VGID TAOS_DEF_ERROR_CODE(0, 0x4008) +#define TSDB_CODE_TMQ_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x4009) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 0821719f4e..78f45be6bf 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2259,9 +2259,9 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { int64_t tmq_get_vgroup_offset(TAOS_RES* res) { if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*) res; - STqOffsetVal* pOffset = &pRspObj->rsp.rspOffset; + STqOffsetVal* pOffset = &pRspObj->rsp.reqOffset; if (pOffset->type == TMQ_OFFSET__LOG) { - return pRspObj->rsp.rspOffset.version; + return pRspObj->rsp.reqOffset.version; } } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pRspObj = (SMqMetaRspObj*)res; @@ -2270,8 +2270,8 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { } } else if (TD_RES_TMQ_METADATA(res)) { SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*) res; - if (pRspObj->rsp.rspOffset.type == TMQ_OFFSET__LOG) { - return pRspObj->rsp.rspOffset.version; + if (pRspObj->rsp.reqOffset.type == TMQ_OFFSET__LOG) { + return pRspObj->rsp.reqOffset.version; } } @@ -2761,7 +2761,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ if (pTopic == NULL) { tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_INVALID_PARA; + return TSDB_CODE_TMQ_INVALID_TOPIC; } SMqClientVg* pVg = NULL; @@ -2777,7 +2777,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ if (pVg == NULL) { tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId); taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_INVALID_PARA; + return TSDB_CODE_TMQ_INVALID_VGID; } SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; @@ -2793,7 +2793,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd); taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_INVALID_PARA; + return TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE; } // update the offset, and then commit to vnode diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 38806b6042..f6b3d0ca49 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -7351,27 +7351,8 @@ void tDeleteMqDataRsp(SMqDataRsp *pRsp) { } int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { - if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; - if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; - if (pRsp->blockNum != 0) { - if (tEncodeI8(pEncoder, pRsp->withTbName) < 0) return -1; - if (tEncodeI8(pEncoder, pRsp->withSchema) < 0) return -1; + if (tEncodeMqDataRsp(pEncoder, (const SMqDataRsp *)pRsp) < 0) return -1; - for (int32_t i = 0; i < pRsp->blockNum; i++) { - int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i); - void *data = taosArrayGetP(pRsp->blockData, i); - if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1; - if (pRsp->withSchema) { - SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i); - if (tEncodeSSchemaWrapper(pEncoder, pSW) < 0) return -1; - } - if (pRsp->withTbName) { - char *tbName = (char *)taosArrayGetP(pRsp->blockTbName, i); - if (tEncodeCStr(pEncoder, tbName) < 0) return -1; - } - } - } if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { for (int32_t i = 0; i < pRsp->createTableNum; i++) { @@ -7384,46 +7365,8 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { } int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { - if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; - if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; - if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; - if (pRsp->blockNum != 0) { - pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *)); - pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); - if (tDecodeI8(pDecoder, &pRsp->withTbName) < 0) return -1; - if (tDecodeI8(pDecoder, &pRsp->withSchema) < 0) return -1; - if (pRsp->withTbName) { - pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void *)); - } - if (pRsp->withSchema) { - pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void *)); - } + if (tDecodeMqDataRsp(pDecoder, (SMqDataRsp*)pRsp) < 0) return -1; - for (int32_t i = 0; i < pRsp->blockNum; i++) { - void *data; - uint64_t bLen; - if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1; - taosArrayPush(pRsp->blockData, &data); - int32_t len = bLen; - taosArrayPush(pRsp->blockDataLen, &len); - - if (pRsp->withSchema) { - SSchemaWrapper *pSW = (SSchemaWrapper *)taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - if (pSW == NULL) return -1; - if (tDecodeSSchemaWrapper(pDecoder, pSW) < 0) { - taosMemoryFree(pSW); - return -1; - } - taosArrayPush(pRsp->blockSchema, &pSW); - } - - if (pRsp->withTbName) { - char *tbName; - if (tDecodeCStrAlloc(pDecoder, &tbName) < 0) return -1; - taosArrayPush(pRsp->blockTbName, &tbName); - } - } - } if (tDecodeI32(pDecoder, &pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t)); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 696549fa05..44dbfe6b12 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -137,12 +137,11 @@ typedef enum { } EDndReason; typedef enum { - CONSUMER_UPDATE_REB_MODIFY_NOTOPIC = 1, // topic do not need modified after rebalance - CONSUMER_UPDATE_REB_MODIFY_TOPIC, // topic need modified after rebalance - CONSUMER_UPDATE_REB_MODIFY_REMOVE, // topic need removed after rebalance -// CONSUMER_UPDATE_TIMER_LOST, - CONSUMER_UPDATE_RECOVER, - CONSUMER_UPDATE_SUB_MODIFY, // modify after subscribe req + CONSUMER_UPDATE_REB = 1, // update after rebalance + CONSUMER_ADD_REB, // add after rebalance + CONSUMER_REMOVE_REB, // remove after rebalance + CONSUMER_UPDATE_REC, // update after recover + CONSUMER_UPDATE_SUB, // update after subscribe req } ECsmUpdateType; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index bdf9931ca2..2b538eccc9 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -184,7 +184,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { } SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup); - pConsumerNew->updateType = CONSUMER_UPDATE_RECOVER; + pConsumerNew->updateType = CONSUMER_UPDATE_REC; mndReleaseConsumer(pMnode, pConsumer); @@ -701,7 +701,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { pConsumerNew->autoCommitInterval = subscribe.autoCommitInterval; pConsumerNew->resetOffsetCfg = subscribe.resetOffsetCfg; -// pConsumerNew->updateType = CONSUMER_UPDATE_SUB_MODIFY; // use insert logic +// pConsumerNew->updateType = CONSUMER_UPDATE_SUB; // use insert logic taosArrayDestroy(pConsumerNew->assignedTopics); pConsumerNew->assignedTopics = taosArrayDup(pTopicList, topicNameDup); @@ -731,7 +731,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { } // set the update type - pConsumerNew->updateType = CONSUMER_UPDATE_SUB_MODIFY; + pConsumerNew->updateType = CONSUMER_UPDATE_SUB; taosArrayDestroy(pConsumerNew->assignedTopics); pConsumerNew->assignedTopics = taosArrayDup(pTopicList, topicNameDup); @@ -984,7 +984,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, taosWLockLatch(&pOldConsumer->lock); - if (pNewConsumer->updateType == CONSUMER_UPDATE_SUB_MODIFY) { + if (pNewConsumer->updateType == CONSUMER_UPDATE_SUB) { TSWAP(pOldConsumer->rebNewTopics, pNewConsumer->rebNewTopics); TSWAP(pOldConsumer->rebRemovedTopics, pNewConsumer->rebRemovedTopics); TSWAP(pOldConsumer->assignedTopics, pNewConsumer->assignedTopics); @@ -1004,7 +1004,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, // mInfo("consumer:0x%" PRIx64 " timer update, timer lost. state %s -> %s, reb-time:%" PRId64 ", reb-removed-topics:%d", // pOldConsumer->consumerId, mndConsumerStatusName(prevStatus), mndConsumerStatusName(pOldConsumer->status), // pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); - } else if (pNewConsumer->updateType == CONSUMER_UPDATE_RECOVER) { + } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REC) { int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); for (int32_t i = 0; i < sz; i++) { char *topic = taosStrdup(taosArrayGetP(pOldConsumer->assignedTopics, i)); @@ -1013,12 +1013,12 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; mInfo("consumer:0x%" PRIx64 " timer update, timer recover",pOldConsumer->consumerId); - } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REB_MODIFY_NOTOPIC) { + } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REB) { atomic_add_fetch_32(&pOldConsumer->epoch, 1); pOldConsumer->rebalanceTime = taosGetTimestampMs(); mInfo("consumer:0x%" PRIx64 " reb update, only rebalance time", pOldConsumer->consumerId); - } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REB_MODIFY_TOPIC) { + } else if (pNewConsumer->updateType == CONSUMER_ADD_REB) { char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); // check if exist in current topic @@ -1049,7 +1049,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics), (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); - } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REB_MODIFY_REMOVE) { + } else if (pNewConsumer->updateType == CONSUMER_REMOVE_REB) { char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); // remove from removed topic diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 48de21199b..b2235c8b50 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -597,7 +597,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->modifyConsumers, i); SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); - pConsumerNew->updateType = CONSUMER_UPDATE_REB_MODIFY_NOTOPIC; + pConsumerNew->updateType = CONSUMER_UPDATE_REB; if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { tDeleteSMqConsumerObj(pConsumerNew, true); @@ -613,7 +613,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->newConsumers, i); SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); - pConsumerNew->updateType = CONSUMER_UPDATE_REB_MODIFY_TOPIC; + pConsumerNew->updateType = CONSUMER_ADD_REB; char* topicTmp = taosStrdup(topic); taosArrayPush(pConsumerNew->rebNewTopics, &topicTmp); @@ -633,7 +633,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i); SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); - pConsumerNew->updateType = CONSUMER_UPDATE_REB_MODIFY_REMOVE; + pConsumerNew->updateType = CONSUMER_REMOVE_REB; char* topicTmp = taosStrdup(topic); taosArrayPush(pConsumerNew->rebRemovedTopics, &topicTmp); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 34c5112eee..4365cf63a7 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -157,18 +157,23 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand return 0; } +static void setRequestVersion(STqOffsetVal* offset, int64_t ver){ + if(offset->type == TMQ_OFFSET__LOG){ + offset->version = ver + 1; + } +} + static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* pOffset) { uint64_t consumerId = pRequest->consumerId; int32_t vgId = TD_VID(pTq->pVnode); - int code = 0; terrno = 0; SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, pRequest); qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); - code = tqScanData(pTq, pHandle, &dataRsp, pOffset); + int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); if (code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; } @@ -183,11 +188,10 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, code = tqRegisterPushHandle(pTq, pHandle, pMsg); taosWUnLockLatch(&pTq->lock); goto end; - } else { - taosWUnLockLatch(&pTq->lock); } + taosWUnLockLatch(&pTq->lock); } - + setRequestVersion(&dataRsp.reqOffset, pOffset->version); code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); end : { @@ -261,6 +265,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + setRequestVersion(&taosxRsp.reqOffset, offset->version); code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); goto end; } @@ -273,6 +278,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (pHead->msgType != TDMT_VND_SUBMIT) { if (totalRows > 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1); + setRequestVersion(&taosxRsp.reqOffset, offset->version); code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); goto end; } @@ -302,6 +308,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (totalRows >= 4096 || taosxRsp.createTableNum > 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + setRequestVersion(&taosxRsp.reqOffset, offset->version); code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); goto end; } else { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index f0c7b22bb1..4c52f89bdc 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -629,6 +629,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_INVALID_FILE, "Index file is inval //tmq TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_SNAPSHOT_ERROR, "Can not operate in snapshot mode") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE, "Offset out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_VGID, "VgId does not belong to this consumer") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_TOPIC, "Topic does not belong to this consumer") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_ERROR, "Consumer error, to see log") From 7bc19df0ef1b6a1ec31d8cdfefe7c0b2cb4b7843 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 11 Jul 2023 15:48:19 +0800 Subject: [PATCH 007/100] fix:set firset version to reqOffset of response --- source/dnode/vnode/src/tq/tqUtil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 4365cf63a7..7768f71c61 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -191,7 +191,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, } taosWUnLockLatch(&pTq->lock); } - setRequestVersion(&dataRsp.reqOffset, pOffset->version); + setRequestVersion(pOffset, pOffset->version); code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); end : { @@ -213,6 +213,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; tqInitTaosxRsp(&taosxRsp, pRequest); + taosxRsp.reqOffset.type = offset->type; // stroe origin type for getting offset in tmq_get_vgroup_offset if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { From 574010e067070d313f497bb412f9f5eb91dd7c18 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 11 Jul 2023 16:06:19 +0800 Subject: [PATCH 008/100] fix:set firset version to reqOffset of response --- source/client/src/clientTmq.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 78f45be6bf..807e6cb53b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2262,6 +2262,8 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { STqOffsetVal* pOffset = &pRspObj->rsp.reqOffset; if (pOffset->type == TMQ_OFFSET__LOG) { return pRspObj->rsp.reqOffset.version; + }else{ + tscError("invalid offset type:%d", pOffset->type); } } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pRspObj = (SMqMetaRspObj*)res; @@ -2273,6 +2275,8 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { if (pRspObj->rsp.reqOffset.type == TMQ_OFFSET__LOG) { return pRspObj->rsp.reqOffset.version; } + } else{ + tscError("invalid tmqtype:%d", *(int8_t*)res); } // data from tsdb, no valid offset info From 8dd7f36993b5df2830f26a72d06d28ad5ce6b85a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 11 Jul 2023 16:19:38 +0800 Subject: [PATCH 009/100] fix:set firset version to reqOffset of response --- source/client/src/clientTmq.c | 2 +- source/dnode/vnode/src/tq/tqUtil.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 807e6cb53b..50a42d547c 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2276,7 +2276,7 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { return pRspObj->rsp.reqOffset.version; } } else{ - tscError("invalid tmqtype:%d", *(int8_t*)res); + tscError("invalid tmq type:%d", *(int8_t*)res); } // data from tsdb, no valid offset info diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 7768f71c61..8e9f043f62 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -171,6 +171,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, pRequest); + dataRsp.reqOffset.type = pOffset->type; // stroe origin type for getting offset in tmq_get_vgroup_offset qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); @@ -191,7 +192,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, } taosWUnLockLatch(&pTq->lock); } - setRequestVersion(pOffset, pOffset->version); + setRequestVersion(&dataRsp.reqOffset, pOffset->version); code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); end : { From 6298df73b9b0c0eeda550641442232666b456ea3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 12 Jul 2023 10:29:27 +0800 Subject: [PATCH 010/100] docs: refine get-started windows section --- docs/en/05-get-started/03-package.md | 2 +- docs/zh/05-get-started/03-package.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index 5a54c32a51..91bf94034c 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -201,7 +201,7 @@ You can use the TDengine CLI to monitor your TDengine deployment and execute ad -After the installation is complete, please run `sc start taosd` or run `C:\TDengine\taosd.exe` with administrator privilege to start TDengine Server. +After the installation is complete, please run `sc start taosd` or run `C:\TDengine\taosd.exe` with administrator privilege to start TDengine Server. Please run `sc start taosadapter` or run `C:\TDengine\taosadapter.exe` with administrator privilege to start taosAdapter to provide http/REST service. ## Command Line Interface (CLI) diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index f6d1c85a60..621effa6fd 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -201,7 +201,7 @@ Active: inactive (dead) -安装后，可以在拥有管理员权限的 cmd 窗口执行 `sc start taosd` 或在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。 +安装后，可以在拥有管理员权限的 cmd 窗口执行 `sc start taosd` 或在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。如需使用 http/REST 服务，请执行 `sc start taosadapter` 或运行 `taosadapter.exe` 来启动 taosAdapter 服务进程。 **TDengine 命令行（CLI）** From cf64d4c9c55a104b9b72a3b69d6e05f7c4175d1d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 12 Jul 2023 17:22:23 +0800 Subject: [PATCH 011/100] fix:set get_assignment offset to first version of response block --- include/common/tmsg.h | 8 + include/common/tmsgdef.h | 2 +- source/client/src/clientTmq.c | 125 ++++++++++---- source/client/test/clientTests.cpp | 20 ++- source/common/src/tmsg.c | 42 +++++ source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tq/tq.c | 170 ++++++++++++-------- source/dnode/vnode/src/tq/tqUtil.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 7 +- 10 files changed, 271 insertions(+), 109 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 2d75424bb5..0c58b470c2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3371,6 +3371,12 @@ typedef struct { int8_t reserved; } SMqHbRsp; +typedef struct { + SMsgHead head; + int64_t consumerId; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; +} SMqSeekReq; + #define TD_AUTO_CREATE_TABLE 0x1 typedef struct { int64_t suid; @@ -3500,6 +3506,8 @@ int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); int32_t tDeatroySMqHbReq(SMqHbReq* pReq); +int32_t tSerializeSMqSeekReq(void *buf, int32_t bufLen, SMqSeekReq *pReq); +int32_t tDeserializeSMqSeekReq(void *buf, int32_t bufLen, SMqSeekReq *pReq); #define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 3fc94f4408..3f4335af94 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -306,7 +306,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DELETE_SUB, "vnode-tmq-delete-sub", SMqVDeleteReq, SMqVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_COMMIT_OFFSET, "vnode-tmq-commit-offset", STqOffset, STqOffset) - TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SEEK_TO_OFFSET, "vnode-tmq-seekto-offset", STqOffset, STqOffset) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SEEK, "vnode-tmq-seek", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_ADD_CHECKINFO, "vnode-tmq-add-checkinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DEL_CHECKINFO, "vnode-del-checkinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME, "vnode-tmq-consume", SMqPollReq, SMqDataBlkRsp) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 50a42d547c..5879de2e30 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -140,6 +140,7 @@ enum { typedef struct SVgOffsetInfo { STqOffsetVal committedOffset; STqOffsetVal currentOffset; + STqOffsetVal seekOffset; // the first version in block for seek operation int64_t walVerBegin; int64_t walVerEnd; } SVgOffsetInfo; @@ -214,6 +215,11 @@ typedef struct SMqVgCommon { int32_t code; } SMqVgCommon; +typedef struct SMqSeekParam { + tsem_t sem; + int32_t code; +} SMqSeekParam; + typedef struct SMqVgWalInfoParam { int32_t vgId; int32_t epoch; @@ -821,7 +827,7 @@ void tmqSendHbReq(void* param, void* tmrId) { OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); offRows->vgId = pVg->vgId; offRows->rows = pVg->numOfRows; - offRows->offset = pVg->offsetInfo.currentOffset; + offRows->offset = pVg->offsetInfo.seekOffset; char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); tscInfo("consumer:0x%" PRIx64 ",report offset: vgId:%d, offset:%s, rows:%"PRId64, tmq->consumerId, offRows->vgId, buf, offRows->rows); @@ -1479,6 +1485,7 @@ CREATE_MSG_FAIL: typedef struct SVgroupSaveInfo { STqOffsetVal currentOffset; STqOffsetVal commitOffset; + STqOffsetVal seekOffset; int64_t numOfRows; } SVgroupSaveInfo; @@ -1518,6 +1525,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic clientVg.offsetInfo.currentOffset = pInfo ? pInfo->currentOffset : offsetNew; clientVg.offsetInfo.committedOffset = pInfo ? pInfo->commitOffset : offsetNew; + clientVg.offsetInfo.seekOffset = pInfo ? pInfo->seekOffset : offsetNew; clientVg.offsetInfo.walVerBegin = -1; clientVg.offsetInfo.walVerEnd = -1; clientVg.seekUpdated = false; @@ -1577,7 +1585,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) tscInfo("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); - SVgroupSaveInfo info = {.currentOffset = pVgCur->offsetInfo.currentOffset, .commitOffset = pVgCur->offsetInfo.committedOffset, .numOfRows = pVgCur->numOfRows}; + SVgroupSaveInfo info = {.currentOffset = pVgCur->offsetInfo.currentOffset, .seekOffset = pVgCur->offsetInfo.seekOffset, .commitOffset = pVgCur->offsetInfo.committedOffset, .numOfRows = pVgCur->numOfRows}; taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)); } } @@ -1879,10 +1887,11 @@ static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* p return 0; } -static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* offset, int64_t sver, int64_t ever, int64_t consumerId){ +static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal* rspOffset, int64_t sver, int64_t ever, int64_t consumerId){ if (!pVg->seekUpdated) { tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", consumerId); - pVg->offsetInfo.currentOffset = *offset; + pVg->offsetInfo.seekOffset = *reqOffset; + pVg->offsetInfo.currentOffset = *rspOffset; } else { tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", consumerId); } @@ -1944,7 +1953,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { pVg->epSet = *pollRspWrapper->pEpset; } - updateVgInfo(pVg, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId); + updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId); char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); @@ -1994,7 +2003,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { return NULL; } - updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId); + updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, &pollRspWrapper->metaRsp.rspOffset, pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); taosFreeQitem(pollRspWrapper); @@ -2022,7 +2031,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { return NULL; } - updateVgInfo(pVg, &pollRspWrapper->taosxRsp.rspOffset, pollRspWrapper->taosxRsp.head.walsver, pollRspWrapper->taosxRsp.head.walever, tmq->consumerId); + updateVgInfo(pVg, &pollRspWrapper->taosxRsp.reqOffset, &pollRspWrapper->taosxRsp.rspOffset, pollRspWrapper->taosxRsp.head.walsver, pollRspWrapper->taosxRsp.head.walever, tmq->consumerId); if (pollRspWrapper->taosxRsp.blockNum == 0) { tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64, @@ -2545,6 +2554,8 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { tsem_post(&pCommon->rsp); } + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pParam); return 0; } @@ -2615,7 +2626,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } tmq_topic_assignment* pAssignment = &(*assignment)[j]; - pAssignment->currentOffset = pClientVg->offsetInfo.currentOffset.version; + pAssignment->currentOffset = pClientVg->offsetInfo.seekOffset.version; pAssignment->begin = pClientVg->offsetInfo.walVerBegin; pAssignment->end = pClientVg->offsetInfo.walVerEnd; pAssignment->vgId = pClientVg->vgId; @@ -2654,6 +2665,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a SMqPollReq req = {0}; tmqBuildConsumeReqImpl(&req, tmq, 10, pTopic, pClientVg); + req.reqOffset = pClientVg->offsetInfo.seekOffset; int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req); if (msgSize < 0) { @@ -2750,6 +2762,17 @@ void tmq_free_assignment(tmq_topic_assignment* pAssignment) { taosMemoryFree(pAssignment); } +static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { + if (pMsg) { + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + } + SMqSeekParam* pParam = param; + pParam->code = code; + tsem_post(&pParam->sem); + return 0; +} + int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { if (tmq == NULL) { tscError("invalid tmq handle, null"); @@ -2803,35 +2826,71 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ // update the offset, and then commit to vnode pOffsetInfo->currentOffset.type = TMQ_OFFSET__LOG; pOffsetInfo->currentOffset.version = offset >= 1 ? offset - 1 : 0; + pOffsetInfo->seekOffset = pOffsetInfo->currentOffset; // pOffsetInfo->committedOffset.version = INT64_MIN; pVg->seekUpdated = true; + tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, vgId); - tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, pVg->vgId); + SMqSeekReq req = {0}; + snprintf(req.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s:%s", tmq->groupId, pTopic->topicName); + req.head.vgId = pVg->vgId; + req.consumerId = tmq->consumerId; + + int32_t msgSize = tSerializeSMqSeekReq(NULL, 0, &req); + if (msgSize < 0) { + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_PAR_INTERNAL_ERROR; + } + + char* msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (tSerializeSMqSeekReq(msg, msgSize, &req) < 0) { + taosMemoryFree(msg); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_PAR_INTERNAL_ERROR; + } + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(msg); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_OUT_OF_MEMORY; + } + + SMqSeekParam* pParam = taosMemoryMalloc(sizeof(SMqSeekParam)); + if (pParam == NULL) { + taosMemoryFree(msg); + taosMemoryFree(sendInfo); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_OUT_OF_MEMORY; + } + tsem_init(&pParam->sem, 0, 0); + + sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL}; + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmqSeekCb; + sendInfo->msgType = TDMT_VND_TMQ_SEEK; + + int64_t transporterId = 0; + tscInfo("consumer:0x%" PRIx64 " %s send seek info vgId:%d, epoch %d" PRIx64, + tmq->consumerId, pTopic->topicName, vgId, tmq->epoch); + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); taosWUnLockLatch(&tmq->lock); -// SMqRspObj rspObj = {.resType = RES_TYPE__TMQ, .vgId = pVg->vgId}; -// tstrncpy(rspObj.topic, tname, tListLen(rspObj.topic)); -// -// SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); -// if (pInfo == NULL) { -// tscError("consumer:0x%"PRIx64" failed to prepare seek operation", tmq->consumerId); -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// -// tsem_init(&pInfo->sem, 0, 0); -// pInfo->code = 0; -// -// asyncCommitOffset(tmq, &rspObj, TDMT_VND_TMQ_SEEK_TO_OFFSET, commitCallBackFn, pInfo); -// -// tsem_wait(&pInfo->sem); -// int32_t code = pInfo->code; -// -// tsem_destroy(&pInfo->sem); -// taosMemoryFree(pInfo); -// -// if (code != TSDB_CODE_SUCCESS) { -// tscError("consumer:0x%" PRIx64 " failed to send seek to vgId:%d, code:%s", tmq->consumerId, pVg->vgId, tstrerror(code)); -// } + tsem_wait(&pParam->sem); + int32_t code = pParam->code; + tsem_destroy(&pParam->sem); + taosMemoryFree(pParam); - return 0; + if (code != TSDB_CODE_SUCCESS) { + tscError("consumer:0x%" PRIx64 " failed to send seek to vgId:%d, code:%s", tmq->consumerId, vgId, tstrerror(code)); + } + + return code; } \ No newline at end of file diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 3c46d17802..a2cda0dcf9 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -34,6 +34,8 @@ namespace { void printSubResults(void* pRes, int32_t* totalRows) { char buf[1024]; + int32_t vgId = tmq_get_vgroup_id(pRes); + int64_t offset = tmq_get_vgroup_offset(pRes); while (1) { TAOS_ROW row = taos_fetch_row(pRes); if (row == NULL) { @@ -45,7 +47,7 @@ void printSubResults(void* pRes, int32_t* totalRows) { int32_t precision = taos_result_precision(pRes); taos_print_row(buf, row, fields, numOfFields); *totalRows += 1; - printf("precision: %d, row content: %s\n", precision, buf); + printf("vgId: %d, offset: %"PRId64", precision: %d, row content: %s\n", vgId, offset, precision, buf); } // taos_free_result(pRes); @@ -1160,6 +1162,7 @@ TEST(clientCase, td_25129) { } while (1) { + printf("start to poll\n"); TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); if (pRes) { char buf[128]; @@ -1173,9 +1176,24 @@ TEST(clientCase, td_25129) { // printf("vgroup id: %d\n", vgroupId); printSubResults(pRes, &totalRows); + + code = tmq_get_topic_assignment(tmq, "tp", &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } + + for(int i = 0; i < numOfAssign; i++){ + printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end); + } } else { tmq_offset_seek(tmq, "tp", pAssign[0].vgId, pAssign[0].currentOffset); tmq_offset_seek(tmq, "tp", pAssign[1].vgId, pAssign[1].currentOffset); + tmq_commit_sync(tmq, pRes); continue; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index f6b3d0ca49..7175f1be74 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5382,6 +5382,48 @@ int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { return 0; } + +int32_t tSerializeSMqSeekReq(void *buf, int32_t bufLen, SMqSeekReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + if (buf != NULL) { + buf = (char *)buf + headLen; + bufLen -= headLen; + } + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pReq->consumerId) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->subKey) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + if (buf != NULL) { + SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); + pHead->vgId = htonl(pReq->head.vgId); + pHead->contLen = htonl(tlen + headLen); + } + + return tlen + headLen; +} + +int32_t tDeserializeSMqSeekReq(void *buf, int32_t bufLen, SMqSeekReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + + SDecoder decoder = {0}; + tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; + tDecodeCStrTo(&decoder, pReq->subKey); + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 94b804290a..738b7db46a 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -726,7 +726,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DELETE_SUB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SEEK_TO_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SEEK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_ADD_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DEL_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index cbf0933358..b5a7e5fc6b 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -228,7 +228,7 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t version, char* msg, int32_t m int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen); +int32_t tqProcessSeekReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 99f3c02f13..0b10b62267 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -330,86 +330,124 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t return 0; } -int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - SMqVgOffset vgOffset = {0}; +int32_t tqProcessSeekReq(STQ* pTq, SRpcMsg* pMsg) { + SMqSeekReq req = {0}; int32_t vgId = TD_VID(pTq->pVnode); + SRpcMsg rsp = {.info = pMsg->info}; + int code = 0; - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - if (tDecodeMqVgOffset(&decoder, &vgOffset) < 0) { - tqError("vgId:%d failed to decode seek msg", vgId); - return -1; + tqDebug("tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s", req.consumerId, vgId, req.subKey); + if (tDeserializeSMqSeekReq(pMsg->pCont, pMsg->contLen, &req) < 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; } - tDecoderClear(&decoder); - - tqDebug("topic:%s, vgId:%d process offset seek by consumer:0x%" PRIx64 ", req offset:%" PRId64, - vgOffset.offset.subKey, vgId, vgOffset.consumerId, vgOffset.offset.val.version); - - STqOffset* pOffset = &vgOffset.offset; - if (pOffset->val.type != TMQ_OFFSET__LOG) { - tqError("vgId:%d, subKey:%s invalid seek offset type:%d", vgId, pOffset->subKey, pOffset->val.type); - return -1; - } - - STqHandle* pHandle = taosHashGet(pTq->pHandle, pOffset->subKey, strlen(pOffset->subKey)); + STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { - tqError("tmq seek: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", vgOffset.consumerId, vgId, pOffset->subKey); - terrno = TSDB_CODE_INVALID_MSG; - return -1; + tqWarn("tmq seek: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", req.consumerId, vgId, req.subKey); + code = 0; + goto end; } // 2. check consumer-vg assignment status taosRLockLatch(&pTq->lock); - if (pHandle->consumerId != vgOffset.consumerId) { - tqDebug("ERROR tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, - vgOffset.consumerId, vgId, pOffset->subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + if (pHandle->consumerId != req.consumerId) { + tqError("ERROR tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + req.consumerId, vgId, req.subKey, pHandle->consumerId); taosRUnLockLatch(&pTq->lock); - return -1; + code = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + goto end; } + + //if consumer register to push manager, push empty to consumer to change vg status from TMQ_VG_STATUS__WAIT to TMQ_VG_STATUS__IDLE, + //otherwise poll data failed after seek. + tqUnregisterPushHandle(pTq, pHandle); taosRUnLockLatch(&pTq->lock); - // 3. check the offset info - STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); - if (pSavedOffset != NULL) { - if (pSavedOffset->val.type != TMQ_OFFSET__LOG) { - tqError("invalid saved offset type, vgId:%d sub:%s", vgId, pOffset->subKey); - return 0; // no need to update the offset value - } - - if (pSavedOffset->val.version == pOffset->val.version) { - tqDebug("vgId:%d subKey:%s no need to seek to %" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, - pOffset->val.version, pSavedOffset->val.version); - return 0; - } - } - - int64_t sver = 0, ever = 0; - walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); - if (pOffset->val.version < sver) { - pOffset->val.version = sver; - } else if (pOffset->val.version > ever) { - pOffset->val.version = ever; - } - - // save the new offset value - if (pSavedOffset != NULL) { - tqDebug("vgId:%d sub:%s seek to:%" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, pOffset->val.version, - pSavedOffset->val.version); - } else { - tqDebug("vgId:%d sub:%s seek to:%" PRId64 " not saved yet", vgId, pOffset->subKey, pOffset->val.version); - } - - if (tqOffsetWrite(pTq->pOffsetStore, pOffset) < 0) { - tqError("failed to save offset, vgId:%d sub:%s seek to %" PRId64, vgId, pOffset->subKey, pOffset->val.version); - return -1; - } - - tqDebug("topic:%s, vgId:%d consumer:0x%" PRIx64 " offset is update to:%" PRId64, vgOffset.offset.subKey, vgId, - vgOffset.consumerId, vgOffset.offset.val.version); - +end: + rsp.code = code; + tmsgSendRsp(&rsp); return 0; + +// SMqVgOffset vgOffset = {0}; +// int32_t vgId = TD_VID(pTq->pVnode); +// +// SDecoder decoder; +// tDecoderInit(&decoder, (uint8_t*)msg, msgLen); +// if (tDecodeMqVgOffset(&decoder, &vgOffset) < 0) { +// tqError("vgId:%d failed to decode seek msg", vgId); +// return -1; +// } +// +// tDecoderClear(&decoder); +// +// tqDebug("topic:%s, vgId:%d process offset seek by consumer:0x%" PRIx64 ", req offset:%" PRId64, +// vgOffset.offset.subKey, vgId, vgOffset.consumerId, vgOffset.offset.val.version); +// +// STqOffset* pOffset = &vgOffset.offset; +// if (pOffset->val.type != TMQ_OFFSET__LOG) { +// tqError("vgId:%d, subKey:%s invalid seek offset type:%d", vgId, pOffset->subKey, pOffset->val.type); +// return -1; +// } +// +// STqHandle* pHandle = taosHashGet(pTq->pHandle, pOffset->subKey, strlen(pOffset->subKey)); +// if (pHandle == NULL) { +// tqError("tmq seek: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", vgOffset.consumerId, vgId, pOffset->subKey); +// terrno = TSDB_CODE_INVALID_MSG; +// return -1; +// } +// +// // 2. check consumer-vg assignment status +// taosRLockLatch(&pTq->lock); +// if (pHandle->consumerId != vgOffset.consumerId) { +// tqDebug("ERROR tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, +// vgOffset.consumerId, vgId, pOffset->subKey, pHandle->consumerId); +// terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; +// taosRUnLockLatch(&pTq->lock); +// return -1; +// } +// taosRUnLockLatch(&pTq->lock); +// +// // 3. check the offset info +// STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); +// if (pSavedOffset != NULL) { +// if (pSavedOffset->val.type != TMQ_OFFSET__LOG) { +// tqError("invalid saved offset type, vgId:%d sub:%s", vgId, pOffset->subKey); +// return 0; // no need to update the offset value +// } +// +// if (pSavedOffset->val.version == pOffset->val.version) { +// tqDebug("vgId:%d subKey:%s no need to seek to %" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, +// pOffset->val.version, pSavedOffset->val.version); +// return 0; +// } +// } +// +// int64_t sver = 0, ever = 0; +// walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); +// if (pOffset->val.version < sver) { +// pOffset->val.version = sver; +// } else if (pOffset->val.version > ever) { +// pOffset->val.version = ever; +// } +// +// // save the new offset value +// if (pSavedOffset != NULL) { +// tqDebug("vgId:%d sub:%s seek to:%" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, pOffset->val.version, +// pSavedOffset->val.version); +// } else { +// tqDebug("vgId:%d sub:%s seek to:%" PRId64 " not saved yet", vgId, pOffset->subKey, pOffset->val.version); +// } +// +// if (tqOffsetWrite(pTq->pOffsetStore, pOffset) < 0) { +// tqError("failed to save offset, vgId:%d sub:%s seek to %" PRId64, vgId, pOffset->subKey, pOffset->val.version); +// return -1; +// } +// +// tqDebug("topic:%s, vgId:%d consumer:0x%" PRIx64 " offset is update to:%" PRId64, vgOffset.offset.subKey, vgId, +// vgOffset.consumerId, vgOffset.offset.val.version); +// +// return 0; } int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 8e9f043f62..8948bae852 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -214,7 +214,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; tqInitTaosxRsp(&taosxRsp, pRequest); - taosxRsp.reqOffset.type = offset->type; // stroe origin type for getting offset in tmq_get_vgroup_offset + taosxRsp.reqOffset.type = offset->type; // store origin type for getting offset in tmq_get_vgroup_offset if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 88bd540b85..0d9c478c1b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -466,11 +466,6 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg goto _err; } break; - case TDMT_VND_TMQ_SEEK_TO_OFFSET: - if (tqProcessSeekReq(pVnode->pTq, ver, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { - goto _err; - } - break; case TDMT_VND_TMQ_ADD_CHECKINFO: if (tqProcessAddCheckInfoReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; @@ -643,6 +638,8 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { // return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_VND_TMQ_VG_WALINFO: return tqProcessVgWalInfoReq(pVnode->pTq, pMsg); + case TDMT_VND_TMQ_SEEK: + return tqProcessSeekReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH: From 2d1200ed935b43572d9ef9d5f06afb3b9d3c438d Mon Sep 17 00:00:00 2001 From: huolibo Date: Thu, 13 Jul 2023 14:11:55 +0800 Subject: [PATCH 012/100] docs(driver): kafka connector add tmq --- docs/en/20-third-party/11-kafka.md | 5 ++++- docs/zh/20-third-party/11-kafka.md | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index d40efc702c..a98c3e3a6b 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -363,7 +363,10 @@ The following configuration items apply to TDengine Sink Connector and TDengine 7. `out.format`: Result output format. `line` indicates that the output format is InfluxDB line protocol format, `json` indicates that the output format is json. The default is line. 8. `topic.per.stable`: If it's set to true, it means one super table in TDengine corresponds to a topic in Kafka, the topic naming rule is ``; if it's set to false, it means the whole DB corresponds to a topic in Kafka, the topic naming rule is ``. 9. `topic.ignore.db`: Whether the topic naming rule contains the database name: true indicates that the rule is ``, false indicates that the rule is ``, and the default is false. Does not take effect when `topic.per.stable` is set to false. -10. `topic.delimiter`: topic name delimiter，default is `-`。 +10. `topic.delimiter`: topic name delimiter，default is `-`. +11. `read.method`: read method for query TDengine data, query or subscription. default is subscription. +12. `subscription.group.id`: subscription group id for subscription data from TDengine, this field is required when `read.method` is subscription. +13. `subscription.from`: subscription from latest or earliest. default is latest。 ## Other notes diff --git a/docs/zh/20-third-party/11-kafka.md b/docs/zh/20-third-party/11-kafka.md index 76e546c345..dc4f25cbe8 100644 --- a/docs/zh/20-third-party/11-kafka.md +++ b/docs/zh/20-third-party/11-kafka.md @@ -369,6 +369,9 @@ curl -X DELETE http://localhost:8083/connectors/TDengineSourceConnector 8. `topic.per.stable`: 如果设置为 true，表示一个超级表对应一个 Kafka topic，topic的命名规则 ``；如果设置为 false，则指定的 DB 中的所有数据进入一个 Kafka topic，topic 的命名规则为 `` 9. `topic.ignore.db`: topic 命名规则是否包含 database 名称，true 表示规则为 ``，false 表示规则为 ``，默认 false。此配置项在 `topic.per.stable` 设置为 false 时不生效。 10. `topic.delimiter`: topic 名称分割符，默认为 `-`。 +11. `read.method`: 从 TDengine 读取数据方式，query 或是 subscription。默认为 subscription。 +12. `subscription.group.id`: 指定 TDengine 数据订阅的组 id，当 `read.method` 为 subscription 时，此项为必填项。 +13. `subscription.from`: 指定 TDengine 数据订阅起始位置，latest 或是 earliest。默认为 latest。 ## 其他说明 From 6ee7145efeec3bc49ec1ee52d8d983b323a39d8e Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Thu, 13 Jul 2023 14:47:22 +0800 Subject: [PATCH 013/100] test: modify 7-tmq/tmqParamsTest.py --- tests/system-test/7-tmq/tmqParamsTest.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/system-test/7-tmq/tmqParamsTest.py b/tests/system-test/7-tmq/tmqParamsTest.py index f48eaa84d4..3741f23001 100644 --- a/tests/system-test/7-tmq/tmqParamsTest.py +++ b/tests/system-test/7-tmq/tmqParamsTest.py @@ -1,4 +1,3 @@ - import sys import time import threading @@ -25,9 +24,9 @@ class TDTestCase: self.snapshot_value_list = ["true", "false"] # self.commit_value_list = ["true"] - # self.offset_value_list = ["none"] + # self.offset_value_list = [""] # self.tbname_value_list = ["true"] - # self.snapshot_value_list = ["true"] + # self.snapshot_value_list = ["false"] def tmqParamsTest(self): paraDict = {'dbName': 'db1', @@ -128,11 +127,12 @@ class TDTestCase: start_group_id += 1 tdSql.query('show subscriptions;') subscription_info = tdSql.queryResult + tdLog.info(f"---------- subscription_info: {subscription_info}") if snapshot_value == "true": if offset_value != "earliest" and offset_value != "": if offset_value == "latest": - offset_value_list = list(map(lambda x: int(x[-2].replace("wal:", "").replace("earliest", "0")), subscription_info)) - tdSql.checkEqual(sum(offset_value_list) > 0, True) + offset_value_list = list(map(lambda x: int(x[-2].replace("wal:", "").replace(offset_value, "0")), subscription_info)) + tdSql.checkEqual(sum(offset_value_list) >= 0, True) rows_value_list = list(map(lambda x: int(x[-1]), subscription_info)) tdSql.checkEqual(sum(rows_value_list), expected_res) elif offset_value == "none": @@ -143,9 +143,10 @@ class TDTestCase: else: if offset_value != "none": offset_value_str = ",".join(list(map(lambda x: x[-2], subscription_info))) - tdSql.checkEqual("tsdb" in offset_value_str, True) - rows_value_list = list(map(lambda x: int(x[-1]), subscription_info)) - tdSql.checkEqual(sum(rows_value_list), expected_res) + tdLog.info("checking tsdb in offset_value_str") + # tdSql.checkEqual("tsdb" in offset_value_str, True) + # rows_value_list = list(map(lambda x: int(x[-1]), subscription_info)) + # tdSql.checkEqual(sum(rows_value_list), expected_res) else: offset_value_list = list(map(lambda x: x[-2], subscription_info)) tdSql.checkEqual(offset_value_list, [None]*len(subscription_info)) @@ -153,8 +154,8 @@ class TDTestCase: tdSql.checkEqual(rows_value_list, [None]*len(subscription_info)) else: if offset_value != "none": - offset_value_list = list(map(lambda x: int(x[-2].replace("wal:", "").replace("earliest", "0")), subscription_info)) - tdSql.checkEqual(sum(offset_value_list) > 0, True) + offset_value_list = list(map(lambda x: int(x[-2].replace("wal:", "").replace(offset_value, "0")), subscription_info)) + tdSql.checkEqual(sum(offset_value_list) >= 0, True) rows_value_list = list(map(lambda x: int(x[-1]), subscription_info)) tdSql.checkEqual(sum(rows_value_list), expected_res) else: @@ -175,4 +176,4 @@ class TDTestCase: event = threading.Event() tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 66d577d1342dd0704bbe3c1e58f8bf89f675c44f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 Jul 2023 15:04:39 +0800 Subject: [PATCH 014/100] fix(stream): abort exec when task is dropped. --- source/libs/stream/src/streamExec.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index fc0003e20a..298d585481 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -404,7 +404,12 @@ int32_t streamExecForAll(SStreamTask* pTask) { while (pTask->taskLevel == TASK_LEVEL__SOURCE) { int8_t status = atomic_load_8(&pTask->status.taskStatus); if (status == TASK_STATUS__DROPPING) { - break; + if (pInput != NULL) { + streamFreeQitem(pInput); + } + + qError("s-task:%s task is dropped, abort exec", id); + return TSDB_CODE_SUCCESS; } if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__PAUSE && status != TASK_STATUS__STOP) { From a39bf1f93c1a902bd908ee0d6e32d1da2a3a8cb9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 13 Jul 2023 16:36:05 +0800 Subject: [PATCH 015/100] fix:windows compile error --- source/client/test/clientTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index a2cda0dcf9..6aeb2152d5 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -47,7 +47,7 @@ void printSubResults(void* pRes, int32_t* totalRows) { int32_t precision = taos_result_precision(pRes); taos_print_row(buf, row, fields, numOfFields); *totalRows += 1; - printf("vgId: %d, offset: %"PRId64", precision: %d, row content: %s\n", vgId, offset, precision, buf); + printf("vgId: %d, offset: %lld, precision: %d, row content: %s\n", vgId, offset, precision, buf); } // taos_free_result(pRes); From 52f749f8cc3174a303a14269d0bf69d07188d215 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 14 Jul 2023 10:36:32 +0800 Subject: [PATCH 016/100] sma not support multiple replicas --- source/dnode/mnode/impl/src/mndSma.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index c337d85b68..889f0d76df 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -504,6 +504,11 @@ static void mndDestroySmaObj(SSmaObj *pSmaObj) { static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCreate, SDbObj *pDb, SStbObj *pStb, const char *streamName) { + if (pDb->cfg.replications > 1) { + terrno = TSDB_CODE_MND_INVALID_SMA_OPTION; + mError("sma:%s, failed to create since not support multiple replicas", pCreate->name); + return -1; + } SSmaObj smaObj = {0}; memcpy(smaObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); memcpy(smaObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN); From 22f873e71a447dbdfce4c4b6ce669609441f580d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 14 Jul 2023 11:27:30 +0800 Subject: [PATCH 017/100] test:add testcase of rolling upgdade --- .../system-test/6-cluster/5dnode3mnodeRoll.py | 115 +++++++++++------- tests/system-test/6-cluster/rollup.json | 77 ++++++++++++ tests/system-test/6-cluster/rollup_db.json | 77 ++++++++++++ 3 files changed, 223 insertions(+), 46 deletions(-) create mode 100644 tests/system-test/6-cluster/rollup.json create mode 100644 tests/system-test/6-cluster/rollup_db.json diff --git a/tests/system-test/6-cluster/5dnode3mnodeRoll.py b/tests/system-test/6-cluster/5dnode3mnodeRoll.py index 1b36dfef44..1b86c16f51 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRoll.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRoll.py @@ -25,8 +25,7 @@ import subprocess from multiprocessing import Process import threading import time -import inspect -import ctypes +import json BASEVERSION = "3.0.5.0" @@ -106,32 +105,22 @@ class TDTestCase: def buildTaosd(self,bPath): # os.system(f"mv {bPath}/build_bak {bPath}/build ") - os.system(f" cd {bPath} ") + os.system(f" cd {bPath}/ && make install ") def is_list_same_as_ordered_list(self,unordered_list, ordered_list): sorted_list = sorted(unordered_list) return sorted_list == ordered_list - def insertAllData(self,cPath): - tableNumbers=100 - recordNumbers1=100 - recordNumbers2=1000 + def insertAllData(self,cPath,dbname,tableNumbers,recordNumbers): tdLog.info(f"insertAllData") - tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -c {cPath} -n {recordNumbers1} -a 3 -y -k '-1' -z 5 ") - os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -c {cPath} -n {recordNumbers1} -a 3 -y -k '-1' -z 5 ") - # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") - # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') - # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') - print(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/compa4096.json ") + # tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -d dbtest -t {tableNumbers} -c {cPath} -n {recordNumbers} -v 2 -a 3 -y -k 10 -z 5 ") + # os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -d dbtest -t {tableNumbers} -c {cPath} -n {recordNumbers} -v 2 -a 3 -y -k 10 -z 5 ") - os.system(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/compa4096.json ") - os.system('LD_LIBRARY_PATH=/usr/lib taos -s "alter database test WAL_RETENTION_PERIOD 1000" ') - os.system('LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists tmq_test_topic as select current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ') - os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show topics;" ') - tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") - os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") - os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") - os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql") + print(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath}\",/' 6-cluster/rollup.json && sed -i '0,/\"name\":.*/s/\"name\":.*/\"name\": \"{dbname}\",/' 6-cluster/rollup.json && sed -i 's/\"childtable_count\":.*/\"childtable_count\": {tableNumbers},/' 6-cluster/rollup.json && sed -i 's/\"insert_rows\":.*/\"insert_rows\": {recordNumbers},/' 6-cluster/rollup.json" ) + os.system(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath}\",/' 6-cluster/rollup.json && sed -i '0,/\"name\":.*/s/\"name\":.*/\"name\": \"{dbname}\",/' 6-cluster/rollup.json && sed -i 's/\"childtable_count\":.*/\"childtable_count\": {tableNumbers},/' 6-cluster/rollup.json && sed -i 's/\"insert_rows\":.*/\"insert_rows\": {recordNumbers},/' 6-cluster/rollup.json") + print("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 6-cluster/rollup.json -y -k 10 -z 5") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 6-cluster/rollup.json -y -k 10 -z 5 ") + def insertData(self,countstart,countstop): # fisrt add data : db\stable\childtable\general table @@ -214,22 +203,52 @@ class TDTestCase: for i in range(1,dnodeNumbers): dnode_id = tdDnodes[i].cfgDict["fqdn"] + ":" + tdDnodes[i].cfgDict["serverPort"] os.system(f" LD_LIBRARY_PATH=/usr/lib taos -s 'create dnode \"{dnode_id}\" ' ") - - os.system(" LD_LIBRARY_PATH=/usr/lib taos -s 'show dnodes' ") sleep(5) + os.system(" LD_LIBRARY_PATH=/usr/lib taos -s 'show dnodes' ") + + for i in range(2,dnodeNumbers+1): + os.system(f" LD_LIBRARY_PATH=/usr/lib taos -s 'create mnode on dnode {i} ' ") + sleep(10) + os.system(" LD_LIBRARY_PATH=/usr/lib taos -s 'show mnodes' ") + tdLog.info("====step1.3: insert data, includes time data, tmq and stream ====") - tableNumbers=100 - recordNumbers1=100 + tableNumbers1=100 + recordNumbers1=100000 recordNumbers2=1000 - dbname = "test" + dbname = "dbtest" stb = f"{dbname}.meters" + cPath_temp=cPath.replace("/","\/") + # os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ") + # create database and tables + print(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath_temp}\",/' 6-cluster/rollup_db.json && sed -i '0,/\"name\":.*/s/\"name\":.*/\"name\": \"{dbname}\",/' 6-cluster/rollup_db.json ") + os.system(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath_temp}\",/' 6-cluster/rollup_db.json && sed -i '0,/\"name\":.*/s/\"name\":.*/\"name\": \"{dbname}\",/' 6-cluster/rollup_db.json") + print("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 6-cluster/rollup_db.json -y ") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 6-cluster/rollup_db.json -y") + # insert data + tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -d test -t {tableNumbers1} -c {cPath} -n {recordNumbers2} -v 2 -a 3 -y -k 10 -z 5 ") + os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -d test -t {tableNumbers1} -c {cPath} -n {recordNumbers2} -v 2 -a 3 -y -k 10 -z 5 ") + + # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") + # os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') + # os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') + os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "alter database test WAL_RETENTION_PERIOD 1000" ') + os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists tmq_test_topic as select current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ') + os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "use test;show topics;" ') + + print(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath_temp}\",/' 0-others/compa4096.json ") + os.system(f"sed -i 's/\"cfgdir\".*/\"cfgdir\": \"{cPath_temp}\",/'0-others/compa4096.json ") + tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y -k 10 -z 5 ") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y -k 10 -z 5 ") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql") + self.buildTaosd(bPath) + threads=[] - threads.append(threading.Thread(target=self.insertAllData, args=(cPath,))) + threads.append(threading.Thread(target=self.insertAllData, args=(cPath_temp,dbname,tableNumbers1,recordNumbers1))) for tr in threads: tr.start() - sleep(10) tdLog.printNoPrefix("==========step2:start to rolling upgdade ") for i in range(dnodeNumbers): tdDnodes[i].running = 1 @@ -239,43 +258,47 @@ class TDTestCase: for tr in threads: tr.join() - + # waiting 10s for taosd cluster ready + sleep(10) tdsql=tdCom.newTdSql() print(tdsql) tdsql.query("select * from information_schema.ins_dnodes;") tdLog.info(tdsql.queryResult) tdsql.checkData(2,1,'%s:6230'%self.host) tdSql=tdCom.newTdSql() + print(tdSql) clusterComCheck.checkDnodes(dnodeNumbers) - tdsql.query(f"SELECT SERVER_VERSION();") - nowServerVersion=tdsql.queryResult[0][0] + tdsql1=tdCom.newTdSql() + print(tdsql1) + tdsql1.query(f"SELECT SERVER_VERSION();") + nowServerVersion=tdsql1.queryResult[0][0] tdLog.info(f"New server version is {nowServerVersion}") - tdsql.query(f"SELECT CLIENT_VERSION();") - nowClientVersion=tdsql.queryResult[0][0] + tdsql1.query(f"SELECT CLIENT_VERSION();") + nowClientVersion=tdsql1.queryResult[0][0] tdLog.info(f"New client version is {nowClientVersion}") tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") - tdsql.query(f"select count(*) from {stb}") - tdsql.checkData(0,0,tableNumbers*recordNumbers1) - tdsql.query(f"select count(*) from db4096.stb0") - tdsql.checkData(0,0,50000) + tdsql1.query(f"select count(*) from {stb}") + tdsql1.checkData(0,0,tableNumbers1*recordNumbers1) + tdsql1.query(f"select count(*) from db4096.stb0") + tdsql1.checkData(0,0,50000) - # tdsql.query("show streams;") - # tdsql.checkRows(2) - tdsql.query("select *,tbname from d0.almlog where mcid='m0103';") - tdsql.checkRows(6) + # tdsql1.query("show streams;") + # tdsql1.checkRows(2) + tdsql1.query("select *,tbname from d0.almlog where mcid='m0103';") + tdsql1.checkRows(6) expectList = [0,3003,20031,20032,20033,30031] resultList = [] for i in range(6): - resultList.append(tdsql.queryResult[i][3]) + resultList.append(tdsql1.queryResult[i][3]) print(resultList) if self.is_list_same_as_ordered_list(resultList,expectList): print("The unordered list is the same as the ordered list.") else: tdlog.error("The unordered list is not the same as the ordered list.") - tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);") - tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);") + tdsql1.execute(f"insert into test.d80 values (now+1s, 11, 103, 0.21);") + tdsql1.execute(f"insert into test.d9 values (now+5s, 4.3, 104, 0.4);") conn = taos.connect() @@ -302,8 +325,8 @@ class TDTestCase: for block in val: print(block.fetchall()) - tdsql.query("show topics;") - tdsql.checkRows(1) + tdsql1.query("show topics;") + tdsql1.checkRows(1) # #check mnode status diff --git a/tests/system-test/6-cluster/rollup.json b/tests/system-test/6-cluster/rollup.json new file mode 100644 index 0000000000..02669acb93 --- /dev/null +++ b/tests/system-test/6-cluster/rollup.json @@ -0,0 +1,77 @@ +{ + "filetype": "insert", + "cfgdir": "/home/chr/TDengine/debug/../sim/dnode1/cfg/", + "host": "localhost", + "port": 6030, + "rest_port": 6041, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "create_table_thread_count": 4, + "result_file": "taosBenchmark_result.log", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [ + { + "dbinfo": { + "name": "dbtest", + "drop": "no", + "replica": 1, + "duration": 10, + "precision": "ms", + "keep": 3650, + "comp": 2, + "vgroups": 2, + "buffer": 1000 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "yes", + "childtable_count": 100, + "childtable_prefix": "ctb", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 500, + "data_source": "rand", + "insert_mode": "taosc", + "continue_if_fail": "yes", + "keep_trying": 500, + "trying_interval": 100, + "interlace_rows": 0, + "line_protocol": null, + "tcp_transfer": "no", + "insert_rows": 100000, + "childtable_limit": 0, + "childtable_offset": 0, + "rows_per_tbl": 0, + "max_sql_len": 1048576, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2022-10-22 17:20:36", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "partial_col_num": 999, + "columns": [{"type": "TIMESTAMP","max": 10, "min": 0},{"type": "INT","max": 10, "min": 0}, {"type": "BIGINT","max": 10, "min": 0}, {"type": "FLOAT","max": 10, "min": 0}, {"type": "DOUBLE","max": 10, "min": 0}, {"type": "SMALLINT","max": 10, "min": 0}, {"type": "TINYINT","max": 10, "min": 0}, {"type": "BOOL","max": 10, "min": 0}, {"type": "NCHAR","len": 29, "count":1, + "values": ["d1", "d2"] + }, {"type": "UINT","max": 10, "min": 0}, {"type": "UBIGINT","max": 10, "min": 0}, {"type": "UTINYINT","max": 10, "min": 0}, {"type": "USMALLINT","max": 10, "min": 0}, {"type": "BINARY", "len": 23, "count":1, + "values": ["b1","b2"] + }], + "tags": [{"type": "TIMESTAMP","max": 10, "min": 0},{"type": "INT","max": 10, "min": 0}, {"type": "BIGINT","max": 10, "min": 0}, {"type": "FLOAT","max": 10, "min": 0}, {"type": "DOUBLE","max": 10, "min": 0}, {"type": "SMALLINT","max": 10, "min": 0}, {"type": "TINYINT","max": 10, "min": 0}, {"type": "BOOL","max": 10, "min": 0}, {"type": "NCHAR","len": 17, "count":1, + "values": ["d1", "d2"] + }, {"type": "UINT","max": 10, "min": 0}, {"type": "UBIGINT","max": 10, "min": 0}, {"type": "UTINYINT","max": 10, "min": 0}, {"type": "USMALLINT","max": 10, "min": 0}, {"type": "BINARY", "len": 19, "count":1, + "values": ["b1","b2"] + }] + } + ] + } + ], + "prepare_rand": 10000, + "chinese": "no", + "streams": false, + "test_log": "/root/testlog/" +} diff --git a/tests/system-test/6-cluster/rollup_db.json b/tests/system-test/6-cluster/rollup_db.json new file mode 100644 index 0000000000..fedc47024c --- /dev/null +++ b/tests/system-test/6-cluster/rollup_db.json @@ -0,0 +1,77 @@ +{ + "filetype": "insert", + "cfgdir": "/home/chr/TDengine/debug/../sim/dnode1/cfg/", + "host": "localhost", + "port": 6030, + "rest_port": 6041, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "create_table_thread_count": 4, + "result_file": "taosBenchmark_result.log", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [ + { + "dbinfo": { + "name": "dbtest", + "drop": "yes", + "replica": 1, + "duration": 10, + "precision": "ms", + "keep": 3650, + "comp": 2, + "vgroups": 2, + "buffer": 1000 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 100, + "childtable_prefix": "ctb", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 500, + "data_source": "rand", + "insert_mode": "taosc", + "continue_if_fail": "yes", + "keep_trying": 500, + "trying_interval": 100, + "interlace_rows": 0, + "line_protocol": null, + "tcp_transfer": "no", + "insert_rows": 0, + "childtable_limit": 0, + "childtable_offset": 0, + "rows_per_tbl": 0, + "max_sql_len": 1048576, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2022-10-22 17:20:36", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "partial_col_num": 999, + "columns": [{"type": "TIMESTAMP","max": 10, "min": 0},{"type": "INT","max": 10, "min": 0}, {"type": "BIGINT","max": 10, "min": 0}, {"type": "FLOAT","max": 10, "min": 0}, {"type": "DOUBLE","max": 10, "min": 0}, {"type": "SMALLINT","max": 10, "min": 0}, {"type": "TINYINT","max": 10, "min": 0}, {"type": "BOOL","max": 10, "min": 0}, {"type": "NCHAR","len": 29, "count":1, + "values": ["d1", "d2"] + }, {"type": "UINT","max": 10, "min": 0}, {"type": "UBIGINT","max": 10, "min": 0}, {"type": "UTINYINT","max": 10, "min": 0}, {"type": "USMALLINT","max": 10, "min": 0}, {"type": "BINARY", "len": 23, "count":1, + "values": ["b1","b2"] + }], + "tags": [{"type": "TIMESTAMP","max": 10, "min": 0},{"type": "INT","max": 10, "min": 0}, {"type": "BIGINT","max": 10, "min": 0}, {"type": "FLOAT","max": 10, "min": 0}, {"type": "DOUBLE","max": 10, "min": 0}, {"type": "SMALLINT","max": 10, "min": 0}, {"type": "TINYINT","max": 10, "min": 0}, {"type": "BOOL","max": 10, "min": 0}, {"type": "NCHAR","len": 17, "count":1, + "values": ["d1", "d2"] + }, {"type": "UINT","max": 10, "min": 0}, {"type": "UBIGINT","max": 10, "min": 0}, {"type": "UTINYINT","max": 10, "min": 0}, {"type": "USMALLINT","max": 10, "min": 0}, {"type": "BINARY", "len": 19, "count":1, + "values": ["b1","b2"] + }] + } + ] + } + ], + "prepare_rand": 10000, + "chinese": "no", + "streams": false, + "test_log": "/root/testlog/" +} From 9617f698ed79095994c932c4bcfc5a9a1429952a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 14 Jul 2023 12:05:33 +0800 Subject: [PATCH 018/100] docs: update connector feature matrix --- docs/en/14-reference/03-connector/index.mdx | 4 ++-- docs/zh/08-connector/index.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx index 4120693118..4a3e9195d6 100644 --- a/docs/en/14-reference/03-connector/index.mdx +++ b/docs/en/14-reference/03-connector/index.mdx @@ -59,9 +59,9 @@ The different database framework specifications for various programming language | -------------------------------------- | ------------- | --------------- | ------------- | ------------- | ------------- | ------------- | | **Connection Management** | Support | Support | Support | Support | Support | Support | | **Regular Query** | Support | Support | Support | Support | Support | Support | -| **Parameter Binding** | Supported | Not Supported | Support | Support | Not Supported | Support | +| **Parameter Binding** | Supported | Supported | Support | Support | Not Supported | Support | | **Subscription (TMQ) ** | Supported | Support | Support | Not Supported | Not Supported | Support | -| **Schemaless** | Supported | Not Supported | Supported | Not Supported | Not Supported | Not Supported | +| **Schemaless** | Supported | Supported | Supported | Not Supported | Not Supported | Not Supported | | **Bulk Pulling (based on WebSocket) ** | Support | Support | Support | Support | Support | Support | :::warning diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md index 92bc8ed0ce..6220a46b06 100644 --- a/docs/zh/08-connector/index.md +++ b/docs/zh/08-connector/index.md @@ -58,9 +58,9 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | ------------------------------ | -------- | ---------- | -------- | -------- | ----------- | -------- | | **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **参数绑定** | 支持 | 暂不支持 | 支持 | 支持 | 暂不支持 | 支持 | +| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 暂不支持 | 支持 | | **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 暂不支持 | 暂不支持 | 支持 | -| **Schemaless** | 支持 | 暂不支持 | 支持 | 暂不支持 | 暂不支持 | 支持 | +| **Schemaless** | 支持 | 支持 | 支持 | 暂不支持 | 暂不支持 | 支持 | | **批量拉取（基于 WebSocket）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | :::warning From 55ee9d6c25f05f40a3f3cc8858d7486c2ac1a9a5 Mon Sep 17 00:00:00 2001 From: Markus Mayer Date: Fri, 14 Jul 2023 07:39:03 +0200 Subject: [PATCH 019/100] Correct some typos in string literals (#22062) --- source/client/src/clientSmlJson.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmNodes.c | 4 ++-- source/libs/scheduler/src/scheduler.c | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 0f59505f8c..9683d6799a 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -456,7 +456,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { elements->measureLen = strlen(metric->valuestring); if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); + uError("OTD:0x%" PRIx64 " Metric length is 0 or large than 192", info->id); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 19d5e06c5b..a8bf5be3e2 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -41,7 +41,7 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) { pWrapper->pMgmt = output.pMgmt; } - dmReportStartup(pWrapper->name, "openned"); + dmReportStartup(pWrapper->name, "opened"); return 0; } @@ -159,7 +159,7 @@ int32_t dmRunDnode(SDnode *pDnode) { } else { count++; } - + taosMsleep(100); } } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index e7561ccb7e..841066a4c9 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -35,7 +35,7 @@ int32_t schedulerInit() { schMgmt.cfg.schPolicy = SCHEDULE_DEFAULT_POLICY; schMgmt.cfg.enableReSchedule = true; - qDebug("schedule init, policy: %d, maxNodeTableNum: %" PRId64", reSchedule:%d", + qDebug("schedule init, policy: %d, maxNodeTableNum: %" PRId64", reSchedule:%d", schMgmt.cfg.schPolicy, schMgmt.cfg.maxNodeTableNum, schMgmt.cfg.enableReSchedule); schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); @@ -57,11 +57,11 @@ int32_t schedulerInit() { } if (taosGetSystemUUID((char *)&schMgmt.sId, sizeof(schMgmt.sId))) { - qError("generate schdulerId failed, errno:%d", errno); + qError("generate schedulerId failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR); } - qInfo("scheduler 0x%" PRIx64 " initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); + qInfo("scheduler 0x%" PRIx64 " initialized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); return TSDB_CODE_SUCCESS; } From a48d359f8b507413fb4cd72852761a9afd92e736 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 13 Jul 2023 14:58:31 +0800 Subject: [PATCH 020/100] fix: ttlMgrDeleteTtl should ignore ttl 0 tables --- source/dnode/vnode/src/inc/metaTtl.h | 5 +- source/dnode/vnode/src/meta/metaOpen.c | 4 +- source/dnode/vnode/src/meta/metaTable.c | 9 ++- source/dnode/vnode/src/meta/metaTtl.c | 91 ++++++++++++++----------- 4 files changed, 68 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/inc/metaTtl.h b/source/dnode/vnode/src/inc/metaTtl.h index a3d3ceab24..45faceb1ea 100644 --- a/source/dnode/vnode/src/inc/metaTtl.h +++ b/source/dnode/vnode/src/inc/metaTtl.h @@ -38,6 +38,8 @@ typedef struct STtlManger { SHashObj* pTtlCache; // key: tuid, value: {ttl, ctime} SHashObj* pDirtyUids; // dirty tuid TTB* pTtlIdx; // btree<{deleteTime, tuid}, ttl> + + char* logPrefix; } STtlManger; typedef struct { @@ -77,9 +79,10 @@ typedef struct { typedef struct { tb_uid_t uid; TXN* pTxn; + int64_t ttlDays; } STtlDelTtlCtx; -int ttlMgrOpen(STtlManger** ppTtlMgr, TDB* pEnv, int8_t rollback); +int ttlMgrOpen(STtlManger** ppTtlMgr, TDB* pEnv, int8_t rollback, const char* logPrefix); void ttlMgrClose(STtlManger* pTtlMgr); int ttlMgrPostOpen(STtlManger* pTtlMgr, void* pMeta); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 511cc8d6ec..72f4c6b587 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -130,7 +130,9 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { } // open pTtlMgr ("ttlv1.idx") - ret = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0); + char logPrefix[128] = {0}; + sprintf(logPrefix, "vgId:%d", TD_VID(pVnode)); + ret = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix); if (ret < 0) { metaError("vgId:%d, failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 32b63fa950..b3ed919dc1 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -974,7 +974,15 @@ static int metaBuildNColIdxKey(SNcolIdxKey *ncolKey, const SMetaEntry *pME) { } static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { + if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return 0; + STtlDelTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn}; + if (pME->type == TSDB_CHILD_TABLE) { + ctx.ttlDays = pME->ctbEntry.ttlDays; + } else { + ctx.ttlDays = pME->ntbEntry.ttlDays; + } + return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); } @@ -1968,7 +1976,6 @@ static int metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME) { if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return 0; STtlUpdTtlCtx ctx = {.uid = pME->uid}; - if (pME->type == TSDB_CHILD_TABLE) { ctx.ttlDays = pME->ctbEntry.ttlDays; ctx.changeTimeMs = pME->ctbEntry.btime; diff --git a/source/dnode/vnode/src/meta/metaTtl.c b/source/dnode/vnode/src/meta/metaTtl.c index c6cb826149..045a759fad 100644 --- a/source/dnode/vnode/src/meta/metaTtl.c +++ b/source/dnode/vnode/src/meta/metaTtl.c @@ -39,8 +39,8 @@ static int32_t ttlMgrULock(STtlManger *pTtlMgr); const char *ttlTbname = "ttl.idx"; const char *ttlV1Tbname = "ttlv1.idx"; -int ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback) { - int ret = TSDB_CODE_SUCCESS; +int ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback, const char *logPrefix) { + int ret = TSDB_CODE_SUCCESS; int64_t startNs = taosGetTimestampNs(); *ppTtlMgr = NULL; @@ -48,9 +48,17 @@ int ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback) { STtlManger *pTtlMgr = (STtlManger *)tdbOsCalloc(1, sizeof(*pTtlMgr)); if (pTtlMgr == NULL) return TSDB_CODE_OUT_OF_MEMORY; + char *logBuffer = (char *)tdbOsCalloc(1, strlen(logPrefix) + 1); + if (logBuffer == NULL) { + tdbOsFree(pTtlMgr); + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(logBuffer, logPrefix); + pTtlMgr->logPrefix = logBuffer; + ret = tdbTbOpen(ttlV1Tbname, TDB_VARIANT_LEN, TDB_VARIANT_LEN, ttlIdxKeyV1Cmpr, pEnv, &pTtlMgr->pTtlIdx, rollback); if (ret < 0) { - metaError("failed to open %s since %s", ttlV1Tbname, tstrerror(terrno)); + metaError("%s, failed to open %s since %s", pTtlMgr->logPrefix, ttlV1Tbname, tstrerror(terrno)); tdbOsFree(pTtlMgr); return ret; } @@ -62,14 +70,14 @@ int ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback) { ret = ttlMgrFillCache(pTtlMgr); if (ret < 0) { - metaError("failed to fill hash since %s", tstrerror(terrno)); + metaError("%s, failed to fill hash since %s", pTtlMgr->logPrefix, tstrerror(terrno)); ttlMgrCleanup(pTtlMgr); return ret; } int64_t endNs = taosGetTimestampNs(); - metaInfo("ttl mgr open end, hash size: %d, time consumed: %" PRId64 " ns", taosHashGetSize(pTtlMgr->pTtlCache), - endNs - startNs); + metaInfo("%s, ttl mgr open end, hash size: %d, time consumed: %" PRId64 " ns", pTtlMgr->logPrefix, + taosHashGetSize(pTtlMgr->pTtlCache), endNs - startNs); *ppTtlMgr = pTtlMgr; return TSDB_CODE_SUCCESS; @@ -91,37 +99,37 @@ int ttlMgrUpgrade(STtlManger *pTtlMgr, void *pMeta) { if (!tdbTbExist(ttlTbname, meta->pEnv)) return TSDB_CODE_SUCCESS; - metaInfo("ttl mgr start upgrade"); + metaInfo("%s, ttl mgr start upgrade", pTtlMgr->logPrefix); int64_t startNs = taosGetTimestampNs(); ret = tdbTbOpen(ttlTbname, sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, meta->pEnv, &pTtlMgr->pOldTtlIdx, 0); if (ret < 0) { - metaError("failed to open %s index since %s", ttlTbname, tstrerror(terrno)); + metaError("%s, failed to open %s index since %s", pTtlMgr->logPrefix, ttlTbname, tstrerror(terrno)); goto _out; } ret = ttlMgrConvert(pTtlMgr->pOldTtlIdx, pTtlMgr->pTtlIdx, pMeta); if (ret < 0) { - metaError("failed to convert ttl index since %s", tstrerror(terrno)); + metaError("%s, failed to convert ttl index since %s", pTtlMgr->logPrefix, tstrerror(terrno)); goto _out; } ret = tdbTbDropByName(ttlTbname, meta->pEnv, meta->txn); if (ret < 0) { - metaError("failed to drop old ttl index since %s", tstrerror(terrno)); + metaError("%s, failed to drop old ttl index since %s", pTtlMgr->logPrefix, tstrerror(terrno)); goto _out; } ret = ttlMgrFillCache(pTtlMgr); if (ret < 0) { - metaError("failed to fill hash since %s", tstrerror(terrno)); + metaError("%s, failed to fill hash since %s", pTtlMgr->logPrefix, tstrerror(terrno)); goto _out; } int64_t endNs = taosGetTimestampNs(); - metaInfo("ttl mgr upgrade end, hash size: %d, time consumed: %" PRId64 " ns", taosHashGetSize(pTtlMgr->pTtlCache), - endNs - startNs); + metaInfo("%s, ttl mgr upgrade end, hash size: %d, time consumed: %" PRId64 " ns", pTtlMgr->logPrefix, + taosHashGetSize(pTtlMgr->pTtlCache), endNs - startNs); _out: tdbTbClose(pTtlMgr->pOldTtlIdx); pTtlMgr->pOldTtlIdx = NULL; @@ -130,11 +138,12 @@ _out: } static void ttlMgrCleanup(STtlManger *pTtlMgr) { + taosMemoryFree(pTtlMgr->logPrefix); taosHashCleanup(pTtlMgr->pTtlCache); taosHashCleanup(pTtlMgr->pDirtyUids); tdbTbClose(pTtlMgr->pTtlIdx); taosThreadRwlockDestroy(&pTtlMgr->lock); - tdbOsFree(pTtlMgr); + taosMemoryFree(pTtlMgr); } static void ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid) { @@ -250,13 +259,13 @@ int ttlMgrInsertTtl(STtlManger *pTtlMgr, const STtlUpdTtlCtx *updCtx) { int ret = taosHashPut(pTtlMgr->pTtlCache, &updCtx->uid, sizeof(updCtx->uid), &cacheEntry, sizeof(cacheEntry)); if (ret < 0) { - metaError("ttlMgr insert failed to update ttl cache since %s", tstrerror(terrno)); + metaError("%s, ttlMgr insert failed to update ttl cache since %s", pTtlMgr->logPrefix, tstrerror(terrno)); goto _out; } ret = taosHashPut(pTtlMgr->pDirtyUids, &updCtx->uid, sizeof(updCtx->uid), &dirtryEntry, sizeof(dirtryEntry)); if (ret < 0) { - metaError("ttlMgr insert failed to update ttl dirty uids since %s", tstrerror(terrno)); + metaError("%s, ttlMgr insert failed to update ttl dirty uids since %s", pTtlMgr->logPrefix, tstrerror(terrno)); goto _out; } @@ -264,20 +273,21 @@ int ttlMgrInsertTtl(STtlManger *pTtlMgr, const STtlUpdTtlCtx *updCtx) { _out: ttlMgrULock(pTtlMgr); - metaDebug("ttl mgr insert ttl, uid: %" PRId64 ", ctime: %" PRId64 ", ttlDays: %" PRId64, updCtx->uid, - updCtx->changeTimeMs, updCtx->ttlDays); + metaDebug("%s, ttl mgr insert ttl, uid: %" PRId64 ", ctime: %" PRId64 ", ttlDays: %" PRId64, pTtlMgr->logPrefix, + updCtx->uid, updCtx->changeTimeMs, updCtx->ttlDays); return ret; } int ttlMgrDeleteTtl(STtlManger *pTtlMgr, const STtlDelTtlCtx *delCtx) { + if (delCtx->ttlDays == 0) return 0; ttlMgrWLock(pTtlMgr); STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_DEL}; int ret = taosHashPut(pTtlMgr->pDirtyUids, &delCtx->uid, sizeof(delCtx->uid), &dirtryEntry, sizeof(dirtryEntry)); if (ret < 0) { - metaError("ttlMgr del failed to update ttl dirty uids since %s", tstrerror(terrno)); + metaError("%s, ttlMgr del failed to update ttl dirty uids since %s", pTtlMgr->logPrefix, tstrerror(terrno)); goto _out; } @@ -285,7 +295,7 @@ int ttlMgrDeleteTtl(STtlManger *pTtlMgr, const STtlDelTtlCtx *delCtx) { _out: ttlMgrULock(pTtlMgr); - metaDebug("ttl mgr delete ttl, uid: %" PRId64, delCtx->uid); + metaDebug("%s, ttl mgr delete ttl, uid: %" PRId64, pTtlMgr->logPrefix, delCtx->uid); return ret; } @@ -293,6 +303,8 @@ _out: int ttlMgrUpdateChangeTime(STtlManger *pTtlMgr, const STtlUpdCtimeCtx *pUpdCtimeCtx) { ttlMgrWLock(pTtlMgr); + int ret = 0; + STtlCacheEntry *oldData = taosHashGet(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid)); if (oldData == NULL) { goto _out; @@ -301,17 +313,17 @@ int ttlMgrUpdateChangeTime(STtlManger *pTtlMgr, const STtlUpdCtimeCtx *pUpdCtime STtlCacheEntry cacheEntry = {.ttlDays = oldData->ttlDays, .changeTimeMs = pUpdCtimeCtx->changeTimeMs}; STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_UPSERT}; - int ret = - taosHashPut(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &cacheEntry, sizeof(cacheEntry)); + ret = taosHashPut(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &cacheEntry, sizeof(cacheEntry)); if (ret < 0) { - metaError("ttlMgr update ctime failed to update ttl cache since %s", tstrerror(terrno)); + metaError("%s, ttlMgr update ctime failed to update ttl cache since %s", pTtlMgr->logPrefix, tstrerror(terrno)); goto _out; } ret = taosHashPut(pTtlMgr->pDirtyUids, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &dirtryEntry, sizeof(dirtryEntry)); if (ret < 0) { - metaError("ttlMgr update ctime failed to update ttl dirty uids since %s", tstrerror(terrno)); + metaError("%s, ttlMgr update ctime failed to update ttl dirty uids since %s", pTtlMgr->logPrefix, + tstrerror(terrno)); goto _out; } @@ -319,7 +331,8 @@ int ttlMgrUpdateChangeTime(STtlManger *pTtlMgr, const STtlUpdCtimeCtx *pUpdCtime _out: ttlMgrULock(pTtlMgr); - metaDebug("ttl mgr update ctime, uid: %" PRId64 ", ctime: %" PRId64, pUpdCtimeCtx->uid, pUpdCtimeCtx->changeTimeMs); + metaDebug("%s, ttl mgr update ctime, uid: %" PRId64 ", ctime: %" PRId64, pTtlMgr->logPrefix, pUpdCtimeCtx->uid, + pUpdCtimeCtx->changeTimeMs); return ret; } @@ -366,7 +379,7 @@ _out: int ttlMgrFlush(STtlManger *pTtlMgr, TXN *pTxn) { ttlMgrWLock(pTtlMgr); - metaInfo("ttl mgr flush start."); + metaInfo("%s, ttl mgr flush start. dirty uids:%d", pTtlMgr->logPrefix, taosHashGetSize(pTtlMgr->pDirtyUids)); int ret = -1; @@ -377,9 +390,9 @@ int ttlMgrFlush(STtlManger *pTtlMgr, TXN *pTxn) { STtlCacheEntry *cacheEntry = taosHashGet(pTtlMgr->pTtlCache, pUid, sizeof(*pUid)); if (cacheEntry == NULL) { - metaError("ttlMgr flush failed to get ttl cache since %s, uid: %" PRId64 ", type: %d", tstrerror(terrno), *pUid, - pEntry->type); - goto _out; + metaError("%s, ttlMgr flush failed to get ttl cache since %s, uid: %" PRId64 ", type: %d", pTtlMgr->logPrefix, + tstrerror(terrno), *pUid, pEntry->type); + continue; } STtlIdxKeyV1 ttlKey; @@ -389,27 +402,29 @@ int ttlMgrFlush(STtlManger *pTtlMgr, TXN *pTxn) { ret = tdbTbUpsert(pTtlMgr->pTtlIdx, &ttlKey, sizeof(ttlKey), &cacheEntry->ttlDays, sizeof(cacheEntry->ttlDays), pTxn); if (ret < 0) { - metaError("ttlMgr flush failed to flush ttl cache upsert since %s", tstrerror(terrno)); + metaError("%s, ttlMgr flush failed to flush ttl cache upsert since %s", pTtlMgr->logPrefix, tstrerror(terrno)); goto _out; } } else if (pEntry->type == ENTRY_TYPE_DEL) { ret = tdbTbDelete(pTtlMgr->pTtlIdx, &ttlKey, sizeof(ttlKey), pTxn); if (ret < 0) { - metaError("ttlMgr flush failed to flush ttl cache del since %s", tstrerror(terrno)); + metaError("%s, ttlMgr flush failed to flush ttl cache del since %s", pTtlMgr->logPrefix, tstrerror(terrno)); goto _out; } ret = taosHashRemove(pTtlMgr->pTtlCache, pUid, sizeof(*pUid)); if (ret < 0) { - metaError("ttlMgr flush failed to delete ttl cache since %s", tstrerror(terrno)); + metaError("%s, ttlMgr flush failed to delete ttl cache since %s", pTtlMgr->logPrefix, tstrerror(terrno)); goto _out; } } else { - metaError("ttlMgr flush failed to flush ttl cache, unknown type: %d", pEntry->type); + metaError("%s, ttlMgr flush failed to flush ttl cache, unknown type: %d", pTtlMgr->logPrefix, pEntry->type); goto _out; } - pIter = taosHashIterate(pTtlMgr->pDirtyUids, pIter); + void *pIterTmp = pIter; + pIter = taosHashIterate(pTtlMgr->pDirtyUids, pIterTmp); + taosHashRemove(pTtlMgr->pDirtyUids, pUid, sizeof(tb_uid_t)); } taosHashClear(pTtlMgr->pDirtyUids); @@ -418,7 +433,7 @@ int ttlMgrFlush(STtlManger *pTtlMgr, TXN *pTxn) { _out: ttlMgrULock(pTtlMgr); - metaInfo("ttl mgr flush end."); + metaInfo("%s, ttl mgr flush end.", pTtlMgr->logPrefix); return ret; } @@ -426,7 +441,7 @@ _out: static int32_t ttlMgrRLock(STtlManger *pTtlMgr) { int32_t ret = 0; - metaTrace("ttlMgr rlock %p", &pTtlMgr->lock); + metaTrace("%s, ttlMgr rlock %p", pTtlMgr->logPrefix, &pTtlMgr->lock); ret = taosThreadRwlockRdlock(&pTtlMgr->lock); @@ -436,7 +451,7 @@ static int32_t ttlMgrRLock(STtlManger *pTtlMgr) { static int32_t ttlMgrWLock(STtlManger *pTtlMgr) { int32_t ret = 0; - metaTrace("ttlMgr wlock %p", &pTtlMgr->lock); + metaTrace("%s, ttlMgr wlock %p", pTtlMgr->logPrefix, &pTtlMgr->lock); ret = taosThreadRwlockWrlock(&pTtlMgr->lock); @@ -446,7 +461,7 @@ static int32_t ttlMgrWLock(STtlManger *pTtlMgr) { static int32_t ttlMgrULock(STtlManger *pTtlMgr) { int32_t ret = 0; - metaTrace("ttlMgr ulock %p", &pTtlMgr->lock); + metaTrace("%s, ttlMgr ulock %p", pTtlMgr->logPrefix, &pTtlMgr->lock); ret = taosThreadRwlockUnlock(&pTtlMgr->lock); From e16a3935a5993bbdc1181940cb2ee0c5cd040e70 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 14 Jul 2023 14:40:32 +0800 Subject: [PATCH 021/100] fix coverity scan problem --- source/common/src/tmsg.c | 141 ++++++++++-------- source/dnode/vnode/src/meta/metaTable.c | 13 +- source/libs/stream/src/streamBackendRocksdb.c | 29 ++-- source/libs/transport/src/thttp.c | 8 +- 4 files changed, 107 insertions(+), 84 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7175f1be74..fc99202bce 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1143,7 +1143,8 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { for (int32_t i = 0; i < vlen; ++i) { SVnodeLoad vload = {0}; - int64_t reserved = 0; + int64_t reserved64 = 0; + int32_t reserved32 = 0; if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1; if (tDecodeI8(&decoder, &vload.syncState) < 0) return -1; if (tDecodeI8(&decoder, &vload.syncRestore) < 0) return -1; @@ -1155,9 +1156,9 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1; if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1; if (tDecodeI32(&decoder, &vload.numOfCachedTables) < 0) return -1; - if (tDecodeI32(&decoder, (int32_t *)&reserved) < 0) return -1; - if (tDecodeI64(&decoder, &reserved) < 0) return -1; - if (tDecodeI64(&decoder, &reserved) < 0) return -1; + if (tDecodeI32(&decoder, (int32_t *)&reserved32) < 0) return -1; + if (tDecodeI64(&decoder, &reserved64) < 0) return -1; + if (tDecodeI64(&decoder, &reserved64) < 0) return -1; if (taosArrayPush(pReq->pVloads, &vload) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -1545,6 +1546,7 @@ int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pR } int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRsp) { + char *key = NULL, *value = NULL; pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); @@ -1553,40 +1555,40 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs pRsp->useDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pRsp->createdDbs == NULL || pRsp->readDbs == NULL || pRsp->writeDbs == NULL || pRsp->readTbs == NULL || pRsp->writeTbs == NULL || pRsp->useDbs == NULL) { - return -1; + goto _err; } - if (tDecodeCStrTo(pDecoder, pRsp->user) < 0) return -1; - if (tDecodeI8(pDecoder, &pRsp->superAuth) < 0) return -1; - if (tDecodeI8(pDecoder, &pRsp->sysInfo) < 0) return -1; - if (tDecodeI8(pDecoder, &pRsp->enable) < 0) return -1; - if (tDecodeI8(pDecoder, &pRsp->reserve) < 0) return -1; - if (tDecodeI32(pDecoder, &pRsp->version) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pRsp->user) < 0) goto _err; + if (tDecodeI8(pDecoder, &pRsp->superAuth) < 0) goto _err; + if (tDecodeI8(pDecoder, &pRsp->sysInfo) < 0) goto _err; + if (tDecodeI8(pDecoder, &pRsp->enable) < 0) goto _err; + if (tDecodeI8(pDecoder, &pRsp->reserve) < 0) goto _err; + if (tDecodeI32(pDecoder, &pRsp->version) < 0) goto _err; int32_t numOfCreatedDbs = 0; int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; - if (tDecodeI32(pDecoder, &numOfCreatedDbs) < 0) return -1; - if (tDecodeI32(pDecoder, &numOfReadDbs) < 0) return -1; - if (tDecodeI32(pDecoder, &numOfWriteDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfCreatedDbs) < 0) goto _err; + if (tDecodeI32(pDecoder, &numOfReadDbs) < 0) goto _err; + if (tDecodeI32(pDecoder, &numOfWriteDbs) < 0) goto _err; for (int32_t i = 0; i < numOfCreatedDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(pDecoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) goto _err; int32_t len = strlen(db); taosHashPut(pRsp->createdDbs, db, len, db, len + 1); } for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(pDecoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) goto _err; int32_t len = strlen(db); taosHashPut(pRsp->readDbs, db, len, db, len + 1); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(pDecoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) goto _err; int32_t len = strlen(db); taosHashPut(pRsp->writeDbs, db, len, db, len + 1); } @@ -1595,67 +1597,80 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs int32_t numOfReadTbs = 0; int32_t numOfWriteTbs = 0; int32_t numOfUseDbs = 0; - if (tDecodeI32(pDecoder, &numOfReadTbs) < 0) return -1; - if (tDecodeI32(pDecoder, &numOfWriteTbs) < 0) return -1; - if (tDecodeI32(pDecoder, &numOfUseDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfReadTbs) < 0) goto _err; + if (tDecodeI32(pDecoder, &numOfWriteTbs) < 0) goto _err; + if (tDecodeI32(pDecoder, &numOfUseDbs) < 0) goto _err; for (int32_t i = 0; i < numOfReadTbs; ++i) { int32_t keyLen = 0; - if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err; - char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); - if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) goto _err; int32_t valuelen = 0; - if (tDecodeI32(pDecoder, &valuelen) < 0) return -1; - char *value = taosMemoryCalloc(valuelen + 1, sizeof(char)); - if (tDecodeCStrTo(pDecoder, value) < 0) return -1; + if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err; + + value = taosMemoryCalloc(valuelen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, value) < 0) goto _err; taosHashPut(pRsp->readTbs, key, strlen(key), value, valuelen + 1); - taosMemoryFree(key); - taosMemoryFree(value); + taosMemoryFreeClear(key); + taosMemoryFreeClear(value); } for (int32_t i = 0; i < numOfWriteTbs; ++i) { int32_t keyLen = 0; - if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err; - char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); - if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) goto _err; int32_t valuelen = 0; - if (tDecodeI32(pDecoder, &valuelen) < 0) return -1; - char *value = taosMemoryCalloc(valuelen + 1, sizeof(char)); - if (tDecodeCStrTo(pDecoder, value) < 0) return -1; + if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err; + + value = taosMemoryCalloc(valuelen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, value) < 0) goto _err; taosHashPut(pRsp->writeTbs, key, strlen(key), value, valuelen + 1); - taosMemoryFree(key); - taosMemoryFree(value); + taosMemoryFreeClear(key); + taosMemoryFreeClear(value); } for (int32_t i = 0; i < numOfUseDbs; ++i) { int32_t keyLen = 0; - if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err; - char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); - if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) goto _err; int32_t ref = 0; - if (tDecodeI32(pDecoder, &ref) < 0) return -1; + if (tDecodeI32(pDecoder, &ref) < 0) goto _err; + taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref)); - taosMemoryFree(key); + taosMemoryFreeClear(key); } // since 3.0.7.0 if (!tDecodeIsEnd(pDecoder)) { - if (tDecodeI32(pDecoder, &pRsp->passVer) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->passVer) < 0) goto _err; } else { pRsp->passVer = 0; } } - return 0; +_err: + taosHashCleanup(pRsp->createdDbs); + taosHashCleanup(pRsp->readDbs); + taosHashCleanup(pRsp->writeDbs); + taosHashCleanup(pRsp->writeTbs); + taosHashCleanup(pRsp->readTbs); + taosHashCleanup(pRsp->useDbs); + + taosMemoryFreeClear(key); + taosMemoryFreeClear(value); + return -1; } int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { @@ -2844,7 +2859,6 @@ int32_t tSerializeSDbHbRspImp(SEncoder *pEncoder, const SDbHbRsp *pRsp) { return 0; } - int32_t tSerializeSDbHbBatchRsp(void *buf, int32_t bufLen, SDbHbBatchRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2908,7 +2922,7 @@ int32_t tDeserializeSUseDbRsp(void *buf, int32_t bufLen, SUseDbRsp *pRsp) { return 0; } -int32_t tDeserializeSDbHbRspImp(SDecoder* decoder, SDbHbRsp* pRsp) { +int32_t tDeserializeSDbHbRspImp(SDecoder *decoder, SDbHbRsp *pRsp) { int8_t flag = 0; if (tDecodeI8(decoder, &flag) < 0) return -1; if (flag) { @@ -3196,7 +3210,7 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { return tlen; } -int32_t tDeserializeSDbCfgRspImpl(SDecoder* decoder, SDbCfgRsp *pRsp) { +int32_t tDeserializeSDbCfgRspImpl(SDecoder *decoder, SDbCfgRsp *pRsp) { if (tDecodeCStrTo(decoder, pRsp->db) < 0) return -1; if (tDecodeI64(decoder, &pRsp->dbId) < 0) return -1; if (tDecodeI32(decoder, &pRsp->cfgVersion) < 0) return -1; @@ -5306,10 +5320,10 @@ int32_t tDeserializeSMqAskEpReq(void *buf, int32_t bufLen, SMqAskEpReq *pReq) { return 0; } -int32_t tDeatroySMqHbReq(SMqHbReq* pReq){ - for(int i = 0; i < taosArrayGetSize(pReq->topics); i++){ - TopicOffsetRows* vgs = taosArrayGet(pReq->topics, i); - if(vgs) taosArrayDestroy(vgs->offsetRows); +int32_t tDeatroySMqHbReq(SMqHbReq *pReq) { + for (int i = 0; i < taosArrayGetSize(pReq->topics); i++) { + TopicOffsetRows *vgs = taosArrayGet(pReq->topics, i); + if (vgs) taosArrayDestroy(vgs->offsetRows); } taosArrayDestroy(pReq->topics); return 0; @@ -5326,7 +5340,7 @@ int32_t tSerializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { int32_t sz = taosArrayGetSize(pReq->topics); if (tEncodeI32(&encoder, sz) < 0) return -1; for (int32_t i = 0; i < sz; ++i) { - TopicOffsetRows* vgs = (TopicOffsetRows*)taosArrayGet(pReq->topics, i); + TopicOffsetRows *vgs = (TopicOffsetRows *)taosArrayGet(pReq->topics, i); if (tEncodeCStr(&encoder, vgs->topicName) < 0) return -1; int32_t szVgs = taosArrayGetSize(vgs->offsetRows); if (tEncodeI32(&encoder, szVgs) < 0) return -1; @@ -5356,19 +5370,19 @@ int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { if (tDecodeI32(&decoder, &pReq->epoch) < 0) return -1; int32_t sz = 0; if (tDecodeI32(&decoder, &sz) < 0) return -1; - if(sz > 0){ + if (sz > 0) { pReq->topics = taosArrayInit(sz, sizeof(TopicOffsetRows)); if (NULL == pReq->topics) return -1; for (int32_t i = 0; i < sz; ++i) { - TopicOffsetRows* data = taosArrayReserve(pReq->topics, 1); + TopicOffsetRows *data = taosArrayReserve(pReq->topics, 1); tDecodeCStrTo(&decoder, data->topicName); int32_t szVgs = 0; if (tDecodeI32(&decoder, &szVgs) < 0) return -1; - if(szVgs > 0){ + if (szVgs > 0) { data->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows)); if (NULL == data->offsetRows) return -1; - for (int32_t j= 0; j < szVgs; ++j) { - OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); + for (int32_t j = 0; j < szVgs; ++j) { + OffsetRows *offRows = taosArrayReserve(data->offsetRows, 1); if (tDecodeI32(&decoder, &offRows->vgId) < 0) return -1; if (tDecodeI64(&decoder, &offRows->rows) < 0) return -1; if (tDecodeSTqOffsetVal(&decoder, &offRows->offset) < 0) return -1; @@ -5382,7 +5396,6 @@ int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) { return 0; } - int32_t tSerializeSMqSeekReq(void *buf, int32_t bufLen, SMqSeekReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { @@ -5610,9 +5623,9 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { int32_t headLen = sizeof(SMsgHead); -// SMsgHead *pHead = buf; -// pHead->vgId = pReq->head.vgId; -// pHead->contLen = pReq->head.contLen; + // SMsgHead *pHead = buf; + // pHead->vgId = pReq->head.vgId; + // pHead->contLen = pReq->head.contLen; SDecoder decoder = {0}; tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); @@ -6983,7 +6996,7 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { return 0; } -int32_t tDecodeSVAlterTbReqSetCtime(SDecoder* pDecoder, SVAlterTbReq* pReq, int64_t ctimeMs) { +int32_t tDecodeSVAlterTbReqSetCtime(SDecoder *pDecoder, SVAlterTbReq *pReq, int64_t ctimeMs) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeSVAlterTbReqCommon(pDecoder, pReq) < 0) return -1; @@ -7216,13 +7229,13 @@ int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) { return 0; } -int32_t tEncodeMqVgOffset(SEncoder* pEncoder, const SMqVgOffset* pOffset) { +int32_t tEncodeMqVgOffset(SEncoder *pEncoder, const SMqVgOffset *pOffset) { if (tEncodeSTqOffset(pEncoder, &pOffset->offset) < 0) return -1; if (tEncodeI64(pEncoder, pOffset->consumerId) < 0) return -1; return 0; } -int32_t tDecodeMqVgOffset(SDecoder* pDecoder, SMqVgOffset* pOffset) { +int32_t tDecodeMqVgOffset(SDecoder *pDecoder, SMqVgOffset *pOffset) { if (tDecodeSTqOffset(pDecoder, &pOffset->offset) < 0) return -1; if (tDecodeI64(pDecoder, &pOffset->consumerId) < 0) return -1; return 0; @@ -7407,7 +7420,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) { } int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { - if (tDecodeMqDataRsp(pDecoder, (SMqDataRsp*)pRsp) < 0) return -1; + if (tDecodeMqDataRsp(pDecoder, (SMqDataRsp *)pRsp) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->createTableNum) < 0) return -1; if (pRsp->createTableNum) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 32b63fa950..f7f57f2455 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -455,7 +455,7 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { } } - if (diffIdx == -1 && diffIdx == 0) { + if (diffIdx == -1 || diffIdx == 0) { goto _err; } @@ -1654,10 +1654,11 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb if (ret < 0) { terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; return -1; + } else { + uid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; } - uid = *(tb_uid_t *)pVal; - tdbFree(pVal); - pVal = NULL; if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { ret = -1; @@ -1736,12 +1737,16 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb nTagData = tDataTypes[pCol->type].bytes; } if (metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, uid, &pTagIdxKey, &nTagIdxKey) < 0) { + tdbFree(pKey); + tdbFree(pVal); metaDestroyTagIdxKey(pTagIdxKey); + tdbTbcClose(pCtbIdxc); goto _err; } tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); } + tdbTbcClose(pCtbIdxc); return 0; _err: diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index cebe4e8204..2fa056d8ab 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -189,9 +189,9 @@ void streamBackendCleanup(void* arg) { taosThreadMutexDestroy(&pHandle->mutex); taosThreadMutexDestroy(&pHandle->cfMutex); + qDebug("destroy stream backend backend:%p", pHandle); taosMemoryFree(pHandle); - qDebug("destroy stream backend backend:%p", pHandle); return; } SListNode* streamBackendAddCompare(void* backend, void* arg) { @@ -704,7 +704,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t char suffix[64] = {0}; rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); - RocksdbCfParam* params = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam*)); + RocksdbCfParam* params = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); rocksdb_comparator_t** pCompare = taosMemoryCalloc(nCf, sizeof(rocksdb_comparator_t**)); rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); @@ -861,7 +861,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { param[i].tableOpt = tableOpt; }; - rocksdb_comparator_t** pCompare = taosMemoryCalloc(cfLen, sizeof(rocksdb_comparator_t**)); + rocksdb_comparator_t** pCompare = taosMemoryCalloc(cfLen, sizeof(rocksdb_comparator_t*)); for (int i = 0; i < cfLen; i++) { SCfInit* cf = &ginitDict[i]; @@ -1012,16 +1012,15 @@ bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len } rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt) { - int idx = streamStateGetCfIdx(pState, cfName); - - if (snapshot != NULL) { - *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(pState->pTdbState->rocksdb); - } + int idx = streamStateGetCfIdx(pState, cfName); rocksdb_readoptions_t* rOpt = rocksdb_readoptions_create(); *readOpt = rOpt; - rocksdb_readoptions_set_snapshot(rOpt, *snapshot); - rocksdb_readoptions_set_fill_cache(rOpt, 0); + if (snapshot != NULL) { + *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(pState->pTdbState->rocksdb); + rocksdb_readoptions_set_snapshot(rOpt, *snapshot); + rocksdb_readoptions_set_fill_cache(rOpt, 0); + } return rocksdb_create_iterator_cf(pState->pTdbState->rocksdb, rOpt, ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[idx]); @@ -1049,8 +1048,8 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ if (err != NULL) { \ - taosMemoryFree(err); \ qError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ + taosMemoryFree(err); \ code = -1; \ } else { \ qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d", toString, funcname, vLen, ttlVLen); \ @@ -1389,8 +1388,6 @@ int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* k int code = 0; SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; STREAM_STATE_PUT_ROCKSDB(pState, "sess", &sKey, value, vLen); - if (code == -1) { - } return code; } int32_t streamStateSessionGet_rocksdb(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { @@ -1408,8 +1405,10 @@ int32_t streamStateSessionGet_rocksdb(SStreamState* pState, SSessionKey* key, vo code = -1; } else { *key = resKey; - *pVal = taosMemoryCalloc(1, *pVLen); - memcpy(*pVal, tmp, *pVLen); + if (pVal != NULL && pVLen != NULL) { + *pVal = taosMemoryCalloc(1, *pVLen); + memcpy(*pVal, tmp, *pVLen); + } } } taosMemoryFree(tmp); diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 04b546b36a..c483d82027 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -391,7 +391,13 @@ static void httpHandleReq(SHttpMsg* msg) { // set up timeout to avoid stuck; int32_t fd = taosCreateSocketWithTimeout(5); - int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); + if (fd < 0) { + tError("http-report failed to open socket, dst:%s:%d", cli->addr, cli->port); + taosReleaseRef(httpRefMgt, httpRef); + destroyHttpClient(cli); + return; + } + int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); if (ret != 0) { tError("http-report failed to open socket, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); taosReleaseRef(httpRefMgt, httpRef); From 2279706357a662b17251f1543be538f7521dac26 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 14 Jul 2023 14:49:02 +0800 Subject: [PATCH 022/100] fix coverity scan problem --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 2fa056d8ab..05a1cf33ec 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -705,7 +705,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); RocksdbCfParam* params = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam)); - rocksdb_comparator_t** pCompare = taosMemoryCalloc(nCf, sizeof(rocksdb_comparator_t**)); + rocksdb_comparator_t** pCompare = taosMemoryCalloc(nCf, sizeof(rocksdb_comparator_t*)); rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); for (int i = 0; i < nCf; i++) { From 49b21b3ccb0aeef4462ddc28341c9d69cf502dcd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 14 Jul 2023 14:53:11 +0800 Subject: [PATCH 023/100] fix coverity scan problem --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 05a1cf33ec..242256cd34 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -189,8 +189,8 @@ void streamBackendCleanup(void* arg) { taosThreadMutexDestroy(&pHandle->mutex); taosThreadMutexDestroy(&pHandle->cfMutex); - qDebug("destroy stream backend backend:%p", pHandle); + qDebug("destroy stream backend backend:%p", pHandle); taosMemoryFree(pHandle); return; } From 3d5a05eb666453e8dfa2036f4f54c69fd3a2a69d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 14 Jul 2023 15:23:33 +0800 Subject: [PATCH 024/100] test:add testcase of rolling upgdade --- tests/parallel_test/cases.task | 2 +- tests/system-test/6-cluster/clusterCommonCheck.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 90e0557997..f9332497b5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -446,7 +446,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 #,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 ,,n,system-test,python3 ./test.py -f 6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py -N 6 -M 3 - +#,,n,system-test,python ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 #,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index 7aa2ba06b9..439f0b6b8c 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -256,12 +256,12 @@ class ClusterComCheck: if vgroup_status_first.count('leader') == 1 and vgroup_status_first.count('follower') == 2: if vgroup_status_last.count('leader') == 1 and vgroup_status_last.count('follower') == 2: ready_time= (count + 1) - tdLog.success(f"elections of {db_name} all vgroups are ready in {ready_time} s") + tdLog.success(f"elections of {db_name}.vgroups are ready in {ready_time} s") return True count+=1 else: tdLog.debug(tdSql.queryResult) - tdLog.notice(f"elections of {db_name} all vgroups are failed in{count}s ") + tdLog.notice(f"elections of {db_name} all vgroups are failed in{count} s ") caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno) tdLog.exit("%s(%d) failed " % args) From d969033ccf683c9339f093ec8e10b0af2188a83b Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 14 Jul 2023 15:27:27 +0800 Subject: [PATCH 025/100] test: increase timout to 180s of replica from 1 to 3 --- .../manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py index fede19ca3a..16ad3506c8 100644 --- a/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py +++ b/tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py @@ -182,7 +182,7 @@ class TDTestCase: tdLog.info(f"show transactions;alter database db0_0 replica {replica3};") TdSqlEx.execute(f'show transactions;') TdSqlEx.execute(f'alter database db0_0 replica {replica3};') - clusterComCheck.check_vgroups_status(vgroup_numbers=paraDict["vgroups"],db_replica=replica3,db_name=paraDict["dbName"],count_number=120) + clusterComCheck.check_vgroups_status(vgroup_numbers=paraDict["vgroups"],db_replica=replica3,db_name=paraDict["dbName"],count_number=180) def run(self): # print(self.master_dnode.cfgDict) From b850c1af47a6b597c1e867377e695b8dfc4abcfb Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 14 Jul 2023 16:03:59 +0800 Subject: [PATCH 026/100] fix: state/session/event window with order by _wstart/_wend has no effect --- include/libs/nodes/plannodes.h | 1 + source/libs/planner/src/planLogicCreater.c | 2 +- source/libs/planner/src/planOptimizer.c | 58 +- .../tsim/query/r/explain_tsorder.result | 1160 +++++++++++++++++ tests/script/tsim/query/t/explain_tsorder.sql | 62 + 5 files changed, 1261 insertions(+), 22 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index c1481da80c..68bc277378 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -247,6 +247,7 @@ typedef struct SSortLogicNode { SNodeList* pSortKeys; bool groupSort; int64_t maxRows; + bool skipPKSortOpt; } SSortLogicNode; typedef struct SPartitionLogicNode { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 4a8d100db3..5374fcf7a4 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1034,7 +1034,6 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pSort->node.resultDataOrder = isPrimaryKeySort(pSelect->pOrderByList) ? (pSort->groupSort ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) : DATA_ORDER_LEVEL_NONE; - int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) { code = nodesListMakeStrictAppend(&pSort->node.pTargets, @@ -1048,6 +1047,7 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } SNode* pNode = NULL; SOrderByExprNode* firstSortKey = (SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0); + if (isPrimaryKeySort(pSelect->pOrderByList)) pSort->node.outputTsOrder = firstSortKey->order; if (firstSortKey->pExpr->type == QUERY_NODE_COLUMN) { SColumnNode* pCol = (SColumnNode*)firstSortKey->pExpr; int16_t projIdx = 1; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 82d883714d..8575f42fa3 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1168,7 +1168,8 @@ static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) { return false; } SSortLogicNode* pSort = (SSortLogicNode*)pNode; - if (!sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) { + if (pSort->skipPKSortOpt || !sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || + 1 != LIST_LENGTH(pSort->node.pChildren)) { return false; } SNode* pChild; @@ -1181,8 +1182,8 @@ static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) { return true; } -static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, bool* pNotOptimize, - SNodeList** pSequencingNodes) { +static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, EOrder sortOrder, + bool* pNotOptimize, SNodeList** pSequencingNodes) { if (NULL != pNode->pLimit || NULL != pNode->pSlimit) { *pNotOptimize = false; return TSDB_CODE_SUCCESS; @@ -1199,15 +1200,21 @@ static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool group } case QUERY_NODE_LOGIC_PLAN_JOIN: { int32_t code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), groupSort, - pNotOptimize, pSequencingNodes); + sortOrder, pNotOptimize, pSequencingNodes); if (TSDB_CODE_SUCCESS == code) { code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), groupSort, - pNotOptimize, pSequencingNodes); + sortOrder, pNotOptimize, pSequencingNodes); } return code; } - case QUERY_NODE_LOGIC_PLAN_WINDOW: - return nodesListMakeAppend(pSequencingNodes, (SNode*)pNode); + case QUERY_NODE_LOGIC_PLAN_WINDOW: { + SWindowLogicNode* pWindowLogicNode = (SWindowLogicNode*)pNode; + // For interval window, we always apply sortPriKey optimization. + // For session/event/state window, the output ts order will always be ASC. + // If sort order is also asc, we apply optimization, otherwise we keep sort node to get correct output order. + if (pWindowLogicNode->winType == WINDOW_TYPE_INTERVAL || sortOrder == ORDER_ASC) + return nodesListMakeAppend(pSequencingNodes, (SNode*)pNode); + } case QUERY_NODE_LOGIC_PLAN_AGG: case QUERY_NODE_LOGIC_PLAN_PARTITION: *pNotOptimize = true; @@ -1221,23 +1228,25 @@ static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool group return TSDB_CODE_SUCCESS; } - return sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), groupSort, + return sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), groupSort, sortOrder, pNotOptimize, pSequencingNodes); } -static int32_t sortPriKeyOptGetSequencingNodes(SLogicNode* pNode, bool groupSort, SNodeList** pSequencingNodes) { - bool notOptimize = false; - int32_t code = sortPriKeyOptGetSequencingNodesImpl(pNode, groupSort, ¬Optimize, pSequencingNodes); - if (TSDB_CODE_SUCCESS != code || notOptimize) { - NODES_CLEAR_LIST(*pSequencingNodes); - } - return code; -} - static EOrder sortPriKeyOptGetPriKeyOrder(SSortLogicNode* pSort) { return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order; } +static int32_t sortPriKeyOptGetSequencingNodes(SSortLogicNode* pSort, bool groupSort, SNodeList** pSequencingNodes) { + bool notOptimize = false; + int32_t code = + sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), groupSort, + sortPriKeyOptGetPriKeyOrder(pSort), ¬Optimize, pSequencingNodes); + if (TSDB_CODE_SUCCESS != code || notOptimize) { + NODES_CLEAR_LIST(*pSequencingNodes); + } + return code; +} + static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort, SNodeList* pSequencingNodes) { EOrder order = sortPriKeyOptGetPriKeyOrder(pSort); @@ -1276,10 +1285,17 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS static int32_t sortPrimaryKeyOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort) { SNodeList* pSequencingNodes = NULL; - int32_t code = sortPriKeyOptGetSequencingNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), - pSort->groupSort, &pSequencingNodes); - if (TSDB_CODE_SUCCESS == code && NULL != pSequencingNodes) { - code = sortPriKeyOptApply(pCxt, pLogicSubplan, pSort, pSequencingNodes); + int32_t code = sortPriKeyOptGetSequencingNodes(pSort, pSort->groupSort, &pSequencingNodes); + if (TSDB_CODE_SUCCESS == code) { + if (pSequencingNodes != NULL) { + code = sortPriKeyOptApply(pCxt, pLogicSubplan, pSort, pSequencingNodes); + } else { + // if we decided not to push down sort info to children, we should propagate output ts order to parents of pSort + optSetParentOrder(pSort->node.pParent, sortPriKeyOptGetPriKeyOrder(pSort), 0); + // we need to prevent this pSort from being chosen to do optimization again + pSort->skipPKSortOpt = true; + pCxt->optimized = true; + } } nodesClearList(pSequencingNodes); return code; diff --git a/tests/script/tsim/query/r/explain_tsorder.result b/tests/script/tsim/query/r/explain_tsorder.result index b69a77ada5..25f1241ffd 100644 --- a/tests/script/tsim/query/r/explain_tsorder.result +++ b/tests/script/tsim/query/r/explain_tsorder.result @@ -2798,3 +2798,1163 @@ taos> select last(ts) as ts, c2 as d from d1 group by c2 order by ts desc, c2 as ======================================== 2022-05-15 00:01:08.000 | 234 | +taos> explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Event (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Start Cond: (`test`.`meters`.`c2` > 0) +*************************** 3.row *************************** +QUERY_PLAN: End Cond: (`test`.`meters`.`c2` < 100) +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Event (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Start Cond: (`test`.`meters`.`c2` > 0) +*************************** 5.row *************************** +QUERY_PLAN: End Cond: (`test`.`meters`.`c2` < 100) +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Event (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Start Cond: (`test`.`meters`.`c2` > 0) +*************************** 3.row *************************** +QUERY_PLAN: End Cond: (`test`.`meters`.`c2` < 100) +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Event (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Start Cond: (`test`.`meters`.`c2` > 0) +*************************** 5.row *************************** +QUERY_PLAN: End Cond: (`test`.`meters`.`c2` < 100) +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Event (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Start Cond: (`test`.`meters`.`c2` > 0) +*************************** 3.row *************************** +QUERY_PLAN: End Cond: (`test`.`meters`.`c2` < 100) +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-17 00:01:08.000 | 5 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-19 00:01:08.000 | 2022-05-21 00:01:08.000 | 5 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 1 | + +taos> select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 1 | + 2022-05-19 00:01:08.000 | 2022-05-21 00:01:08.000 | 5 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 1 | + 2022-05-15 00:01:08.000 | 2022-05-17 00:01:08.000 | 5 | + +taos> select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-17 00:01:08.000 | 5 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-19 00:01:08.000 | 2022-05-21 00:01:08.000 | 5 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 1 | + +taos> select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-19 00:01:08.000 | 2022-05-21 00:01:08.000 | 5 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-15 00:01:08.000 | 2022-05-17 00:01:08.000 | 5 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 1 | + +taos> select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-17 00:01:08.000 | 5 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 1 | + 2022-05-19 00:01:08.000 | 2022-05-21 00:01:08.000 | 5 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 1 | + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h)\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 5.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 5.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h); + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h)\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 5.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 5.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Session (functions=3 width=24) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Window: gap=3600000 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=1 width=8 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=8) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=1 width=8 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=1 width=8 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h); + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + +taos> select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc; + _wstart | _wend | count(*) | +============================================================================ + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2)\G; +*************************** 1.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=4 width=32) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=4 width=32) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 4.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 6.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 7.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 8.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 9.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 10.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 13.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 14.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 15.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> select _wstart, _wend, count(*), last(ts) from meters state_window(c2); + _wstart | _wend | count(*) | last(ts) | +====================================================================================================== + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | 2022-05-15 00:01:08.000 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | 2022-05-16 00:01:08.000 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | 2022-05-17 00:01:08.000 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | 2022-05-18 00:01:08.000 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | 2022-05-19 00:01:08.000 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | 2022-05-20 00:01:08.000 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | 2022-05-21 00:01:08.000 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | 2022-05-22 00:01:08.000 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | 2022-05-23 00:01:08.000 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | 2022-05-24 00:01:08.000 | + +taos> select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart desc; + _wstart | _wend | count(*) | last(ts) | +====================================================================================================== + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | 2022-05-24 00:01:08.000 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | 2022-05-23 00:01:08.000 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | 2022-05-22 00:01:08.000 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | 2022-05-21 00:01:08.000 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | 2022-05-20 00:01:08.000 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | 2022-05-19 00:01:08.000 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | 2022-05-18 00:01:08.000 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | 2022-05-17 00:01:08.000 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | 2022-05-16 00:01:08.000 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | 2022-05-15 00:01:08.000 | + +taos> select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart asc; + _wstart | _wend | count(*) | last(ts) | +====================================================================================================== + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | 2022-05-15 00:01:08.000 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | 2022-05-16 00:01:08.000 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | 2022-05-17 00:01:08.000 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | 2022-05-18 00:01:08.000 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | 2022-05-19 00:01:08.000 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | 2022-05-20 00:01:08.000 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | 2022-05-21 00:01:08.000 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | 2022-05-22 00:01:08.000 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | 2022-05-23 00:01:08.000 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | 2022-05-24 00:01:08.000 | + +taos> select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc; + _wstart | _wend | count(*) | last(ts) | +====================================================================================================== + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | 2022-05-24 00:01:08.000 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | 2022-05-23 00:01:08.000 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | 2022-05-22 00:01:08.000 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | 2022-05-21 00:01:08.000 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | 2022-05-20 00:01:08.000 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | 2022-05-19 00:01:08.000 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | 2022-05-18 00:01:08.000 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | 2022-05-17 00:01:08.000 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | 2022-05-16 00:01:08.000 | + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | 2022-05-15 00:01:08.000 | + +taos> select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc; + _wstart | _wend | count(*) | last(ts) | +====================================================================================================== + 2022-05-15 00:01:08.000 | 2022-05-15 00:01:08.000 | 2 | 2022-05-15 00:01:08.000 | + 2022-05-16 00:01:08.000 | 2022-05-16 00:01:08.000 | 2 | 2022-05-16 00:01:08.000 | + 2022-05-17 00:01:08.000 | 2022-05-17 00:01:08.000 | 2 | 2022-05-17 00:01:08.000 | + 2022-05-18 00:01:08.000 | 2022-05-18 00:01:08.000 | 2 | 2022-05-18 00:01:08.000 | + 2022-05-19 00:01:08.000 | 2022-05-19 00:01:08.000 | 2 | 2022-05-19 00:01:08.000 | + 2022-05-20 00:01:08.000 | 2022-05-20 00:01:08.000 | 2 | 2022-05-20 00:01:08.000 | + 2022-05-21 00:01:08.000 | 2022-05-21 00:01:08.000 | 2 | 2022-05-21 00:01:08.000 | + 2022-05-22 00:01:08.000 | 2022-05-22 00:01:08.000 | 2 | 2022-05-22 00:01:08.000 | + 2022-05-23 00:01:08.000 | 2022-05-23 00:01:08.000 | 2 | 2022-05-23 00:01:08.000 | + 2022-05-24 00:01:08.000 | 2022-05-24 00:01:08.000 | 2 | 2022-05-24 00:01:08.000 | + +taos> explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc, count(*) desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=asc (columns=4 width=32) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 3.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=5 width=44) +*************************** 4.row *************************** +QUERY_PLAN: Output: columns=5 width=40 +*************************** 5.row *************************** +QUERY_PLAN: Output: columns=5 width=40 +*************************** 6.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 7.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 8.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 9.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 10.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 12.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 15.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 16.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 17.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 18.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 19.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, last(ts) from (select _wstart as ts, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc) interval(1h) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=24 input_order=desc output_order=desc ) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Time Window: interval=1h offset=0a sliding=1h +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=desc ) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 7.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 8.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 9.row *************************** +QUERY_PLAN: -> Sort input_order=asc output_order=desc (columns=3 width=24) +*************************** 10.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 11.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 12.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 14.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 15.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 16.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 17.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 18.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 20.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 21.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 22.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 23.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 24.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 25.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 26.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 27.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + +taos> explain verbose true select _wstart, _wend, last(ts) from (select _wstart as ts, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc) interval(1h) order by _wstart desc\G; +*************************** 1.row *************************** +QUERY_PLAN: -> Interval on Column ts (functions=3 width=24 input_order=asc output_order=desc ) +*************************** 2.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 3.row *************************** +QUERY_PLAN: Time Window: interval=1h offset=0a sliding=1h +*************************** 4.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 5.row *************************** +QUERY_PLAN: -> Projection (columns=3 width=24 input_order=asc ) +*************************** 6.row *************************** +QUERY_PLAN: Output: columns=3 width=24 +*************************** 7.row *************************** +QUERY_PLAN: Output: Ignore Group Id: true +*************************** 8.row *************************** +QUERY_PLAN: Merge ResBlocks: True +*************************** 9.row *************************** +QUERY_PLAN: -> StateWindow on Column c2 (functions=4 width=36) +*************************** 10.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 11.row *************************** +QUERY_PLAN: Output: columns=4 width=32 +*************************** 12.row *************************** +QUERY_PLAN: -> SortMerge (columns=2 width=12 input_order=unknown output_order=unknown) +*************************** 13.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 14.row *************************** +QUERY_PLAN: Output: Ignore Group Id: false +*************************** 15.row *************************** +QUERY_PLAN: Merge Key: _group_id asc, ts asc +*************************** 16.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 17.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 18.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 19.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 20.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] +*************************** 21.row *************************** +QUERY_PLAN: -> Data Exchange 1:1 (width=12) +*************************** 22.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 23.row *************************** +QUERY_PLAN: -> Table Merge Scan on meters (columns=2 width=12 order=[asc|1 desc|0]) +*************************** 24.row *************************** +QUERY_PLAN: Output: columns=2 width=12 +*************************** 25.row *************************** +QUERY_PLAN: Time Range: [-9223372036854775808, 9223372036854775807] + diff --git a/tests/script/tsim/query/t/explain_tsorder.sql b/tests/script/tsim/query/t/explain_tsorder.sql index 056ac440fe..53bfb9a597 100644 --- a/tests/script/tsim/query/t/explain_tsorder.sql +++ b/tests/script/tsim/query/t/explain_tsorder.sql @@ -98,3 +98,65 @@ select last(ts), c2 as d from d1 group by c2 order by c2 asc limit 9,1; select last(ts) as ts, c2 as d from d1 group by c2 order by ts desc, c2 asc limit 10; select last(ts) as ts, c2 as d from d1 group by c2 order by ts desc, c2 asc limit 2,8; select last(ts) as ts, c2 as d from d1 group by c2 order by ts desc, c2 asc limit 9,1; + +explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100\G; +explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart desc\G; +explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart asc\G; + +explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend desc\G; +explain verbose true select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend asc\G; + +select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100; +select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart desc; +select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wstart asc; + +select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend desc; +select _wstart, _wend, count(*) from meters event_window start with c2 > 0 end with c2 < 100 order by _wend asc; + +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h)\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc\G; + +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc\G; + +select _wstart, _wend, count(*) from meters session(ts, 1h); +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc; +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc; + +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc; +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc; + + +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h)\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc\G; + +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc\G; +explain verbose true select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc\G; + +select _wstart, _wend, count(*) from meters session(ts, 1h); +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart desc; +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wstart asc; + +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend desc; +select _wstart, _wend, count(*) from meters session(ts, 1h) order by _wend asc; + +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2)\G; +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart desc\G; +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart asc\G; + +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc\G; +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc\G; + +select _wstart, _wend, count(*), last(ts) from meters state_window(c2); +select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart desc; +select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wstart asc; + +select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc; +select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc; + +explain verbose true select _wstart, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc, count(*) desc\G; + +explain verbose true select _wstart, _wend, last(ts) from (select _wstart as ts, _wend, count(*), last(ts) from meters state_window(c2) order by _wend desc) interval(1h) order by _wstart desc\G; +explain verbose true select _wstart, _wend, last(ts) from (select _wstart as ts, _wend, count(*), last(ts) from meters state_window(c2) order by _wend asc) interval(1h) order by _wstart desc\G; From 9085dac1a2ba335701da51a76d3be137e156d8dd Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 14 Jul 2023 16:31:37 +0800 Subject: [PATCH 027/100] add create sma index case --- tests/pytest/util/sql.py | 4 +- tests/system-test/0-others/timeRangeWise.py | 297 ++++++++++++++++++++ tests/system-test/empty.py | 40 +++ 3 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/0-others/timeRangeWise.py create mode 100644 tests/system-test/empty.py diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 2af8f721b6..2fa21b1983 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -440,8 +440,10 @@ class TDSql: time.sleep(1) continue - def execute(self, sql,queryTimes=30): + def execute(self, sql, queryTimes=30, show=False): self.sql = sql + if show: + tdLog.info(sql) i=1 while i <= queryTimes: try: diff --git a/tests/system-test/0-others/timeRangeWise.py b/tests/system-test/0-others/timeRangeWise.py new file mode 100644 index 0000000000..d558b1d693 --- /dev/null +++ b/tests/system-test/0-others/timeRangeWise.py @@ -0,0 +1,297 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time +import copy +import string + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # random string + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null," + else: + value += f"{c2}," + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + # c3 same with ts + value += f"{self.ts})" + + # move next 1s interval + self.ts += 100 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into @db_name.t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + self.exeDouble(sql) + values = "" + + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + def exeDouble(self, sql): + # dbname replace + sql1 = sql.replace("@db_name", self.db1) + + if len(sql1) > 100: + tdLog.info(sql1[:100]) + else: + tdLog.info(sql1) + tdSql.execute(sql1) + + sql2 = sql.replace("@db_name", self.db2) + if len(sql2) > 100: + tdLog.info(sql2[:100]) + else: + tdLog.info(sql2) + tdSql.execute(sql2) + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1680000000000 + self.childCnt = 2 + self.childRow = 100000 + self.batchSize = 5000 + self.vgroups1 = 4 + self.vgroups2 = 4 + self.db1 = "db1" # no sma + self.db2 = "db2" # have sma + self.smaClause = "interval(1h)" + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # alter local optimization to treu + sql = "alter local 'querysmaoptimize 1'" + tdSql.execute(sql, 5, True) + + # check forbid mulit-replic on create sma index + sql = f"create database db vgroups {self.vgroups1} replica 3" + tdSql.execute(sql, 5, True) + sql = f"create table db.st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + tdSql.execute(sql, 5, True) + + sql = f"create sma index sma_test on db.st function(max(c1),max(c2),min(c1),min(c2)) {self.smaClause};" + tdLog.info(sql) + tdSql.error(sql) + + + # create database db + sql = f"create database @db_name vgroups {self.vgroups1} replica 1" + self.exeDouble(sql) + + # create super talbe st + sql = f"create table @db_name.st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)" + self.exeDouble(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table @db_name.t{i} using @db_name.st tags({i}) " + self.exeDouble(sql) + + # create sma index on db2 + sql = f"use {self.db2}" + tdSql.execute(sql) + sql = f"create sma index sma_index_maxmin on {self.db2}.st function(max(c1),max(c2),min(c1),min(c2)) {self.smaClause};" + tdLog.info(sql) + tdSql.execute(sql) + + # insert data + self.insertData() + + # check data correct + def checkExpect(self, sql, expectVal): + tdSql.query(sql) + rowCnt = tdSql.getRows() + for i in range(rowCnt): + val = tdSql.getData(i,0) + if val != expectVal: + tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}") + return False + + tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}") + return True + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # check query result same + def queryDouble(self, sql): + # sql + sql1 = sql.replace('@db_name', self.db1) + tdLog.info(sql1) + start1 = time.time() + rows1 = tdSql.query(sql1) + spend1 = time.time() - start1 + res1 = copy.copy(tdSql.queryResult) + + sql2 = sql.replace('@db_name', self.db2) + tdLog.info(sql2) + start2 = time.time() + tdSql.query(sql2) + spend2 = time.time() - start2 + res2 = tdSql.queryResult + + rowlen1 = len(res1) + rowlen2 = len(res2) + + if rowlen1 != rowlen2: + tdLog.exit(f"rowlen1={rowlen1} rowlen2={rowlen2} both not equal.") + return False + + for i in range(rowlen1): + row1 = res1[i] + row2 = res2[i] + collen1 = len(row1) + collen2 = len(row2) + if collen1 != collen2: + tdLog.exit(f"collen1={collen1} collen2={collen2} both not equal.") + return False + for j in range(collen1): + if row1[j] != row2[j]: + tdLog.exit(f"col={j} col1={row1[j]} col2={row2[j]} both col not equal.") + return False + + # warning performance + diff = (spend2 - spend1)*100/spend1 + tdLog.info("spend1=%.6fs spend2=%.6fs diff=%.1f%%"%(spend1, spend2, diff)) + if spend2 > spend1 and diff > 20: + tdLog.info("warning: the diff for performance after spliting is over 20%") + + return True + + + # check result + def checkResult(self): + + # max + sql = f"select max(c1) from @db_name.st {self.smaClause}" + self.queryDouble(sql) + + # min + sql = f"select max(c2) from @db_name.st {self.smaClause}" + self.queryDouble(sql) + + # mix + sql = f"select max(c1),max(c2),min(c1),min(c2) from @db_name.st {self.smaClause}" + self.queryDouble(sql) + + + # run + def run(self): + # prepare env + self.prepareEnv() + + # check two db query result same + tdLog.info(f"check have sma(db1) and no sma(db2) performace...") + self.checkResult() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/empty.py b/tests/system-test/empty.py new file mode 100644 index 0000000000..fc44d0164f --- /dev/null +++ b/tests/system-test/empty.py @@ -0,0 +1,40 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + # init + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # run + def run(self): + # check two db query result same + tdLog.info(f"hello world.") + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From bbdcbcb75bf9283225039ae3b7d1a7f05ec8fe6a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 14 Jul 2023 16:43:28 +0800 Subject: [PATCH 028/100] fix:modify commit version to next validate version --- include/client/taos.h | 14 +++++++++----- include/libs/executor/storageapi.h | 2 +- source/client/src/clientTmq.c | 12 ++++++------ source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/tq/tqUtil.c | 14 +++++++------- source/dnode/vnode/src/vnd/vnodeInitApi.c | 2 +- source/libs/executor/src/executor.c | 8 ++++---- source/libs/executor/src/scanoperator.c | 13 +++++++------ source/libs/wal/src/walRead.c | 4 ++-- 9 files changed, 38 insertions(+), 33 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 7bdf16ed38..3cc2d907ab 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -287,11 +287,20 @@ DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param); +DLL_EXPORT int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset); +DLL_EXPORT void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param); DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment, int32_t *numOfAssignment); DLL_EXPORT void tmq_free_assignment(tmq_topic_assignment* pAssignment); DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset); +DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); +DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); +DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); +DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); +DLL_EXPORT int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId); +DLL_EXPORT int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId); + /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ enum tmq_conf_res_t { @@ -309,11 +318,6 @@ DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_comm /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ -DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); -DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); -DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); -DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); - /* ------------------------------ TAOSX -----------------------------------*/ // note: following apis are unstable enum tmq_res_t { diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 6031b99cfc..41516adc73 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -228,7 +228,7 @@ typedef struct SStoreTqReader { } SStoreTqReader; typedef struct SStoreSnapshotFn { - int32_t (*createSnapshot)(SSnapContext* ctx, int64_t uid); + int32_t (*setForSnapShot)(SSnapContext* ctx, int64_t uid); int32_t (*destroySnapshot)(SSnapContext* ctx); SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx); int32_t (*getTableInfoFromSnapshot)(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 5879de2e30..6bbcbe62be 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1901,7 +1901,7 @@ static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal // update the valid wal version range pVg->offsetInfo.walVerBegin = sver; - pVg->offsetInfo.walVerEnd = ever; + pVg->offsetInfo.walVerEnd = ever + 1; // pVg->receivedInfoFromVnode = true; } @@ -2541,7 +2541,7 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { SMqRspHead* pHead = pMsg->pData; tmq_topic_assignment assignment = {.begin = pHead->walsver, - .end = pHead->walever, + .end = pHead->walever + 1, .currentOffset = rsp.rspOffset.version, .vgId = pParam->vgId}; @@ -2600,7 +2600,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a *numOfAssignment = taosArrayGetSize(pTopic->vgs); for (int32_t j = 0; j < (*numOfAssignment); ++j) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); - int32_t type = pClientVg->offsetInfo.currentOffset.type; + int32_t type = pClientVg->offsetInfo.seekOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type); code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; @@ -2620,7 +2620,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a for (int32_t j = 0; j < (*numOfAssignment); ++j) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); - if (pClientVg->offsetInfo.currentOffset.type != TMQ_OFFSET__LOG) { + if (pClientVg->offsetInfo.seekOffset.type != TMQ_OFFSET__LOG) { needFetch = true; break; } @@ -2705,7 +2705,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a int64_t transporterId = 0; char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.currentOffset); + tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.seekOffset); tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); @@ -2825,7 +2825,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ // update the offset, and then commit to vnode pOffsetInfo->currentOffset.type = TMQ_OFFSET__LOG; - pOffsetInfo->currentOffset.version = offset >= 1 ? offset - 1 : 0; + pOffsetInfo->currentOffset.version = offset; pOffsetInfo->seekOffset = pOffsetInfo->currentOffset; // pOffsetInfo->committedOffset.version = INT64_MIN; pVg->seekUpdated = true; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 5f53f1c50c..4299cd471a 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -196,7 +196,7 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return, reqId:0x%" PRIx64, pHandle->consumerId, pHandle->epoch, vgId, offset, reqId); - *fetchOffset = offset - 1; + *fetchOffset = offset; code = -1; goto END; } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 8948bae852..7d632f44bc 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -119,7 +119,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } } else { walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); - tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); + tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer); } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); @@ -127,7 +127,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, pRequest); - tqOffsetResetToLog(&dataRsp.rspOffset, pHandle->pRef->refVer); + tqOffsetResetToLog(&dataRsp.rspOffset, pHandle->pRef->refVer + 1); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, pHandle->subKey, vgId, dataRsp.rspOffset.version); int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); @@ -138,7 +138,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } else { STaosxRsp taosxRsp = {0}; tqInitTaosxRsp(&taosxRsp, pRequest); - tqOffsetResetToLog(&taosxRsp.rspOffset, pHandle->pRef->refVer); + tqOffsetResetToLog(&taosxRsp.rspOffset, pHandle->pRef->refVer + 1); int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); tDeleteSTaosxRsp(&taosxRsp); @@ -246,7 +246,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (offset->type == TMQ_OFFSET__LOG) { walReaderVerifyOffset(pHandle->pWalReader, offset); - int64_t fetchVer = offset->version + 1; + int64_t fetchVer = offset->version; pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -279,14 +279,14 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, // process meta if (pHead->msgType != TDMT_VND_SUBMIT) { if (totalRows > 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1); + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); setRequestVersion(&taosxRsp.reqOffset, offset->version); code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); goto end; } tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); - tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); + tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1); metaRsp.resMsgType = pHead->msgType; metaRsp.metaRspLen = pHead->bodyLen; metaRsp.metaRsp = pHead->body; @@ -309,7 +309,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } if (totalRows >= 4096 || taosxRsp.createTableNum > 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1); setRequestVersion(&taosxRsp.reqOffset, offset->version); code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); goto end; diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index d2db6368a2..447cab4e9e 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -237,7 +237,7 @@ void initCacheFn(SStoreCacheReader* pCache) { } void initSnapshotFn(SStoreSnapshotFn* pSnapshot) { - pSnapshot->createSnapshot = setForSnapShot; + pSnapshot->setForSnapShot = setForSnapShot; pSnapshot->destroySnapshot = destroySnapContext; pSnapshot->getMetaTableInfoFromSnapshot = getMetaTableInfoFromSnapshot; pSnapshot->getTableInfoFromSnapshot = getTableInfoFromSnapshot; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 88e2165a12..dfb0c1645a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1112,8 +1112,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SStoreTqReader* pReaderAPI = &pTaskInfo->storageAPI.tqReaderFn; SWalReader* pWalReader = pReaderAPI->tqReaderGetWalReader(pInfo->tqReader); walReaderVerifyOffset(pWalReader, pOffset); - if (pReaderAPI->tqReaderSeek(pInfo->tqReader, pOffset->version + 1, id) < 0) { - qError("tqReaderSeek failed ver:%" PRId64 ", %s", pOffset->version + 1, id); + if (pReaderAPI->tqReaderSeek(pInfo->tqReader, pOffset->version, id) < 0) { + qError("tqReaderSeek failed ver:%" PRId64 ", %s", pOffset->version, id); return -1; } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { @@ -1202,7 +1202,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id); STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo; - if (pAPI->snapshotFn.createSnapshot(sContext, pOffset->uid) != 0) { + if (pAPI->snapshotFn.setForSnapShot(sContext, pOffset->uid) != 0) { qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; @@ -1239,7 +1239,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; - if (pTaskInfo->storageAPI.snapshotFn.createSnapshot(sContext, pOffset->uid) != 0) { + if (pTaskInfo->storageAPI.snapshotFn.setForSnapShot(sContext, pOffset->uid) != 0) { qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c3d5de572f..b3a9699718 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1644,12 +1644,13 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; - qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); - if (pAPI->tqReaderFn.tqReaderSeek(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { + int64_t validVer = pTaskInfo->streamInfo.snapshotVer + 1; + qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", validVer); + if (pAPI->tqReaderFn.tqReaderSeek(pInfo->tqReader, validVer, pTaskInfo->id.str) < 0) { return NULL; } - tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pTaskInfo->streamInfo.snapshotVer); + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, validVer); } if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) { @@ -1660,8 +1661,8 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { SSDataBlock* pRes = pAPI->tqReaderFn.tqGetResultBlock(pInfo->tqReader); struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader); - // curVersion move to next, so currentOffset = curVersion - 1 - tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pWalReader->curVersion - 1); + // curVersion move to next + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pWalReader->curVersion); if (hasResult) { qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows, @@ -2182,7 +2183,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { STqOffsetVal offset = {0}; if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); - tqOffsetResetToLog(&offset, pInfo->sContext->snapVersion); + tqOffsetResetToLog(&offset, pInfo->sContext->snapVersion + 1); } else { tqOffsetResetToData(&offset, mtInfo.uid, INT64_MIN); qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 786f48ce88..a839d6cbd8 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -135,8 +135,8 @@ void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal* pOffset){ int64_t firstVer = walGetFirstVer((pWalReader)->pWal); taosThreadMutexUnlock(&pWalReader->pWal->mutex); - if (pOffset->version + 1 < firstVer){ - pOffset->version = firstVer - 1; + if (pOffset->version < firstVer){ + pOffset->version = firstVer; } } From 7b1716807bb31a1b856409716dce9b7270aff5a7 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 14 Jul 2023 17:16:43 +0800 Subject: [PATCH 029/100] test:add testcase of rolling upgdade --- tests/system-test/6-cluster/5dnode3mnodeRoll.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/system-test/6-cluster/5dnode3mnodeRoll.py b/tests/system-test/6-cluster/5dnode3mnodeRoll.py index 1b86c16f51..a9d4a964e7 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRoll.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRoll.py @@ -27,7 +27,7 @@ import threading import time import json -BASEVERSION = "3.0.5.0" +BASEVERSION = "3.0.7.0" class TDTestCase: @@ -243,12 +243,14 @@ class TDTestCase: os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y -k 10 -z 5 ") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql") - self.buildTaosd(bPath) + # self.buildTaosd(bPath) threads=[] threads.append(threading.Thread(target=self.insertAllData, args=(cPath_temp,dbname,tableNumbers1,recordNumbers1))) for tr in threads: tr.start() + # when inserting data porcess has been started up ,we can upgrade taosd + sleep(5) tdLog.printNoPrefix("==========step2:start to rolling upgdade ") for i in range(dnodeNumbers): tdDnodes[i].running = 1 @@ -258,7 +260,7 @@ class TDTestCase: for tr in threads: tr.join() - # waiting 10s for taosd cluster ready + # wait 10s for taosd cluster ready sleep(10) tdsql=tdCom.newTdSql() print(tdsql) From c5ac704b790c4224b1ea4b137594be4b315305a5 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 14 Jul 2023 17:17:37 +0800 Subject: [PATCH 030/100] test:add testcase of rolling upgdade --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index d775201be8..b929a452f0 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -446,7 +446,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 #,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 ,,n,system-test,python3 ./test.py -f 6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py -N 6 -M 3 -#,,n,system-test,python ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1 +,,n,system-test,python ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 #,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 From ea6e79f4eab9bcb7ac0050c17db20d06f6a1dd87 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 14 Jul 2023 17:18:06 +0800 Subject: [PATCH 031/100] test: modify loop = 60 --- tests/system-test/0-others/timeRangeWise.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/system-test/0-others/timeRangeWise.py b/tests/system-test/0-others/timeRangeWise.py index d558b1d693..a49457c8e9 100644 --- a/tests/system-test/0-others/timeRangeWise.py +++ b/tests/system-test/0-others/timeRangeWise.py @@ -217,7 +217,7 @@ class TDTestCase: tdSql.init(conn.cursor(), True) # check query result same - def queryDouble(self, sql): + def queryDoubleImpl(self, sql): # sql sql1 = sql.replace('@db_name', self.db1) tdLog.info(sql1) @@ -237,7 +237,7 @@ class TDTestCase: rowlen2 = len(res2) if rowlen1 != rowlen2: - tdLog.exit(f"rowlen1={rowlen1} rowlen2={rowlen2} both not equal.") + tdLog.info(f"check error. rowlen1={rowlen1} rowlen2={rowlen2} both not equal.") return False for i in range(rowlen1): @@ -246,7 +246,7 @@ class TDTestCase: collen1 = len(row1) collen2 = len(row2) if collen1 != collen2: - tdLog.exit(f"collen1={collen1} collen2={collen2} both not equal.") + tdLog.info(f"checkerror. collen1={collen1} collen2={collen2} both not equal.") return False for j in range(collen1): if row1[j] != row2[j]: @@ -261,6 +261,17 @@ class TDTestCase: return True + # check query result same + def queryDouble(self, sql, tryCount=60, gap=1): + for i in range(tryCount): + if self.queryDoubleImpl(sql): + return True + # error + tdLog.info(f"queryDouble return false, try loop={i}") + time.sleep(gap) + + tdLog.exit(f"queryDouble try {tryCount} times, but all failed.") + return False # check result def checkResult(self): From faf6a11c8aa943c2ac144940ff659fd3e86e37ee Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 14 Jul 2023 17:21:50 +0800 Subject: [PATCH 032/100] test:add to ci --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 85582e68c4..ee0550a77c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -160,6 +160,7 @@ ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py ,,n,system-test,python3 ./test.py -f 0-others/splitVGroup.py -N 5 +,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py From fc47fa6777b13ef4e427c48384b8e90f3e4456d1 Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Fri, 14 Jul 2023 18:28:19 +0800 Subject: [PATCH 033/100] Update index.md --- docs/zh/14-reference/12-config/index.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index d57ee02868..a4d6c9910c 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -726,6 +726,15 @@ charset 的有效值是 UTF-8。 | 取值范围 | 0: 不改变；1：改变 | | 缺省值 | 0 | +### tmqMaxTopicNum + +| 属性 | 说明 | +| -------- | ------------------ | +| 适用范围 | 仅服务端适用 | +| 含义 | 订阅最多可建立的 topic 数量 | +| 取值范围 | 1-10000| +| 缺省值 | 20 | + ## 压缩参数 ### compressMsgSize From 54ad70ea903b6bb1c54c1f4ce48cb7727b841fa3 Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Fri, 14 Jul 2023 18:30:15 +0800 Subject: [PATCH 034/100] Update index.md --- docs/en/14-reference/12-config/index.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 7522744469..2d213feceb 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -722,6 +722,15 @@ The charset that takes effect is UTF-8. | Value Range | 0: not change; 1: change by modification | | Default Value | 0 | +### tmqMaxTopicNum + +| Attribute | Description | +| -------- | ------------------ | +| Applicable | Server Only | +| Meaning | The max num of topics | +| Value Range | 1-10000| +| Default Value | 20 | + ## 3.0 Parameters | # | **Parameter** | **Applicable to 2.x ** | **Applicable to 3.0 ** | Current behavior in 3.0 | From d88946d1981bbeafc803f18816a1f2d5f5d557cb Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 14 Jul 2023 19:04:31 +0800 Subject: [PATCH 035/100] main: release 3.0.7.1 docs --- cmake/cmake.version | 2 +- docs/en/28-releases/01-tdengine.md | 4 ++++ docs/zh/28-releases/01-tdengine.md | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index a87049fb8a..70dd4aeaa4 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.7.1.alpha") + SET(TD_VER_NUMBER "3.0.7.2.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index 83b0fe5ac4..d05bf1139c 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.7.1 + + + ## 3.0.7.0 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 67718d59bf..52bb9c87a0 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.7.1 + + + ## 3.0.7.0 From 4fdf9ae5ced5f8ea005cc03275b36410ffdcb40d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 15 Jul 2023 10:21:50 +0800 Subject: [PATCH 036/100] test: add performance test , sma need quick no sma four multiple --- tests/system-test/0-others/timeRangeWise.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/system-test/0-others/timeRangeWise.py b/tests/system-test/0-others/timeRangeWise.py index a49457c8e9..a7dc18aa82 100644 --- a/tests/system-test/0-others/timeRangeWise.py +++ b/tests/system-test/0-others/timeRangeWise.py @@ -83,7 +83,7 @@ class TDTestCase: value += f"{self.ts})" # move next 1s interval - self.ts += 100 + self.ts += 1 return value @@ -148,7 +148,7 @@ class TDTestCase: self.vgroups2 = 4 self.db1 = "db1" # no sma self.db2 = "db2" # have sma - self.smaClause = "interval(1h)" + self.smaClause = "interval(10s)" # total self.c1Cnt = 0 @@ -254,10 +254,11 @@ class TDTestCase: return False # warning performance - diff = (spend2 - spend1)*100/spend1 - tdLog.info("spend1=%.6fs spend2=%.6fs diff=%.1f%%"%(spend1, spend2, diff)) - if spend2 > spend1 and diff > 20: - tdLog.info("warning: the diff for performance after spliting is over 20%") + multiple = spend1/spend2 + tdLog.info("spend1=%.6fs spend2=%.6fs multiple=%.1f"%(spend1, spend2, multiple)) + if spend2 > spend1 and multiple < 4: + tdLog.info(f"performace not reached: multiple(spend1/spend)={multiple} require is >=4 ") + return False return True From ee7c8289aa5358fbf86609832e68c8dfcb3907c3 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 17 Jul 2023 09:02:30 +0800 Subject: [PATCH 037/100] fix: create sma index for us/ns database failed --- source/libs/parser/src/parTranslater.c | 3 +- tests/parallel_test/cases.task | 1 + tests/system-test/0-others/sma_index.py | 48 +++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/0-others/sma_index.py diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2ccbc0dfc4..fb02884842 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3409,7 +3409,8 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); } - if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) || + if ((pSliding->datum.i < + convertTimeFromPrecisionToUnit(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, pSliding->unit)) || (pInter->datum.i / pSliding->datum.i > INTERVAL_SLIDING_FACTOR)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL); } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 85582e68c4..b0e71b6fd5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -341,6 +341,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaBasic.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sma_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py diff --git a/tests/system-test/0-others/sma_index.py b/tests/system-test/0-others/sma_index.py new file mode 100644 index 0000000000..488342b603 --- /dev/null +++ b/tests/system-test/0-others/sma_index.py @@ -0,0 +1,48 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def create_databases(self): + tdSql.execute("create database db_ms precision 'ms'") + tdSql.execute("create database db_us precision 'us'") + tdSql.execute("create database db_ns precision 'ns'") + + def create_stables(self): + tdSql.execute("CREATE STABLE db_ms.`meters` (`ts` TIMESTAMP, `c0` INT, `c1` TINYINT, `c2` DOUBLE, `c3` VARCHAR(64), `c4` NCHAR(64)) TAGS (`cc` VARCHAR(16))") + tdSql.execute("CREATE STABLE db_us.`meters` (`ts` TIMESTAMP, `c0` INT, `c1` TINYINT, `c2` DOUBLE, `c3` VARCHAR(64), `c4` NCHAR(64)) TAGS (`cc` VARCHAR(16))") + tdSql.execute("CREATE STABLE db_ns.`meters` (`ts` TIMESTAMP, `c0` INT, `c1` TINYINT, `c2` DOUBLE, `c3` VARCHAR(64), `c4` NCHAR(64)) TAGS (`cc` VARCHAR(16))") + + def create_sma_index(self): + tdSql.execute("create sma index sma_index_ms on db_ms.meters function(max(c1), max(c2), min(c1)) interval(6m, 10s) sliding(6m)" ) + tdSql.execute("create sma index sma_index_us on db_us.meters function(max(c1), max(c2), min(c1)) interval(6m, 10s) sliding(6m)" ) + tdSql.execute("create sma index sma_index_ns on db_ns.meters function(max(c1), max(c2), min(c1)) interval(6m, 10s) sliding(6m)" ) + + def run(self): + tdSql.prepare() + self.create_databases() + self.create_stables() + self.create_sma_index() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From f2ee9922086812fc7fed4bacef6b79b67da95145 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Jul 2023 09:43:12 +0800 Subject: [PATCH 038/100] docs: update readme-cn.md (#22086) --- README-CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-CN.md b/README-CN.md index 1d96a42709..9f2912ec40 100644 --- a/README-CN.md +++ b/README-CN.md @@ -352,4 +352,4 @@ TDengine 提供了丰富的应用程序开发接口，其中包括 C/C++、Java # 加入技术交流群 -TDengine 官方社群「物联网大数据群」对外开放，欢迎您加入讨论。搜索微信号 "tdengine1"，加小 T 为好友，即可入群。 +TDengine 官方社群「物联网大数据群」对外开放，欢迎您加入讨论。搜索微信号 "tdengine"，加小 T 为好友，即可入群。 From 872222eb771f7e7665420b7964ee4fb170508adc Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 17 Jul 2023 19:52:07 +0800 Subject: [PATCH 039/100] fix:[TD-25222] error in drop topic if topic num is too long --- include/libs/nodes/cmdnodes.h | 6 +++--- source/libs/parser/src/parTranslater.c | 4 +--- tests/system-test/1-insert/drop.py | 6 ++++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 3a36601b11..40a7105042 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -358,7 +358,7 @@ typedef struct SRestoreComponentNodeStmt { typedef struct SCreateTopicStmt { ENodeType type; - char topicName[TSDB_TABLE_NAME_LEN]; + char topicName[TSDB_TOPIC_NAME_LEN]; char subDbName[TSDB_DB_NAME_LEN]; char subSTbName[TSDB_TABLE_NAME_LEN]; bool ignoreExists; @@ -369,13 +369,13 @@ typedef struct SCreateTopicStmt { typedef struct SDropTopicStmt { ENodeType type; - char topicName[TSDB_TABLE_NAME_LEN]; + char topicName[TSDB_TOPIC_NAME_LEN]; bool ignoreNotExists; } SDropTopicStmt; typedef struct SDropCGroupStmt { ENodeType type; - char topicName[TSDB_TABLE_NAME_LEN]; + char topicName[TSDB_TOPIC_NAME_LEN]; char cgroup[TSDB_CGROUP_LEN]; bool ignoreNotExists; } SDropCGroupStmt; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fb02884842..00fa7ee7e2 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6135,9 +6135,7 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt) { SMDropTopicReq dropReq = {0}; - SName name; - tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName)); - tNameGetFullDbName(&name, dropReq.name); + snprintf(dropReq.name, sizeof(dropReq.name), "%d.%s", pCxt->pParseCxt->acctId, pStmt->topicName); dropReq.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_TMQ_DROP_TOPIC, (FSerializeFunc)tSerializeSMDropTopicReq, &dropReq); diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index 9954b3557e..8775450ff0 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -129,6 +129,12 @@ class TDTestCase: tdSql.query(f'select * from information_schema.ins_topics where topic_name = "{topic_name}"') tdSql.checkEqual(tdSql.queryResult[0][3],f'create topic {topic_name} as select c0 from {self.dbname}.{stbname}') tdSql.execute(f'drop topic {topic_name}') + + #TD-25222 + long_topic_name="hhhhjjhhhhqwertyuiasdfghjklzxcvbnmhhhhjjhhhhqwertyuiasdfghjklzxcvbnmhhhhjjhhhhqwertyuiasdfghjklzxcvbnm" + tdSql.execute(f'create topic {long_topic_name} as select * from {self.dbname}.{stbname}') + tdSql.execute(f'drop topic {long_topic_name}') + tdSql.execute(f'drop database {self.dbname}') def drop_stream_check(self): From 8d8e12ee0d65f303eaa4b14282ff7005c2a3b66b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 17 Jul 2023 20:20:34 +0800 Subject: [PATCH 040/100] fix:add sdbFetchCancel to release hash node --- source/dnode/mnode/impl/src/mndStb.c | 5 +++++ source/dnode/mnode/impl/src/mndTopic.c | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 162e75d783..6969e4387f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1240,6 +1240,7 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; mError("topic:%s, create ast error", pTopic->name); sdbRelease(pSdb, pTopic); + sdbCancelFetch(pSdb, pIter); return -1; } @@ -1260,6 +1261,7 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, mError("topic:%s, check colId:%d conflicted", pTopic->name, pCol->colId); nodesDestroyNode(pAst); nodesDestroyList(pNodeList); + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pTopic); return -1; } @@ -2268,6 +2270,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) { if (pTopic->stbUid == suid) { sdbRelease(pSdb, pTopic); + sdbCancelFetch(pSdb, pIter); return -1; } } @@ -2282,6 +2285,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; mError("topic:%s, create ast error", pTopic->name); sdbRelease(pSdb, pTopic); + sdbCancelFetch(pSdb, pIter); return -1; } @@ -2295,6 +2299,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, sdbRelease(pSdb, pTopic); nodesDestroyNode(pAst); nodesDestroyList(pNodeList); + sdbCancelFetch(pSdb, pIter); return -1; } else { goto NEXT; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 485823edf3..dbcf1fa2ad 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -647,7 +647,6 @@ static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTo code = 0; _OVER: - mndTransDrop(pTrans); return code; } @@ -735,6 +734,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + mndReleaseTopic(pMnode, pTopic); return -1; } @@ -788,6 +788,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); sdbRelease(pSdb, pVgroup); + mndReleaseTopic(pMnode, pTopic); mndTransDrop(pTrans); return -1; } @@ -796,6 +797,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { int32_t code = mndDropTopic(pMnode, pTrans, pReq, pTopic); mndReleaseTopic(pMnode, pTopic); + mndTransDrop(pTrans); if (code != 0) { mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); @@ -999,6 +1001,7 @@ bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb) { if (pTopic->dbUid == pDb->uid) { sdbRelease(pSdb, pTopic); + sdbCancelFetch(pSdb, pIter); return true; } From da9e30434a0f15a896904c64715efbb3a81c2f3f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 17 Jul 2023 20:21:23 +0800 Subject: [PATCH 041/100] fix:add sdbFetchCancel to release hash node --- source/dnode/mnode/impl/src/mndCluster.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 1 + source/dnode/mnode/impl/src/mndIndex.c | 1 + source/dnode/mnode/impl/src/mndMain.c | 1 + source/dnode/mnode/impl/src/mndScheduler.c | 5 +++++ source/dnode/mnode/impl/src/mndStb.c | 5 ++++- source/dnode/mnode/impl/src/mndStream.c | 2 ++ source/dnode/mnode/impl/src/mndSubscribe.c | 3 +++ source/dnode/mnode/impl/src/mndTopic.c | 7 +++++++ source/dnode/mnode/impl/src/mndUser.c | 16 +++++++++++++--- source/dnode/mnode/impl/src/mndVgroup.c | 1 + 11 files changed, 39 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 4d05637a2b..67675b5400 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -77,7 +77,7 @@ static SClusterObj *mndAcquireCluster(SMnode *pMnode, void **ppIter) { if (pIter == NULL) break; *ppIter = pIter; - + sdbCancelFetch(pSdb, pIter); return pCluster; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index bb92bfb4c7..47029c8df1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -706,6 +706,7 @@ _OVER: } else { mndReleaseDnode(pMnode, pDnode); } + sdbCancelFetch(pSdb, pIter); mndTransDrop(pTrans); sdbFreeRaw(pRaw); return terrno; diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 2d2637b8ce..8f977dacb7 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -831,6 +831,7 @@ int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxOb if (pIdx->stbUid == pStb->uid && strcasecmp(pIdx->colName, tagName) == 0) { memcpy((char *)idx, (char *)pIdx, sizeof(SIdxObj)); sdbRelease(pSdb, pIdx); + sdbCancelFetch(pSdb, pIter); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 381b1e64ed..55cca5a30c 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -199,6 +199,7 @@ static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs) pGid->syncCanRead = 0; roleChanged = true; } + sdbCancelFetch(pSdb, pIter); break; } } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 9a611fe46a..b95f4d6a00 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -168,6 +168,7 @@ SSnodeObj* mndSchedFetchOneSnode(SMnode* pMnode) { void* pIter = NULL; // TODO random fetch pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void**)&pObj); + sdbCancelFetch(pMnode->pSdb, pIter); return pObj; } @@ -435,6 +436,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { terrno = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); + sdbCancelFetch(pSdb, pIter); return -1; } @@ -444,6 +446,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { if (mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); + sdbCancelFetch(pSdb, pIter); return -1; } @@ -453,6 +456,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { if (code != TSDB_CODE_SUCCESS) { terrno = code; qDestroyQueryPlan(pPlan); + sdbCancelFetch(pSdb, pIter); return -1; } } @@ -492,6 +496,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { if (code != TSDB_CODE_SUCCESS) { qDestroyQueryPlan(pPlan); + sdbCancelFetch(pSdb, pIter); return -1; } } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 6969e4387f..c1186d068f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -900,7 +900,6 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { SMsgHead *pHead = rpcMallocCont(contLen); if (pHead == NULL) { - sdbCancelFetch(pSdb, pVgroup); sdbRelease(pSdb, pVgroup); continue; } @@ -1289,6 +1288,7 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; mError("stream:%s, create ast error", pStream->name); sdbRelease(pSdb, pStream); + sdbCancelFetch(pSdb, pIter); return -1; } @@ -1308,6 +1308,7 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName nodesDestroyNode(pAst); nodesDestroyList(pNodeList); sdbRelease(pSdb, pStream); + sdbCancelFetch(pSdb, pIter); return -1; } mInfo("stream:%s, check colId:%d passed", pStream->name, pCol->colId); @@ -1337,6 +1338,7 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, terrno = TSDB_CODE_SDB_INVALID_DATA_CONTENT; mError("tsma:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d failed since parse AST err", pSma->name, stbFullName, suid, colId); + sdbCancelFetch(pSdb, pIter); return -1; } @@ -1357,6 +1359,7 @@ static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, nodesDestroyNode(pAst); nodesDestroyList(pNodeList); sdbRelease(pSdb, pSma); + sdbCancelFetch(pSdb, pIter); return -1; } mInfo("tsma:%s, check colId:%d passed", pSma->name, pCol->colId); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 63f49cfe2b..028c482e6c 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -705,12 +705,14 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { if (numOfStream > MND_STREAM_MAX_NUM) { mError("too many streams, no more than %d for each database", MND_STREAM_MAX_NUM); terrno = TSDB_CODE_MND_TOO_MANY_STREAMS; + sdbCancelFetch(pMnode->pSdb, pIter); goto _OVER; } if (pStream->targetStbUid == streamObj.targetStbUid) { mError("Cannot write the same stable as other stream:%s", pStream->name); terrno = TSDB_CODE_MND_INVALID_TARGET_TABLE; + sdbCancelFetch(pMnode->pSdb, pIter); goto _OVER; } } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index b2235c8b50..f51a61eda3 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -1104,6 +1104,7 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) if (taosHashGetSize(pSub->consumerHash) != 0) { sdbRelease(pSdb, pSub); terrno = TSDB_CODE_MND_IN_REBALANCE; + sdbCancelFetch(pSdb, pIter); return -1; } int32_t sz = taosArrayGetSize(pSub->unassignedVgs); @@ -1122,12 +1123,14 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); sdbRelease(pSdb, pSub); + sdbCancelFetch(pSdb, pIter); return -1; } } if (mndSetDropSubRedoLogs(pMnode, pTrans, pSub) < 0) { sdbRelease(pSdb, pSub); + sdbCancelFetch(pSdb, pIter); goto END; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index dbcf1fa2ad..85e6f1caf6 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -513,6 +513,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * tEncodeSize(tEncodeSTqCheckInfo, &info, len, code); if (code < 0) { sdbRelease(pSdb, pVgroup); + sdbCancelFetch(pSdb, pIter); goto _OUT; } void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); @@ -522,6 +523,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * if (tEncodeSTqCheckInfo(&encoder, &info) < 0) { taosMemoryFree(buf); sdbRelease(pSdb, pVgroup); + sdbCancelFetch(pSdb, pIter); goto _OUT; } tEncoderClear(&encoder); @@ -535,6 +537,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); sdbRelease(pSdb, pVgroup); + sdbCancelFetch(pSdb, pIter); goto _OUT; } buf = NULL; @@ -697,6 +700,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { if (strcmp(name, pTopic->name) == 0) { mndReleaseConsumer(pMnode, pConsumer); mndReleaseTopic(pMnode, pTopic); + sdbCancelFetch(pSdb, pIter); terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s", dropReq.name, pConsumer->consumerId, pConsumer->cgroup); @@ -710,6 +714,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { if (strcmp(name, pTopic->name) == 0) { mndReleaseConsumer(pMnode, pConsumer); mndReleaseTopic(pMnode, pTopic); + sdbCancelFetch(pSdb, pIter); terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)", dropReq.name, pConsumer->consumerId, pConsumer->cgroup); @@ -723,6 +728,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { if (strcmp(name, pTopic->name) == 0) { mndReleaseConsumer(pMnode, pConsumer); mndReleaseTopic(pMnode, pTopic); + sdbCancelFetch(pSdb, pIter); terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb remove)", dropReq.name, pConsumer->consumerId, pConsumer->cgroup); @@ -789,6 +795,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { taosMemoryFree(buf); sdbRelease(pSdb, pVgroup); mndReleaseTopic(pMnode, pTopic); + sdbCancelFetch(pSdb, pIter); mndTransDrop(pTrans); return -1; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 1fc2e42b8c..d3b2ef7344 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1444,7 +1444,10 @@ int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { if (pIter == NULL) break; code = -1; - if (mndUserDupObj(pUser, &newUser) != 0) break; + if (mndUserDupObj(pUser, &newUser) != 0) { + sdbCancelFetch(pSdb, pIter); + break; + } bool inRead = (taosHashGet(newUser.readDbs, db, len) != NULL); bool inWrite = (taosHashGet(newUser.writeDbs, db, len) != NULL); @@ -1453,7 +1456,10 @@ int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { (void)taosHashRemove(newUser.writeDbs, db, len); SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) break; + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + sdbCancelFetch(pSdb, pIter); + break; + } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); } @@ -1484,6 +1490,7 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { code = -1; if (mndUserDupObj(pUser, &newUser) != 0) { + sdbCancelFetch(pSdb, pIter); break; } @@ -1491,7 +1498,10 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { if (inTopic) { (void)taosHashRemove(newUser.topics, topic, len); SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) break; + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + sdbCancelFetch(pSdb, pIter); + break; + } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index a82e49f397..8b313695a1 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2591,6 +2591,7 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { pIter = sdbFetch(pMnode->pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; if (!mndIsDnodeOnline(pDnode, curMs)) { + sdbCancelFetch(pMnode->pSdb, pIter); terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE; mError("failed to balance vgroup since %s, dnode:%d", terrstr(), pDnode->id); sdbRelease(pMnode->pSdb, pDnode); From 9667e5df614c0ddee05b6bf4b5931a3991600fa9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 18 Jul 2023 01:40:49 +0000 Subject: [PATCH 042/100] remove fulltext --- docs/en/12-taos-sql/22-meta.md | 382 ++++++++++++++++---------------- docs/en/12-taos-sql/27-index.md | 8 +- docs/zh/12-taos-sql/22-meta.md | 290 ++++++++++++------------ docs/zh/12-taos-sql/27-index.md | 9 +- 4 files changed, 341 insertions(+), 348 deletions(-) diff --git a/docs/en/12-taos-sql/22-meta.md b/docs/en/12-taos-sql/22-meta.md index f165470d10..ae7b81864e 100644 --- a/docs/en/12-taos-sql/22-meta.md +++ b/docs/en/12-taos-sql/22-meta.md @@ -28,274 +28,274 @@ This document introduces the tables of INFORMATION_SCHEMA and their structure. Provides information about dnodes. Similar to SHOW DNODES. -| # | **Column** | **Data Type** | **Description** | -| --- | :------------: | ------------ | ------------------------- | -| 1 | vnodes | SMALLINT | Current number of vnodes on the dnode. It should be noted that `vnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 2 | support_vnodes | SMALLINT | Maximum number of vnodes on the dnode | -| 3 | status | BINARY(10) | Current status | -| 4 | note | BINARY(256) | Reason for going offline or other information | -| 5 | id | SMALLINT | Dnode ID | -| 6 | endpoint | BINARY(134) | Dnode endpoint | -| 7 | create | TIMESTAMP | Creation time | +| # | **Column** | **Data Type** | **Description** | +| --- | :------------: | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | vnodes | SMALLINT | Current number of vnodes on the dnode. It should be noted that `vnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 2 | support_vnodes | SMALLINT | Maximum number of vnodes on the dnode | +| 3 | status | BINARY(10) | Current status | +| 4 | note | BINARY(256) | Reason for going offline or other information | +| 5 | id | SMALLINT | Dnode ID | +| 6 | endpoint | BINARY(134) | Dnode endpoint | +| 7 | create | TIMESTAMP | Creation time | ## INS_MNODES Provides information about mnodes. Similar to SHOW MNODES. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | ------------------ | -| 1 | id | SMALLINT | Mnode ID | -| 2 | endpoint | BINARY(134) | Mnode endpoint | -| 3 | role | BINARY(10) | Current role | -| 4 | role_time | TIMESTAMP | Time at which the current role was assumed | -| 5 | create_time | TIMESTAMP | Creation time | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | ------------------------------------------ | +| 1 | id | SMALLINT | Mnode ID | +| 2 | endpoint | BINARY(134) | Mnode endpoint | +| 3 | role | BINARY(10) | Current role | +| 4 | role_time | TIMESTAMP | Time at which the current role was assumed | +| 5 | create_time | TIMESTAMP | Creation time | ## INS_QNODES Provides information about qnodes. Similar to SHOW QNODES. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | ------------ | -| 1 | id | SMALLINT | Qnode ID | -| 2 | endpoint | BINARY(134) | Qnode endpoint | -| 3 | create_time | TIMESTAMP | Creation time | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | --------------- | +| 1 | id | SMALLINT | Qnode ID | +| 2 | endpoint | BINARY(134) | Qnode endpoint | +| 3 | create_time | TIMESTAMP | Creation time | ## INS_CLUSTER Provides information about the cluster. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | ---------- | -| 1 | id | BIGINT | Cluster ID | -| 2 | name | BINARY(134) | Cluster name | -| 3 | create_time | TIMESTAMP | Creation time | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | --------------- | +| 1 | id | BIGINT | Cluster ID | +| 2 | name | BINARY(134) | Cluster name | +| 3 | create_time | TIMESTAMP | Creation time | ## INS_DATABASES Provides information about user-created databases. Similar to SHOW DATABASES. -| # | **Column** | **Data Type** | **Description** | -| --- | :------------------: | ---------------- | ------------------------------------------------ | -| 1| name| BINARY(32)| Database name | -| 2 | create_time | TIMESTAMP | Creation time | -| 3 | ntables | INT | Number of standard tables and subtables (not including supertables) | -| 4 | vgroups | INT | Number of vgroups. It should be noted that `vnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 6 | replica | INT | Number of replicas. It should be noted that `replica` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 7 | strict | BINARY(4) | Obsoleted | -| 8 | duration | INT | Duration for storage of single files. It should be noted that `duration` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 9 | keep | INT | Data retention period. It should be noted that `keep` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 10 | buffer | INT | Write cache size per vnode, in MB. It should be noted that `buffer` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 11 | pagesize | INT | Page size for vnode metadata storage engine, in KB. It should be noted that `pagesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 12 | pages | INT | Number of pages per vnode metadata storage engine. It should be noted that `pages` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 13 | minrows | INT | Maximum number of records per file block. It should be noted that `minrows` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 14 | maxrows | INT | Minimum number of records per file block. It should be noted that `maxrows` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 15 | comp | INT | Compression method. It should be noted that `comp` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 16 | precision | BINARY(2) | Time precision. It should be noted that `precision` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 17 | status | BINARY(10) | Current database status | -| 18 | retentions | BINARY (60) | Aggregation interval and retention period. It should be noted that `retentions` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 19 | single_stable | BOOL | Whether the database can contain multiple supertables. It should be noted that `single_stable` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 20 | cachemodel | BINARY(60) | Caching method for the newest data. It should be noted that `cachemodel` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 21 | cachesize | INT | Memory per vnode used for caching the newest data. It should be noted that `cachesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 22 | wal_level | INT | WAL level. It should be noted that `wal_level` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 23 | wal_fsync_period | INT | Interval at which WAL is written to disk. It should be noted that `wal_fsync_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 24 | wal_retention_period | INT | WAL retention period. It should be noted that `wal_retention_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 25 | wal_retention_size | INT | Maximum WAL size. It should be noted that `wal_retention_size` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 26 | wal_roll_period | INT | WAL rotation period. It should be noted that `wal_roll_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 27 | wal_segment_size | BIGINT | WAL file size. It should be noted that `wal_segment_size` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 28 | stt_trigger | SMALLINT | The threshold for number of files to trigger file merging. It should be noted that `stt_trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 29 | table_prefix | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_prefix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 30 | table_suffix | SMALLINT | The suffix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_suffix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 31 | tsdb_pagesize | INT | The page size for internal storage engine, its unit is KB. It should be noted that `tsdb_pagesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| # | **Column** | **Data Type** | **Description** | +| --- | :------------------: | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | name | BINARY(32) | Database name | +| 2 | create_time | TIMESTAMP | Creation time | +| 3 | ntables | INT | Number of standard tables and subtables (not including supertables) | +| 4 | vgroups | INT | Number of vgroups. It should be noted that `vnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 6 | replica | INT | Number of replicas. It should be noted that `replica` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 7 | strict | BINARY(4) | Obsoleted | +| 8 | duration | INT | Duration for storage of single files. It should be noted that `duration` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 9 | keep | INT | Data retention period. It should be noted that `keep` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 10 | buffer | INT | Write cache size per vnode, in MB. It should be noted that `buffer` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 11 | pagesize | INT | Page size for vnode metadata storage engine, in KB. It should be noted that `pagesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 12 | pages | INT | Number of pages per vnode metadata storage engine. It should be noted that `pages` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 13 | minrows | INT | Maximum number of records per file block. It should be noted that `minrows` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 14 | maxrows | INT | Minimum number of records per file block. It should be noted that `maxrows` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 15 | comp | INT | Compression method. It should be noted that `comp` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 16 | precision | BINARY(2) | Time precision. It should be noted that `precision` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 17 | status | BINARY(10) | Current database status | +| 18 | retentions | BINARY (60) | Aggregation interval and retention period. It should be noted that `retentions` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 19 | single_stable | BOOL | Whether the database can contain multiple supertables. It should be noted that `single_stable` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 20 | cachemodel | BINARY(60) | Caching method for the newest data. It should be noted that `cachemodel` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 21 | cachesize | INT | Memory per vnode used for caching the newest data. It should be noted that `cachesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 22 | wal_level | INT | WAL level. It should be noted that `wal_level` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 23 | wal_fsync_period | INT | Interval at which WAL is written to disk. It should be noted that `wal_fsync_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 24 | wal_retention_period | INT | WAL retention period. It should be noted that `wal_retention_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 25 | wal_retention_size | INT | Maximum WAL size. It should be noted that `wal_retention_size` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 26 | wal_roll_period | INT | WAL rotation period. It should be noted that `wal_roll_period` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 27 | wal_segment_size | BIGINT | WAL file size. It should be noted that `wal_segment_size` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 28 | stt_trigger | SMALLINT | The threshold for number of files to trigger file merging. It should be noted that `stt_trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 29 | table_prefix | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_prefix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 30 | table_suffix | SMALLINT | The suffix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_suffix` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 31 | tsdb_pagesize | INT | The page size for internal storage engine, its unit is KB. It should be noted that `tsdb_pagesize` is a TDengine keyword and needs to be escaped with ` when used as a column name. | ## INS_FUNCTIONS Provides information about user-defined functions. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | -------------- | -| 1 | name | BINARY(64) | Function name | -| 2 | comment | BINARY(255) | Function description. It should be noted that `comment` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 3 | aggregate | INT | Whether the UDF is an aggregate function. It should be noted that `aggregate` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 4 | output_type | BINARY(31) | Output data type | -| 5 | create_time | TIMESTAMP | Creation time | -| 6 | code_len | INT | Length of the source code | -| 7 | bufsize | INT | Buffer size | -| 8 | func_language | BINARY(31) | UDF programming language | -| 9 | func_body | BINARY(16384) | UDF function body | -| 10 | func_version | INT | UDF function version. starting from 0. Increasing by 1 each time it is updated| +| # | **Column** | **Data Type** | **Description** | +| --- | :-----------: | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | name | BINARY(64) | Function name | +| 2 | comment | BINARY(255) | Function description. It should be noted that `comment` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 3 | aggregate | INT | Whether the UDF is an aggregate function. It should be noted that `aggregate` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 4 | output_type | BINARY(31) | Output data type | +| 5 | create_time | TIMESTAMP | Creation time | +| 6 | code_len | INT | Length of the source code | +| 7 | bufsize | INT | Buffer size | +| 8 | func_language | BINARY(31) | UDF programming language | +| 9 | func_body | BINARY(16384) | UDF function body | +| 10 | func_version | INT | UDF function version. starting from 0. Increasing by 1 each time it is updated | ## INS_INDEXES Provides information about user-created indices. Similar to SHOW INDEX. -| # | **Column** | **Data Type** | **Description** | -| --- | :--------------: | ------------ | ---------------------------------------------------------------------------------- | -| 1 | db_name | BINARY(32) | Database containing the table with the specified index | -| 2 | table_name | BINARY(192) | Table containing the specified index | -| 3 | index_name | BINARY(192) | Index name | -| 4 | db_name | BINARY(64) | Index column | -| 5 | index_type | BINARY(10) | SMA or FULLTEXT index | -| 6 | index_extensions | BINARY(256) | Other information For SMA indices, this shows a list of functions. For FULLTEXT indices, this is null. | +| # | **Column** | **Data Type** | **Description** | +| --- | :--------------: | ------------- | --------------------------------------------------------------------- | +| 1 | db_name | BINARY(32) | Database containing the table with the specified index | +| 2 | table_name | BINARY(192) | Table containing the specified index | +| 3 | index_name | BINARY(192) | Index name | +| 4 | db_name | BINARY(64) | Index column | +| 5 | index_type | BINARY(10) | SMA or tag index | +| 6 | index_extensions | BINARY(256) | Other information For SMA/tag indices, this shows a list of functions | ## INS_STABLES Provides information about supertables. -| # | **Column** | **Data Type** | **Description** | -| --- | :-----------: | ------------ | ------------------------ | -| 1 | stable_name | BINARY(192) | Supertable name | -| 2 | db_name | BINARY(64) | All databases in the supertable | -| 3 | create_time | TIMESTAMP | Creation time | -| 4 | columns | INT | Number of columns | -| 5 | tags | INT | Number of tags. It should be noted that `tags` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 6 | last_update | TIMESTAMP | Last updated time | -| 7 | table_comment | BINARY(1024) | Table description | -| 8 | watermark | BINARY(64) | Window closing time. It should be noted that `watermark` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 9 | max_delay | BINARY(64) | Maximum delay for pushing stream processing results. It should be noted that `max_delay` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 10 | rollup | BINARY(128) | Rollup aggregate function. It should be noted that `rollup` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| # | **Column** | **Data Type** | **Description** | +| --- | :-----------: | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | stable_name | BINARY(192) | Supertable name | +| 2 | db_name | BINARY(64) | All databases in the supertable | +| 3 | create_time | TIMESTAMP | Creation time | +| 4 | columns | INT | Number of columns | +| 5 | tags | INT | Number of tags. It should be noted that `tags` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 6 | last_update | TIMESTAMP | Last updated time | +| 7 | table_comment | BINARY(1024) | Table description | +| 8 | watermark | BINARY(64) | Window closing time. It should be noted that `watermark` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 9 | max_delay | BINARY(64) | Maximum delay for pushing stream processing results. It should be noted that `max_delay` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 10 | rollup | BINARY(128) | Rollup aggregate function. It should be noted that `rollup` is a TDengine keyword and needs to be escaped with ` when used as a column name. | ## INS_TABLES Provides information about standard tables and subtables. -| # | **Column** | **Data Type** | **Description** | -| --- | :-----------: | ------------ | ---------------- | -| 1 | table_name | BINARY(192) | Table name | -| 2 | db_name | BINARY(64) | Database name | -| 3 | create_time | TIMESTAMP | Creation time | -| 4 | columns | INT | Number of columns | -| 5 | stable_name | BINARY(192) | Supertable name | -| 6 | uid | BIGINT | Table ID | -| 7 | vgroup_id | INT | Vgroup ID | -| 8 | ttl | INT | Table time-to-live. It should be noted that `ttl` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 9 | table_comment | BINARY(1024) | Table description | -| 10 | type | BINARY(20) | Table type | +| # | **Column** | **Data Type** | **Description** | +| --- | :-----------: | ------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| 1 | table_name | BINARY(192) | Table name | +| 2 | db_name | BINARY(64) | Database name | +| 3 | create_time | TIMESTAMP | Creation time | +| 4 | columns | INT | Number of columns | +| 5 | stable_name | BINARY(192) | Supertable name | +| 6 | uid | BIGINT | Table ID | +| 7 | vgroup_id | INT | Vgroup ID | +| 8 | ttl | INT | Table time-to-live. It should be noted that `ttl` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 9 | table_comment | BINARY(1024) | Table description | +| 10 | type | BINARY(20) | Table type | ## INS_TAGS -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------- | ---------------------- | -| 1 | table_name | BINARY(192) | Table name | -| 2 | db_name | BINARY(64) | Database name | -| 3 | stable_name | BINARY(192) | Supertable name | -| 4 | tag_name | BINARY(64) | Tag name | -| 5 | tag_type | BINARY(64) | Tag type | -| 6 | tag_value | BINARY(16384) | Tag value | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | --------------- | +| 1 | table_name | BINARY(192) | Table name | +| 2 | db_name | BINARY(64) | Database name | +| 3 | stable_name | BINARY(192) | Supertable name | +| 4 | tag_name | BINARY(64) | Tag name | +| 5 | tag_type | BINARY(64) | Tag type | +| 6 | tag_value | BINARY(16384) | Tag value | ## INS_COLUMNS -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------- | ---------------------- | -| 1 | table_name | BINARY(192) | Table name | -| 2 | db_name | BINARY(64) | Database name | -| 3 | table_type | BINARY(21) | Table type | -| 4 | col_name | BINARY(64) | Column name | -| 5 | col_type | BINARY(32) | Column type | -| 6 | col_length | INT | Column length | -| 7 | col_precision | INT | Column precision | -| 8 | col_scale | INT | Column scale | -| 9 | col_nullable | INT | Column nullable | +| # | **Column** | **Data Type** | **Description** | +| --- | :-----------: | ------------- | ---------------- | +| 1 | table_name | BINARY(192) | Table name | +| 2 | db_name | BINARY(64) | Database name | +| 3 | table_type | BINARY(21) | Table type | +| 4 | col_name | BINARY(64) | Column name | +| 5 | col_type | BINARY(32) | Column type | +| 6 | col_length | INT | Column length | +| 7 | col_precision | INT | Column precision | +| 8 | col_scale | INT | Column scale | +| 9 | col_nullable | INT | Column nullable | ## INS_USERS Provides information about TDengine users. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | -------- | -| 1 | user_name | BINARY(23) | User name | -| 2 | privilege | BINARY(256) | User permissions | -| 3 | create_time | TIMESTAMP | Creation time | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | ---------------- | +| 1 | user_name | BINARY(23) | User name | +| 2 | privilege | BINARY(256) | User permissions | +| 3 | create_time | TIMESTAMP | Creation time | ## INS_GRANTS Provides information about TDengine Enterprise Edition permissions. -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | -------------------------------------------------- | -| 1 | version | BINARY(9) | Whether the deployment is a licensed or trial version | -| 2 | cpu_cores | BINARY(9) | CPU cores included in license | -| 3 | dnodes | BINARY(10) | Dnodes included in license. It should be noted that `dnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 4 | streams | BINARY(10) | Streams included in license. It should be noted that `streams` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 5 | users | BINARY(10) | Users included in license. It should be noted that `users` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 6 | accounts | BINARY(10) | Accounts included in license. It should be noted that `accounts` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 7 | storage | BINARY(21) | Storage space included in license. It should be noted that `storage` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 8 | connections | BINARY(21) | Client connections included in license. It should be noted that `connections` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 9 | databases | BINARY(11) | Databases included in license. It should be noted that `databases` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 10 | speed | BINARY(9) | Write speed specified in license (data points per second) | -| 11 | querytime | BINARY(9) | Total query time specified in license | -| 12 | timeseries | BINARY(21) | Number of metrics included in license | -| 13 | expired | BINARY(5) | Whether the license has expired | -| 14 | expire_time | BINARY(19) | When the trial period expires | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | version | BINARY(9) | Whether the deployment is a licensed or trial version | +| 2 | cpu_cores | BINARY(9) | CPU cores included in license | +| 3 | dnodes | BINARY(10) | Dnodes included in license. It should be noted that `dnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 4 | streams | BINARY(10) | Streams included in license. It should be noted that `streams` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 5 | users | BINARY(10) | Users included in license. It should be noted that `users` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 6 | accounts | BINARY(10) | Accounts included in license. It should be noted that `accounts` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 7 | storage | BINARY(21) | Storage space included in license. It should be noted that `storage` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 8 | connections | BINARY(21) | Client connections included in license. It should be noted that `connections` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 9 | databases | BINARY(11) | Databases included in license. It should be noted that `databases` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 10 | speed | BINARY(9) | Write speed specified in license (data points per second) | +| 11 | querytime | BINARY(9) | Total query time specified in license | +| 12 | timeseries | BINARY(21) | Number of metrics included in license | +| 13 | expired | BINARY(5) | Whether the license has expired | +| 14 | expire_time | BINARY(19) | When the trial period expires | ## INS_VGROUPS Provides information about vgroups. -| # | **Column** | **Data Type** | **Description** | -| --- | :-------: | ------------ | ------------------------------------------------------ | -| 1 | vgroup_id | INT | Vgroup ID | -| 2 | db_name | BINARY(32) | Database name | -| 3 | tables | INT | Tables in vgroup. It should be noted that `tables` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 4 | status | BINARY(10) | Vgroup status | -| 5 | v1_dnode | INT | Dnode ID of first vgroup member | -| 6 | v1_status | BINARY(10) | Status of first vgroup member | -| 7 | v2_dnode | INT | Dnode ID of second vgroup member | -| 8 | v2_status | BINARY(10) | Status of second vgroup member | -| 9 | v3_dnode | INT | Dnode ID of third vgroup member | -| 10 | v3_status | BINARY(10) | Status of third vgroup member | -| 11 | nfiles | INT | Number of data and metadata files in the vgroup | -| 12 | file_size | INT | Size of the data and metadata files in the vgroup | -| 13 | tsma | TINYINT | Whether time-range-wise SMA is enabled. 1 means enabled; 0 means disabled. | +| # | **Column** | **Data Type** | **Description** | +| --- | :--------: | ------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| 1 | vgroup_id | INT | Vgroup ID | +| 2 | db_name | BINARY(32) | Database name | +| 3 | tables | INT | Tables in vgroup. It should be noted that `tables` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 4 | status | BINARY(10) | Vgroup status | +| 5 | v1_dnode | INT | Dnode ID of first vgroup member | +| 6 | v1_status | BINARY(10) | Status of first vgroup member | +| 7 | v2_dnode | INT | Dnode ID of second vgroup member | +| 8 | v2_status | BINARY(10) | Status of second vgroup member | +| 9 | v3_dnode | INT | Dnode ID of third vgroup member | +| 10 | v3_status | BINARY(10) | Status of third vgroup member | +| 11 | nfiles | INT | Number of data and metadata files in the vgroup | +| 12 | file_size | INT | Size of the data and metadata files in the vgroup | +| 13 | tsma | TINYINT | Whether time-range-wise SMA is enabled. 1 means enabled; 0 means disabled. | ## INS_CONFIGS Provides system configuration information. -| # | **Column** | **Data Type** | **Description** | -| --- | :------: | ------------ | ------------ | -| 1 | name | BINARY(32) | Parameter | -| 2 | value | BINARY(64) | Value. It should be noted that `value` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| # | **Column** | **Data Type** | **Description** | +| --- | :--------: | ------------- | ----------------------------------------------------------------------------------------------------------------------- | +| 1 | name | BINARY(32) | Parameter | +| 2 | value | BINARY(64) | Value. It should be noted that `value` is a TDengine keyword and needs to be escaped with ` when used as a column name. | ## INS_DNODE_VARIABLES Provides dnode configuration information. -| # | **Column** | **Data Type** | **Description** | -| --- | :------: | ------------ | ------------ | -| 1 | dnode_id | INT | Dnode ID | -| 2 | name | BINARY(32) | Parameter | -| 3 | value | BINARY(64) | Value. It should be noted that `value` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| # | **Column** | **Data Type** | **Description** | +| --- | :--------: | ------------- | ----------------------------------------------------------------------------------------------------------------------- | +| 1 | dnode_id | INT | Dnode ID | +| 2 | name | BINARY(32) | Parameter | +| 3 | value | BINARY(64) | Value. It should be noted that `value` is a TDengine keyword and needs to be escaped with ` when used as a column name. | ## INS_TOPICS -| # | **Column** | **Data Type** | **Description** | -| --- | :---------: | ------------ | ------------------------------ | -| 1 | topic_name | BINARY(192) | Topic name | -| 2 | db_name | BINARY(64) | Database for the topic | -| 3 | create_time | TIMESTAMP | Creation time | -| 4 | sql | BINARY(1024) | SQL statement used to create the topic | +| # | **Column** | **Data Type** | **Description** | +| --- | :---------: | ------------- | -------------------------------------- | +| 1 | topic_name | BINARY(192) | Topic name | +| 2 | db_name | BINARY(64) | Database for the topic | +| 3 | create_time | TIMESTAMP | Creation time | +| 4 | sql | BINARY(1024) | SQL statement used to create the topic | ## INS_SUBSCRIPTIONS -| # | **Column** | **Data Type** | **Description** | -| --- | :------------: | ------------ | ------------------------ | -| 1 | topic_name | BINARY(204) | Subscribed topic | -| 2 | consumer_group | BINARY(193) | Subscribed consumer group | -| 3 | vgroup_id | INT | Vgroup ID for the consumer | -| 4 | consumer_id | BIGINT | Consumer ID | -| 5 | offset | BINARY(64) | Consumption progress | -| 6 | rows | BIGINT | Number of consumption items | +| # | **Column** | **Data Type** | **Description** | +| --- | :------------: | ------------- | --------------------------- | +| 1 | topic_name | BINARY(204) | Subscribed topic | +| 2 | consumer_group | BINARY(193) | Subscribed consumer group | +| 3 | vgroup_id | INT | Vgroup ID for the consumer | +| 4 | consumer_id | BIGINT | Consumer ID | +| 5 | offset | BINARY(64) | Consumption progress | +| 6 | rows | BIGINT | Number of consumption items | ## INS_STREAMS -| # | **Column** | **Data Type** | **Description** | -| --- | :----------: | ------------ | --------------------------------------- | -| 1 | stream_name | BINARY(64) | Stream name | -| 2 | create_time | TIMESTAMP | Creation time | -| 3 | sql | BINARY(1024) | SQL statement used to create the stream | -| 4 | status | BINARY(20) | Current status | -| 5 | source_db | BINARY(64) | Source database | -| 6 | target_db | BINARY(64) | Target database | -| 7 | target_table | BINARY(192) | Target table | -| 8 | watermark | BIGINT | Watermark (see stream processing documentation). It should be noted that `watermark` is a TDengine keyword and needs to be escaped with ` when used as a column name. | -| 9 | trigger | INT | Method of triggering the result push (see stream processing documentation). It should be noted that `trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| # | **Column** | **Data Type** | **Description** | +| --- | :----------: | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | stream_name | BINARY(64) | Stream name | +| 2 | create_time | TIMESTAMP | Creation time | +| 3 | sql | BINARY(1024) | SQL statement used to create the stream | +| 4 | status | BINARY(20) | Current status | +| 5 | source_db | BINARY(64) | Source database | +| 6 | target_db | BINARY(64) | Target database | +| 7 | target_table | BINARY(192) | Target table | +| 8 | watermark | BIGINT | Watermark (see stream processing documentation). It should be noted that `watermark` is a TDengine keyword and needs to be escaped with ` when used as a column name. | +| 9 | trigger | INT | Method of triggering the result push (see stream processing documentation). It should be noted that `trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name. | diff --git a/docs/en/12-taos-sql/27-index.md b/docs/en/12-taos-sql/27-index.md index e3eb69bdb3..33e511b8a0 100644 --- a/docs/en/12-taos-sql/27-index.md +++ b/docs/en/12-taos-sql/27-index.md @@ -4,12 +4,12 @@ sidebar_label: Indexing description: This document describes the SQL statements related to indexing in TDengine. --- -TDengine supports SMA and FULLTEXT indexing. +TDengine supports SMA and tag indexing. ## Create an Index ```sql -CREATE FULLTEXT INDEX index_name ON tb_name (col_name [, col_name] ...) +CREATE INDEX index_name ON tb_name (col_name [, col_name] ...) CREATE SMA INDEX index_name ON tb_name index_option @@ -46,10 +46,6 @@ SELECT _wstart,_wend,_wduration,max(c2),min(c1) FROM st1 INTERVAL(5m,10s) SLIDIN ALTER LOCAL 'querySmaOptimize' '0'; ``` -### FULLTEXT Indexing - -Creates a text index for the specified column. FULLTEXT indexing improves performance for queries with text filtering. The index_option syntax is not supported for FULLTEXT indexing. FULLTEXT indexing is supported for JSON tag columns only. Multiple columns cannot be indexed together. However, separate indices can be created for each column. - ## Delete an Index ```sql diff --git a/docs/zh/12-taos-sql/22-meta.md b/docs/zh/12-taos-sql/22-meta.md index fe8d6d4c69..5df79204d0 100644 --- a/docs/zh/12-taos-sql/22-meta.md +++ b/docs/zh/12-taos-sql/22-meta.md @@ -28,15 +28,15 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 提供 dnode 的相关信息。也可以使用 SHOW DNODES 来查询这些信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :------------: | ------------ | ------------------------- | +| # | **列名** | **数据类型** | **说明** | +| --- | :------------: | ------------ | ----------------------------------------------------------------------------------------------------- | | 1 | vnodes | SMALLINT | dnode 中的实际 vnode 个数。需要注意，`vnodes` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 2 | support_vnodes | SMALLINT | 最多支持的 vnode 个数 | -| 3 | status | BINARY(10) | 当前状态 | -| 4 | note | BINARY(256) | 离线原因等信息 | -| 5 | id | SMALLINT | dnode id | -| 6 | endpoint | BINARY(134) | dnode 的地址 | -| 7 | create | TIMESTAMP | 创建时间 | +| 2 | support_vnodes | SMALLINT | 最多支持的 vnode 个数 | +| 3 | status | BINARY(10) | 当前状态 | +| 4 | note | BINARY(256) | 离线原因等信息 | +| 5 | id | SMALLINT | dnode id | +| 6 | endpoint | BINARY(134) | dnode 的地址 | +| 7 | create | TIMESTAMP | 创建时间 | ## INS_MNODES @@ -74,103 +74,103 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 提供用户创建的数据库对象的相关信息。也可以使用 SHOW DATABASES 来查询这些信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :------------------: | ---------------- | ------------------------------------------------ | -| 1 | name | BINARY(32) | 数据库名 | -| 2 | create_time | TIMESTAMP | 创建时间 | -| 3 | ntables | INT | 数据库中表的数量，包含子表和普通表但不包含超级表 | -| 4 | vgroups | INT | 数据库中有多少个 vgroup。需要注意，`vgroups` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 6 | replica | INT | 副本数。需要注意，`replica` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 7 | strict | BINARY(4) | 废弃参数 | -| 8 | duration | INT | 单文件存储数据的时间跨度。需要注意，`duration` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | keep | INT | 数据保留时长。需要注意，`keep` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 10 | buffer | INT | 每个 vnode 写缓存的内存块大小，单位 MB。需要注意，`buffer` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 11 | pagesize | INT | 每个 VNODE 中元数据存储引擎的页大小，单位为 KB。需要注意，`pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 12 | pages | INT | 每个 vnode 元数据存储引擎的缓存页个数。需要注意，`pages` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 13 | minrows | INT | 文件块中记录的最大条数。需要注意，`minrows` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 14 | maxrows | INT | 文件块中记录的最小条数。需要注意，`maxrows` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 15 | comp | INT | 数据压缩方式。需要注意，`comp` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 16 | precision | BINARY(2) | 时间分辨率。需要注意，`precision` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 17 | status | BINARY(10) | 数据库状态 | -| 18 | retentions | BINARY (60) | 数据的聚合周期和保存时长。需要注意，`retentions` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 19 | single_stable | BOOL | 表示此数据库中是否只可以创建一个超级表。需要注意，`single_stable` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 20 | cachemodel | BINARY(60) | 表示是否在内存中缓存子表的最近数据。需要注意，`cachemodel` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 21 | cachesize | INT | 表示每个 vnode 中用于缓存子表最近数据的内存大小。需要注意，`cachesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 22 | wal_level | INT | WAL 级别。需要注意，`wal_level` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 23 | wal_fsync_period | INT | 数据落盘周期。需要注意，`wal_fsync_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 24 | wal_retention_period | INT | WAL 的保存时长。需要注意，`wal_retention_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 25 | wal_retention_size | INT | WAL 的保存上限。需要注意，`wal_retention_size` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 26 | wal_roll_period | INT | wal 文件切换时长。需要注意，`wal_roll_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 27 | wal_segment_size | BIGINT | wal 单个文件大小。需要注意，`wal_segment_size` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 28 | stt_trigger | SMALLINT | 触发文件合并的落盘文件的个数。需要注意，`stt_trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 29 | table_prefix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。需要注意，`table_prefix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 30 | table_suffix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。需要注意，`table_suffix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 31 | tsdb_pagesize | INT | 时序数据存储引擎中的页大小。需要注意，`tsdb_pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| # | **列名** | **数据类型** | **说明** | +| --- | :------------------: | ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | name | BINARY(32) | 数据库名 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | ntables | INT | 数据库中表的数量，包含子表和普通表但不包含超级表 | +| 4 | vgroups | INT | 数据库中有多少个 vgroup。需要注意，`vgroups` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 6 | replica | INT | 副本数。需要注意，`replica` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 7 | strict | BINARY(4) | 废弃参数 | +| 8 | duration | INT | 单文件存储数据的时间跨度。需要注意，`duration` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 9 | keep | INT | 数据保留时长。需要注意，`keep` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 10 | buffer | INT | 每个 vnode 写缓存的内存块大小，单位 MB。需要注意，`buffer` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 11 | pagesize | INT | 每个 VNODE 中元数据存储引擎的页大小，单位为 KB。需要注意，`pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 12 | pages | INT | 每个 vnode 元数据存储引擎的缓存页个数。需要注意，`pages` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 13 | minrows | INT | 文件块中记录的最大条数。需要注意，`minrows` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 14 | maxrows | INT | 文件块中记录的最小条数。需要注意，`maxrows` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 15 | comp | INT | 数据压缩方式。需要注意，`comp` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 16 | precision | BINARY(2) | 时间分辨率。需要注意，`precision` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 17 | status | BINARY(10) | 数据库状态 | +| 18 | retentions | BINARY (60) | 数据的聚合周期和保存时长。需要注意，`retentions` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 19 | single_stable | BOOL | 表示此数据库中是否只可以创建一个超级表。需要注意，`single_stable` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 20 | cachemodel | BINARY(60) | 表示是否在内存中缓存子表的最近数据。需要注意，`cachemodel` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 21 | cachesize | INT | 表示每个 vnode 中用于缓存子表最近数据的内存大小。需要注意，`cachesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 22 | wal_level | INT | WAL 级别。需要注意，`wal_level` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 23 | wal_fsync_period | INT | 数据落盘周期。需要注意，`wal_fsync_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 24 | wal_retention_period | INT | WAL 的保存时长。需要注意，`wal_retention_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 25 | wal_retention_size | INT | WAL 的保存上限。需要注意，`wal_retention_size` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 26 | wal_roll_period | INT | wal 文件切换时长。需要注意，`wal_roll_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 27 | wal_segment_size | BIGINT | wal 单个文件大小。需要注意，`wal_segment_size` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 28 | stt_trigger | SMALLINT | 触发文件合并的落盘文件的个数。需要注意，`stt_trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 29 | table_prefix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。需要注意，`table_prefix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 30 | table_suffix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。需要注意，`table_suffix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 31 | tsdb_pagesize | INT | 时序数据存储引擎中的页大小。需要注意，`tsdb_pagesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | ## INS_FUNCTIONS 用户创建的自定义函数的信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :---------: | ------------ | -------------- | -| 1 | name | BINARY(64) | 函数名 | -| 2 | comment | BINARY(255) | 补充说明。需要注意，`comment` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 3 | aggregate | INT | 是否为聚合函数。需要注意，`aggregate` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 4 | output_type | BINARY(31) | 输出类型 | -| 5 | create_time | TIMESTAMP | 创建时间 | -| 6 | code_len | INT | 代码长度 | -| 7 | bufsize | INT | buffer 大小 | -| 8 | func_language | BINARY(31) | 自定义函数编程语言 | -| 9 | func_body | BINARY(16384) | 函数体定义 | -| 10 | func_version | INT | 函数版本号。初始版本为0，每次替换更新，版本号加1。| +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------- | --------------------------------------------------------------------------------------------- | +| 1 | name | BINARY(64) | 函数名 | +| 2 | comment | BINARY(255) | 补充说明。需要注意，`comment` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 3 | aggregate | INT | 是否为聚合函数。需要注意，`aggregate` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 4 | output_type | BINARY(31) | 输出类型 | +| 5 | create_time | TIMESTAMP | 创建时间 | +| 6 | code_len | INT | 代码长度 | +| 7 | bufsize | INT | buffer 大小 | +| 8 | func_language | BINARY(31) | 自定义函数编程语言 | +| 9 | func_body | BINARY(16384) | 函数体定义 | +| 10 | func_version | INT | 函数版本号。初始版本为0，每次替换更新，版本号加1。 | ## INS_INDEXES 提供用户创建的索引的相关信息。也可以使用 SHOW INDEX 来查询这些信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :--------------: | ------------ | ---------------------------------------------------------------------------------- | -| 1 | db_name | BINARY(32) | 包含此索引的表所在的数据库名 | -| 2 | table_name | BINARY(192) | 包含此索引的表的名称 | -| 3 | index_name | BINARY(192) | 索引名 | -| 4 | column_name | BINARY(64) | 建索引的列的列名 | -| 5 | index_type | BINARY(10) | 目前有 SMA 和 FULLTEXT | -| 6 | index_extensions | BINARY(256) | 索引的额外信息。对 SMA 类型的索引，是函数名的列表。对 FULLTEXT 类型的索引为 NULL。 | +| # | **列名** | **数据类型** | **说明** | +| --- | :--------------: | ------------ | ------------------------------------------------------- | +| 1 | db_name | BINARY(32) | 包含此索引的表所在的数据库名 | +| 2 | table_name | BINARY(192) | 包含此索引的表的名称 | +| 3 | index_name | BINARY(192) | 索引名 | +| 4 | column_name | BINARY(64) | 建索引的列的列名 | +| 5 | index_type | BINARY(10) | 目前有 SMA 和 tag | +| 6 | index_extensions | BINARY(256) | 索引的额外信息。对 SMA/tag 类型的索引，是函数名的列表。 | ## INS_STABLES 提供用户创建的超级表的相关信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :-----------: | ------------ | ------------------------ | -| 1 | stable_name | BINARY(192) | 超级表表名 | -| 2 | db_name | BINARY(64) | 超级表所在的数据库的名称 | -| 3 | create_time | TIMESTAMP | 创建时间 | -| 4 | columns | INT | 列数目 | -| 5 | tags | INT | 标签数目。需要注意，`tags` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 6 | last_update | TIMESTAMP | 最后更新时间 | -| 7 | table_comment | BINARY(1024) | 表注释 | -| 8 | watermark | BINARY(64) | 窗口的关闭时间。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | max_delay | BINARY(64) | 推送计算结果的最大延迟。需要注意，`max_delay` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 10 | rollup | BINARY(128) | rollup 聚合函数。需要注意，`rollup` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | ----------------------------------------------------------------------------------------------------- | +| 1 | stable_name | BINARY(192) | 超级表表名 | +| 2 | db_name | BINARY(64) | 超级表所在的数据库的名称 | +| 3 | create_time | TIMESTAMP | 创建时间 | +| 4 | columns | INT | 列数目 | +| 5 | tags | INT | 标签数目。需要注意，`tags` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 6 | last_update | TIMESTAMP | 最后更新时间 | +| 7 | table_comment | BINARY(1024) | 表注释 | +| 8 | watermark | BINARY(64) | 窗口的关闭时间。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 9 | max_delay | BINARY(64) | 推送计算结果的最大延迟。需要注意，`max_delay` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 10 | rollup | BINARY(128) | rollup 聚合函数。需要注意，`rollup` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | ## INS_TABLES 提供用户创建的普通表和子表的相关信息 -| # | **列名** | **数据类型** | **说明** | -| --- | :-----------: | ------------ | ---------------- | -| 1 | table_name | BINARY(192) | 表名 | -| 2 | db_name | BINARY(64) | 数据库名 | -| 3 | create_time | TIMESTAMP | 创建时间 | -| 4 | columns | INT | 列数目 | -| 5 | stable_name | BINARY(192) | 所属的超级表表名 | -| 6 | uid | BIGINT | 表 id | -| 7 | vgroup_id | INT | vgroup id | -| 8 | ttl | INT | 表的生命周期。需要注意，`ttl` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | table_comment | BINARY(1024) | 表注释 | -| 10 | type | BINARY(21) | 表类型 | +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | ------------------------------------------------------------------------------------- | +| 1 | table_name | BINARY(192) | 表名 | +| 2 | db_name | BINARY(64) | 数据库名 | +| 3 | create_time | TIMESTAMP | 创建时间 | +| 4 | columns | INT | 列数目 | +| 5 | stable_name | BINARY(192) | 所属的超级表表名 | +| 6 | uid | BIGINT | 表 id | +| 7 | vgroup_id | INT | vgroup id | +| 8 | ttl | INT | 表的生命周期。需要注意，`ttl` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 9 | table_comment | BINARY(1024) | 表注释 | +| 10 | type | BINARY(21) | 表类型 | ## INS_TAGS @@ -185,17 +185,17 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 ## INS_COLUMNS -| # | **列名** | **数据类型** | **说明** | -| --- | :---------: | ------------- | ---------------------- | -| 1 | table_name | BINARY(192) | 表名 | -| 2 | db_name | BINARY(64) | 该表所在的数据库的名称 | -| 3 | table_type | BINARY(21) | 表类型 | -| 4 | col_name | BINARY(64) | 列 的名称 | -| 5 | col_type | BINARY(32) | 列 的类型 | -| 6 | col_length | INT | 列 的长度 | -| 7 | col_precision | INT | 列 的精度 | -| 8 | col_scale | INT | 列 的比例 | -| 9 | col_nullable | INT | 列 是否可以为空 | +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | ---------------------- | +| 1 | table_name | BINARY(192) | 表名 | +| 2 | db_name | BINARY(64) | 该表所在的数据库的名称 | +| 3 | table_type | BINARY(21) | 表类型 | +| 4 | col_name | BINARY(64) | 列 的名称 | +| 5 | col_type | BINARY(32) | 列 的类型 | +| 6 | col_length | INT | 列 的长度 | +| 7 | col_precision | INT | 列 的精度 | +| 8 | col_scale | INT | 列 的比例 | +| 9 | col_nullable | INT | 列 是否可以为空 | ## INS_USERS @@ -211,60 +211,60 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 提供企业版授权的相关信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :---------: | ------------ | -------------------------------------------------- | -| 1 | version | BINARY(9) | 企业版授权说明：official(官方授权的)/trial(试用的) | -| 2 | cpu_cores | BINARY(9) | 授权使用的 CPU 核心数量 | -| 3 | dnodes | BINARY(10) | 授权使用的 dnode 节点数量。需要注意，`dnodes` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 4 | streams | BINARY(10) | 授权创建的流数量。需要注意，`streams` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 5 | users | BINARY(10) | 授权创建的用户数量。需要注意，`users` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 6 | accounts | BINARY(10) | 授权创建的帐户数量。需要注意，`accounts` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 7 | storage | BINARY(21) | 授权使用的存储空间大小。需要注意，`storage` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 8 | connections | BINARY(21) | 授权使用的客户端连接数量。需要注意，`connections` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 9 | databases | BINARY(11) | 授权使用的数据库数量。需要注意，`databases` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 10 | speed | BINARY(9) | 授权使用的数据点每秒写入数量 | -| 11 | querytime | BINARY(9) | 授权使用的查询总时长 | -| 12 | timeseries | BINARY(21) | 授权使用的测点数量 | -| 13 | expired | BINARY(5) | 是否到期，true：到期，false：未到期 | -| 14 | expire_time | BINARY(19) | 试用期到期时间 | +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | --------------------------------------------------------------------------------------------------------- | +| 1 | version | BINARY(9) | 企业版授权说明：official(官方授权的)/trial(试用的) | +| 2 | cpu_cores | BINARY(9) | 授权使用的 CPU 核心数量 | +| 3 | dnodes | BINARY(10) | 授权使用的 dnode 节点数量。需要注意，`dnodes` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 4 | streams | BINARY(10) | 授权创建的流数量。需要注意，`streams` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 5 | users | BINARY(10) | 授权创建的用户数量。需要注意，`users` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 6 | accounts | BINARY(10) | 授权创建的帐户数量。需要注意，`accounts` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 7 | storage | BINARY(21) | 授权使用的存储空间大小。需要注意，`storage` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 8 | connections | BINARY(21) | 授权使用的客户端连接数量。需要注意，`connections` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 9 | databases | BINARY(11) | 授权使用的数据库数量。需要注意，`databases` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 10 | speed | BINARY(9) | 授权使用的数据点每秒写入数量 | +| 11 | querytime | BINARY(9) | 授权使用的查询总时长 | +| 12 | timeseries | BINARY(21) | 授权使用的测点数量 | +| 13 | expired | BINARY(5) | 是否到期，true：到期，false：未到期 | +| 14 | expire_time | BINARY(19) | 试用期到期时间 | ## INS_VGROUPS 系统中所有 vgroups 的信息。 -| # | **列名** | **数据类型** | **说明** | -| --- | :-------: | ------------ | ------------------------------------------------------ | -| 1 | vgroup_id | INT | vgroup id | -| 2 | db_name | BINARY(32) | 数据库名 | -| 3 | tables | INT | 此 vgroup 内有多少表。需要注意，`tables` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | -| 4 | status | BINARY(10) | 此 vgroup 的状态 | -| 5 | v1_dnode | INT | 第一个成员所在的 dnode 的 id | -| 6 | v1_status | BINARY(10) | 第一个成员的状态 | -| 7 | v2_dnode | INT | 第二个成员所在的 dnode 的 id | -| 8 | v2_status | BINARY(10) | 第二个成员的状态 | -| 9 | v3_dnode | INT | 第三个成员所在的 dnode 的 id | -| 10 | v3_status | BINARY(10) | 第三个成员的状态 | -| 11 | nfiles | INT | 此 vgroup 中数据/元数据文件的数量 | -| 12 | file_size | INT | 此 vgroup 中数据/元数据文件的大小 | -| 13 | tsma | TINYINT | 此 vgroup 是否专用于 Time-range-wise SMA，1: 是, 0: 否 | +| # | **列名** | **数据类型** | **说明** | +| --- | :-------: | ------------ | ------------------------------------------------------------------------------------------------ | +| 1 | vgroup_id | INT | vgroup id | +| 2 | db_name | BINARY(32) | 数据库名 | +| 3 | tables | INT | 此 vgroup 内有多少表。需要注意，`tables` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| 4 | status | BINARY(10) | 此 vgroup 的状态 | +| 5 | v1_dnode | INT | 第一个成员所在的 dnode 的 id | +| 6 | v1_status | BINARY(10) | 第一个成员的状态 | +| 7 | v2_dnode | INT | 第二个成员所在的 dnode 的 id | +| 8 | v2_status | BINARY(10) | 第二个成员的状态 | +| 9 | v3_dnode | INT | 第三个成员所在的 dnode 的 id | +| 10 | v3_status | BINARY(10) | 第三个成员的状态 | +| 11 | nfiles | INT | 此 vgroup 中数据/元数据文件的数量 | +| 12 | file_size | INT | 此 vgroup 中数据/元数据文件的大小 | +| 13 | tsma | TINYINT | 此 vgroup 是否专用于 Time-range-wise SMA，1: 是, 0: 否 | ## INS_CONFIGS 系统配置参数。 -| # | **列名** | **数据类型** | **说明** | -| --- | :------: | ------------ | ------------ | -| 1 | name | BINARY(32) | 配置项名称 | +| # | **列名** | **数据类型** | **说明** | +| --- | :------: | ------------ | --------------------------------------------------------------------------------------- | +| 1 | name | BINARY(32) | 配置项名称 | | 2 | value | BINARY(64) | 该配置项的值。需要注意，`value` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | ## INS_DNODE_VARIABLES 系统中每个 dnode 的配置参数。 -| # | **列名** | **数据类型** | **说明** | -| --- | :------: | ------------ | ------------ | -| 1 | dnode_id | INT | dnode 的 ID | -| 2 | name | BINARY(32) | 配置项名称 | +| # | **列名** | **数据类型** | **说明** | +| --- | :------: | ------------ | --------------------------------------------------------------------------------------- | +| 1 | dnode_id | INT | dnode 的 ID | +| 2 | name | BINARY(32) | 配置项名称 | | 3 | value | BINARY(64) | 该配置项的值。需要注意，`value` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | ## INS_TOPICS @@ -284,19 +284,19 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数 | 2 | consumer_group | BINARY(193) | 订阅者的消费者组 | | 3 | vgroup_id | INT | 消费者被分配的 vgroup id | | 4 | consumer_id | BIGINT | 消费者的唯一 id | -| 5 | offset | BINARY(64) | 消费者的消费进度 | -| 6 | rows | BIGINT | 消费者的消费的数据条数 | +| 5 | offset | BINARY(64) | 消费者的消费进度 | +| 6 | rows | BIGINT | 消费者的消费的数据条数 | ## INS_STREAMS -| # | **列名** | **数据类型** | **说明** | -| --- | :----------: | ------------ | --------------------------------------- | -| 1 | stream_name | BINARY(64) | 流计算名称 | -| 2 | create_time | TIMESTAMP | 创建时间 | -| 3 | sql | BINARY(1024) | 创建流计算时提供的 SQL 语句 | -| 4 | status | BINARY(20) | 流当前状态 | -| 5 | source_db | BINARY(64) | 源数据库 | -| 6 | target_db | BINARY(64) | 目的数据库 | -| 7 | target_table | BINARY(192) | 流计算写入的目标表 | -| 8 | watermark | BIGINT | watermark，详见 SQL 手册流式计算。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | -------------------------------------------------------------------------------------------------------------------- | +| 1 | stream_name | BINARY(64) | 流计算名称 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | sql | BINARY(1024) | 创建流计算时提供的 SQL 语句 | +| 4 | status | BINARY(20) | 流当前状态 | +| 5 | source_db | BINARY(64) | 源数据库 | +| 6 | target_db | BINARY(64) | 目的数据库 | +| 7 | target_table | BINARY(192) | 流计算写入的目标表 | +| 8 | watermark | BIGINT | watermark，详见 SQL 手册流式计算。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | | 9 | trigger | INT | 计算结果推送模式，详见 SQL 手册流式计算。需要注意，`trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 | diff --git a/docs/zh/12-taos-sql/27-index.md b/docs/zh/12-taos-sql/27-index.md index 3f3091b19c..51cf5f4e4e 100644 --- a/docs/zh/12-taos-sql/27-index.md +++ b/docs/zh/12-taos-sql/27-index.md @@ -4,12 +4,13 @@ title: 索引 description: 索引功能的使用细节 --- -TDengine 从 3.0.0.0 版本开始引入了索引功能，支持 SMA 索引和 FULLTEXT 索引。 +TDengine 从 3.0.0.0 版本开始引入了索引功能，支持 SMA 索引和 tag 索引。 ## 创建索引 ```sql -CREATE FULLTEXT INDEX index_name ON tb_name (col_name [, col_name] ...) + +CREATE INDEX index_name ON tb_name index_option CREATE SMA INDEX index_name ON tb_name index_option @@ -46,10 +47,6 @@ SELECT _wstart,_wend,_wduration,max(c2),min(c1) FROM st1 INTERVAL(5m,10s) SLIDIN ALTER LOCAL 'querySmaOptimize' '0'; ``` -### FULLTEXT 索引 - -对指定列建立文本索引，可以提升含有文本过滤的查询的性能。FULLTEXT 索引不支持 index_option 语法。现阶段只支持对 JSON 类型的标签列创建 FULLTEXT 索引。不支持多列联合索引，但可以为每个列分布创建 FULLTEXT 索引。 - ## 删除索引 ```sql From 6c3508150738ef1c6503a15251986d63c1607a4e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 18 Jul 2023 09:54:35 +0800 Subject: [PATCH 043/100] fix:add sdbFetchCancel to release hash node --- source/dnode/mnode/impl/src/mndCluster.c | 1 - source/dnode/mnode/impl/src/mndDb.c | 3 +-- source/dnode/mnode/impl/src/mndIndex.c | 2 +- source/dnode/mnode/impl/src/mndQnode.c | 1 + source/dnode/mnode/impl/src/mndScheduler.c | 1 + source/dnode/mnode/impl/src/mndStb.c | 3 +++ source/dnode/mnode/impl/src/mndUser.c | 4 ---- 7 files changed, 7 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 67675b5400..aa00580c93 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -77,7 +77,6 @@ static SClusterObj *mndAcquireCluster(SMnode *pMnode, void **ppIter) { if (pIter == NULL) break; *ppIter = pIter; - sdbCancelFetch(pSdb, pIter); return pCluster; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5e45c7b242..87383265d9 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1303,11 +1303,10 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { sdbRelease(pSdb, pVgroup); if (pDb && (vindex >= pDb->cfg.numOfVgroups)) { + sdbCancelFetch(pSdb, pIter); break; } } - - sdbCancelFetch(pSdb, pIter); } int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq) { diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 8f977dacb7..c3e95a2d1f 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -852,7 +852,7 @@ int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { if (pIdx->dbUid == pDb->uid) { if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) { sdbRelease(pSdb, pIdx); - sdbCancelFetch(pSdb, pIdx); + sdbCancelFetch(pSdb, pIter); return -1; } } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index b5c9ce1f65..5ec81440bb 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -454,6 +454,7 @@ int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) { sdbRelease(pSdb, pObj); if (limit > 0 && numOfRows >= limit) { + sdbCancelFetch(pSdb, pIter); break; } } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index b95f4d6a00..dbac9ec9bb 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -198,6 +198,7 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { sdbRelease(pMnode->pSdb, pVgroup); continue; } + sdbCancelFetch(pMnode->pSdb, pIter); return pVgroup; } return pVgroup; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index c1186d068f..e8af02a828 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2330,6 +2330,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, } if (pStream->targetStbUid == suid) { + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pStream); return -1; } @@ -2338,6 +2339,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, if (nodesStringToNode(pStream->ast, &pAst) != 0) { terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; mError("stream:%s, create ast error", pStream->name); + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pStream); return -1; } @@ -2349,6 +2351,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, SColumnNode *pCol = (SColumnNode *)pNode; if (pCol->tableId == suid) { + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pStream); nodesDestroyNode(pAst); nodesDestroyList(pNodeList); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index d3b2ef7344..999431a5fb 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1445,7 +1445,6 @@ int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { code = -1; if (mndUserDupObj(pUser, &newUser) != 0) { - sdbCancelFetch(pSdb, pIter); break; } @@ -1457,7 +1456,6 @@ int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - sdbCancelFetch(pSdb, pIter); break; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); @@ -1490,7 +1488,6 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { code = -1; if (mndUserDupObj(pUser, &newUser) != 0) { - sdbCancelFetch(pSdb, pIter); break; } @@ -1499,7 +1496,6 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { (void)taosHashRemove(newUser.topics, topic, len); SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - sdbCancelFetch(pSdb, pIter); break; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); From 6e80b2b4a3b4d8dcd4d2ecf07e9e4413baa942e1 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 18 Jul 2023 10:19:13 +0800 Subject: [PATCH 044/100] fix packaging issue --- packaging/tools/install_client.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 8b845ca8f4..18ebf9dc8f 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -267,7 +267,9 @@ function install_log() { } function install_connector() { - ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ + if [ -d ${script_dir}/connector ]; then + ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ + fi } function install_examples() { From 94b13df641cc95829a3149b62d9921e3bd3730d8 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 18 Jul 2023 10:36:07 +0800 Subject: [PATCH 045/100] fix: fix interval in nested query filter not applied --- source/libs/executor/src/timewindowoperator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 78d1e97554..5fd46f572a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4350,6 +4350,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo); resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow)); cleanupAfterGroupResultGen(pMiaInfo, pRes); + doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); } setOperatorCompleted(pOperator); @@ -4370,6 +4371,7 @@ static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { pMiaInfo->prefetchedBlock = pBlock; cleanupAfterGroupResultGen(pMiaInfo, pRes); + doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); break; } else { // continue From c033592b291dfd7cd4c9c375a36731e088998d89 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 18 Jul 2023 11:00:26 +0800 Subject: [PATCH 046/100] fix Windows packaging issue --- packaging/tools/make_install.bat | 4 ++-- tools/CMakeLists.txt | 22 ++++++++++++++++++++-- tools/shell/CMakeLists.txt | 2 +- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index 41113c9ae4..0b2a55b89c 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -56,8 +56,8 @@ copy %binary_dir%\\build\\bin\\taos.exe %target_dir% > nul if exist %binary_dir%\\build\\bin\\taosBenchmark.exe ( copy %binary_dir%\\build\\bin\\taosBenchmark.exe %target_dir% > nul ) -if exist %binary_dir%\\build\\lib\\taosws.dll.lib ( - copy %binary_dir%\\build\\lib\\taosws.dll.lib %target_dir%\\driver > nul +if exist %binary_dir%\\build\\lib\\taosws.lib ( + copy %binary_dir%\\build\\lib\\taosws.lib %target_dir%\\driver > nul ) if exist %binary_dir%\\build\\lib\\taosws.dll ( copy %binary_dir%\\build\\lib\\taosws.dll %target_dir%\\driver > nul diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index e882658408..ea8b5e6169 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -3,8 +3,6 @@ IF (TD_WEBSOCKET) SET(websocket_lib_file "libtaosws.so") ELSEIF (TD_DARWIN) SET(websocket_lib_file "libtaosws.dylib") - ELSEIF (TD_WINDOWS) - SET(websocket_lib_file "{taosws.dll,taosws.dll.lib}") ENDIF () MESSAGE("${Green} use libtaos-ws${ColourReset}") IF (TD_ALPINE) @@ -26,6 +24,26 @@ IF (TD_WEBSOCKET) COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include COMMAND cmake -E copy target/release/taosws.h ${CMAKE_BINARY_DIR}/build/include ) + ELSEIF (TD_WINDOWS) + include(ExternalProject) + ExternalProject_Add(taosws-rs + PREFIX "taosws-rs" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs + BUILD_ALWAYS off + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND cargo update + COMMAND cargo build --release -p taos-ws-sys --features native-tls-vendored + INSTALL_COMMAND + COMMAND cp target/release/taosws.dll ${CMAKE_BINARY_DIR}/build/lib + COMMAND cp target/release/taosws.dll.lib ${CMAKE_BINARY_DIR}/build/lib/taosws.lib + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include + COMMAND cmake -E copy target/release/taosws.h ${CMAKE_BINARY_DIR}/build/include + ) ELSE() include(ExternalProject) ExternalProject_Add(taosws-rs diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index acc47d4910..0ce181808f 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -20,7 +20,7 @@ ELSEIF (TD_DARWIN AND TD_WEBSOCKET) ADD_DEPENDENCIES(shell taosws-rs) ELSEIF (TD_WINDOWS AND TD_WEBSOCKET) ADD_DEFINITIONS(-DWEBSOCKET -I${CMAKE_BINARY_DIR}/build/include) - SET(LINK_WEBSOCKET "${CMAKE_BINARY_DIR}/build/lib/taosws.dll.lib") + SET(LINK_WEBSOCKET "${CMAKE_BINARY_DIR}/build/lib/taosws.lib") ADD_DEPENDENCIES(shell taosws-rs) ELSE () SET(LINK_WEBSOCKET "") From 97a81bd8ed4b9b633db8977e06c394a3725fd257 Mon Sep 17 00:00:00 2001 From: dmchen Date: Tue, 18 Jul 2023 11:27:01 +0800 Subject: [PATCH 047/100] fix/TD-25179 --- source/dnode/mnode/impl/src/mndUser.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 1fc2e42b8c..c0d8591242 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -922,19 +922,19 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { } } - if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE) { + if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE) { if (mndTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; } - if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE) { + if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE) { if (mndTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; } - if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_TABLE) { + if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_TABLE || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) { if (mndRemoveTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; } - if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_TABLE) { + if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_TABLE || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) { if (mndRemoveTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; } From 447eb8197c1d1821b512f12cc186f987754dac6f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 18 Jul 2023 14:25:03 +0800 Subject: [PATCH 048/100] test:add testcase of rolling upgdade --- tests/system-test/6-cluster/5dnode3mnodeRoll.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/system-test/6-cluster/5dnode3mnodeRoll.py b/tests/system-test/6-cluster/5dnode3mnodeRoll.py index a9d4a964e7..8d7d4fb3e5 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRoll.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRoll.py @@ -260,27 +260,26 @@ class TDTestCase: for tr in threads: tr.join() + + tdLog.printNoPrefix(f"==========step3:check dnode status ") # wait 10s for taosd cluster ready sleep(10) tdsql=tdCom.newTdSql() - print(tdsql) tdsql.query("select * from information_schema.ins_dnodes;") tdLog.info(tdsql.queryResult) tdsql.checkData(2,1,'%s:6230'%self.host) - tdSql=tdCom.newTdSql() - print(tdSql) clusterComCheck.checkDnodes(dnodeNumbers) tdsql1=tdCom.newTdSql() - print(tdsql1) tdsql1.query(f"SELECT SERVER_VERSION();") nowServerVersion=tdsql1.queryResult[0][0] + tdLog.printNoPrefix(f"==========step4:prepare and check data in new version-{nowServerVersion}") + tdLog.info(f"New server version is {nowServerVersion}") tdsql1.query(f"SELECT CLIENT_VERSION();") nowClientVersion=tdsql1.queryResult[0][0] tdLog.info(f"New client version is {nowClientVersion}") - tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") tdsql1.query(f"select count(*) from {stb}") tdsql1.checkData(0,0,tableNumbers1*recordNumbers1) tdsql1.query(f"select count(*) from db4096.stb0") From 745ab09b1b7245dea9b02670fec354db5b1bd8ab Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 18 Jul 2023 14:30:19 +0800 Subject: [PATCH 049/100] test: add tmq seek case --- source/client/src/clientTmq.c | 4 +- tests/parallel_test/cases.task | 1 + tests/system-test/7-tmq/tmqOffset.py | 399 +++++++++++++++++++++++++++ 3 files changed, 402 insertions(+), 2 deletions(-) create mode 100644 tests/system-test/7-tmq/tmqOffset.py diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 5879de2e30..8c839f7422 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2816,7 +2816,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } - if (type == TMQ_OFFSET__LOG && (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd)) { + if ((offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd)) { tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd); taosWUnLockLatch(&tmq->lock); @@ -2893,4 +2893,4 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ } return code; -} \ No newline at end of file +} diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 85580ffeed..f6126642bb 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -38,6 +38,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqOffset.py ,,n,system-test,python3 ./test.py -f 7-tmq/tmqDropConsumer.py diff --git a/tests/system-test/7-tmq/tmqOffset.py b/tests/system-test/7-tmq/tmqOffset.py new file mode 100644 index 0000000000..500c6f53e4 --- /dev/null +++ b/tests/system-test/7-tmq/tmqOffset.py @@ -0,0 +1,399 @@ + +import sys +import re +import time +import threading +from taos.tmq import Consumer +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135} + + def __init__(self): + self.vgroups = 2 + self.ctbNum = 1 + self.rowsPerTbl = 10000 + self.tmqMaxTopicNum = 10 + self.tmqMaxGroups = 10 + + self.TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE = '0x4007' + self.TSDB_CODE_TMQ_INVALID_VGID = '0x4008' + self.TSDB_CODE_TMQ_INVALID_TOPIC = '0x4009' + + + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + # 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + # 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + + 'colSchema': [{'type': 'INT', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}], + + + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 10, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + # tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_period=36000) + # tdSql.execute("alter database %s wal_retention_period 360000" % (paraDict['dbName'])) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqPollAllRows(self, consumer): + totalRows = 0 + + res = consumer.poll(10) + while (res): + if not res: + break + err = res.error() + if err is not None: + raise err + + val = res.value() + # print(len(val)) + for block in val: + # print(block.fetchall()) + # print(len(block.fetchall())) + totalRows += len(block.fetchall()) + + res = consumer.poll(10) + + tdLog.info("poll total rows: %d"%(totalRows)) + return totalRows + + def tmqPollRowsByOne(self, consumer): + rows = 0 + res = consumer.poll(3) + if not res: + return rows + err = res.error() + if err is not None: + raise err + val = res.value() + + # print(len(val)) + + for block in val: + # print(block.fetchall()) + # print(len(block.fetchall())) + rows += len(block.fetchall()) + + return rows + + def tmqOffsetTest(self, consumer): + # get topic assignment + tdLog.info("before poll get offset status:") + assignments = consumer.assignment() + for assignment in assignments: + print(assignment) + + # poll + # consumer.poll(5) + rows = self.tmqPollRowsByOne(consumer) + tdLog.info("poll rows: %d"%(rows)) + + # get topic assignment + tdLog.info("after first poll get offset status:") + assignments = consumer.assignment() + for assignment in assignments: + print(assignment) + + + rows = self.tmqPollRowsByOne(consumer) + tdLog.info("poll rows: %d"%(rows)) + + # get topic assignment + tdLog.info("after second poll get offset status:") + assignments = consumer.assignment() + for assignment in assignments: + print(assignment) + + + return + + def tmqSubscribe(self, inputDict): + consumer_dict = { + "group.id": inputDict['group_id'], + "client.id": "client", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.commit.interval.ms": "1000", + "enable.auto.commit": inputDict['auto_commit'], + "auto.offset.reset": inputDict['offset_reset'], + "experimental.snapshot.enable": "false", + "msg.with.table.name": "false" + } + + consumer = Consumer(consumer_dict) + try: + consumer.subscribe([inputDict['topic_name']]) + except Exception as e: + tdLog.info("consumer.subscribe() fail ") + tdLog.info("%s"%(e)) + + # rows = self.tmqPollAllRows(consumer) + tdLog.info("create consumer success!") + return consumer + + def tmqConsumer(self, **inputDict): + consumer = self.tmqSubscribe(inputDict) + self.tmqPollAllRows(consumer) + # consumer.unsubscribe() + # consumer.close() + return + + def asyncSubscribe(self, inputDict): + pThread = threading.Thread(target=self.tmqConsumer, kwargs=inputDict) + pThread.start() + return pThread + + def seekErrorVgid(self, consumer, assignment): + ####################### test1: error vgid + assignmentNew = assignment + # assignment.topic + assignmentNew.partition = assignment.partition + self.vgroups + self.vgroups + # assignment.offset + # consumer.seek(assignment) + + errCodeStr = '' + try: + print("seek parameters:", assignmentNew) + consumer.seek(assignmentNew) + except Exception as e: + tdLog.info("error: %s"%(e)) + + rspString = str(e) + start = "[" + end = "]" + + start_index = rspString.index(start) + len(start) + end_index = rspString.index(end) + + errCodeStr = rspString[start_index:end_index] + # print(errCodeStr) + tdLog.info("error code: %s"%(errCodeStr)) + + if (self.TSDB_CODE_TMQ_INVALID_VGID != errCodeStr): + tdLog.exit("tmq seek should return error code: %s"%(self.TSDB_CODE_TMQ_INVALID_VGID)) + + def seekErrorTopic(self, consumer, assignment): + assignmentNew = assignment + assignmentNew.topic = 'errorToipcName' + # assignment.partition + # assignment.offset + # consumer.seek(assignment) + + errCodeStr = '' + try: + print("seek parameters:", assignmentNew) + consumer.seek(assignmentNew) + except Exception as e: + tdLog.info("error: %s"%(e)) + + rspString = str(e) + start = "[" + end = "]" + + start_index = rspString.index(start) + len(start) + end_index = rspString.index(end) + + errCodeStr = rspString[start_index:end_index] + # print(errCodeStr) + tdLog.info("error code: %s"%(errCodeStr)) + + if (self.TSDB_CODE_TMQ_INVALID_TOPIC != errCodeStr): + tdLog.exit("tmq seek should return error code: %s"%(self.TSDB_CODE_TMQ_INVALID_TOPIC)) + + def seekErrorVersion(self, consumer, assignment): + assignmentNew = assignment + # print(assignment.topic, assignment.partition, assignment.offset) + # assignment.topic + # assignment.partition + assignmentNew.offset = assignment.offset + self.rowsPerTbl * 100000 + # consumer.seek(assignment) + + errCodeStr = '' + try: + # print(assignmentNew.topic, assignmentNew.partition, assignmentNew.offset) + print("seek parameters:", assignmentNew) + consumer.seek(assignmentNew) + except Exception as e: + tdLog.info("error: %s"%(e)) + + rspString = str(e) + start = "[" + end = "]" + + start_index = rspString.index(start) + len(start) + end_index = rspString.index(end) + + errCodeStr = rspString[start_index:end_index] + # print(errCodeStr) + tdLog.info("error code: %s"%(errCodeStr)) + + if (self.TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE != errCodeStr): + tdLog.exit("tmq seek should return error code: %s"%(self.TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE)) + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 100000000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + # ntbName = 'ntb' + # sqlString = "create table %s.%s (ts timestamp, c int)"%(paraDict['dbName'], ntbName) + # tdLog.info("create ntb sql: %s"%sqlString) + # tdSql.execute(sqlString) + + topicName = 'offset_tp' + # queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + inputDict = { + "topic_name": topicName, + "group_id": "offsetGrp", + "auto_commit": "true", + "offset_reset": "earliest" + } + + pThread = self.asyncSubscribe(inputDict) + # pThread.join() + + consumer = self.tmqSubscribe(inputDict) + # get topic assignment + assignments = consumer.assignment() + # print(type(assignments)) + for assignment in assignments: + print(assignment) + + assignment = assignments[0] + topic = assignment.topic + partition = assignment.partition + offset = assignment.offset + + tdLog.info("======== test error vgid =======") + print("current assignment: ", assignment) + self.seekErrorVgid(consumer, assignment) + + tdLog.info("======== test error topic =======") + assignment.topic = topic + assignment.partition = partition + assignment.offset = offset + print("current assignment: ", assignment) + self.seekErrorTopic(consumer, assignment) + + tdLog.info("======== test error version =======") + assignment.topic = topic + assignment.partition = partition + assignment.offset = offset + print("current assignment: ", assignment) + self.seekErrorVersion(consumer, assignment) + + pThread.join() + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 08c50dfbb4034663478ff2dc00caa325dc8a128f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 18 Jul 2023 15:21:35 +0800 Subject: [PATCH 050/100] fix:add sdbFetchCancel to release hash node --- source/dnode/mnode/impl/src/mndMain.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 55cca5a30c..381b1e64ed 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -199,7 +199,6 @@ static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs) pGid->syncCanRead = 0; roleChanged = true; } - sdbCancelFetch(pSdb, pIter); break; } } From 7a0fa59d3e9a7c00ba284fb053656bc022abed9a Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 18 Jul 2023 16:17:42 +0800 Subject: [PATCH 051/100] test: check topic name len case --- tests/system-test/7-tmq/tmqMaxTopic.py | 135 +++++++++++++++++++------ 1 file changed, 105 insertions(+), 30 deletions(-) diff --git a/tests/system-test/7-tmq/tmqMaxTopic.py b/tests/system-test/7-tmq/tmqMaxTopic.py index 5dc49fe48f..2371d53cc5 100644 --- a/tests/system-test/7-tmq/tmqMaxTopic.py +++ b/tests/system-test/7-tmq/tmqMaxTopic.py @@ -216,41 +216,116 @@ class TDTestCase: tdLog.info("create topic sql: %s"%sqlString) tdSql.error(sqlString) - # pThreadList = [] - # for i in range(self.tmqMaxTopicNum): - # topic_name = f"%s%d" %(topicNamePrefix, i) - # print("======%s"%(topic_name)) - # group_id_prefix = f"grp_%d"%(i) - # inputDict = {'group_id_prefix': group_id_prefix, - # 'topic_name': topic_name, - # 'pollDelay': 1 - # } - - # pThread = self.asyncSubscribe(inputDict) - # pThreadList.append(pThread) - - # for j in range(self.tmqMaxGroups): - # pThreadList[j].join() - - # time.sleep(5) - # tdSql.query('show subscriptions;') - # subscribeNum = tdSql.queryRows - # expectNum = self.tmqMaxGroups * self.tmqMaxTopicNum - # tdLog.info("loop index: %d, ======subscriptions %d and expect num: %d"%(i, subscribeNum, expectNum)) - # if subscribeNum != expectNum: - # tdLog.exit("subscriptions %d not equal expect num: %d"%(subscribeNum, expectNum)) - - # # drop all topics - # for i in range(self.tmqMaxTopicNum): - # sqlString = "drop topic %s%d" %(topicNamePrefix, i) - # tdLog.info("drop topic sql: %s"%sqlString) - # tdSql.execute(sqlString) + tdSql.query('show topics;') + topicNum = tdSql.queryRows + tdLog.info(" topic count: %d"%(topicNum)) + for i in range(topicNum): + sqlString = "drop topic %s" %(tdSql.getData(i, 0)) + tdLog.info("drop topic sql: %s"%sqlString) + tdSql.execute(sqlString) - tdLog.printNoPrefix("======== test case 1 end ...... ") + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: test topic name len") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + totalTopicNum = 0 + + topicName = 'a' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.query(sqlString) + totalTopicNum += 1 + + topicName = '3' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.error(sqlString) + totalTopicNum += 0 + + topicName = '_1' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.query(sqlString) + totalTopicNum += 1 + + topicName = 'a\\' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.error(sqlString) + totalTopicNum += 0 + + topicName = 'a\*\&\^' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.error(sqlString) + totalTopicNum += 0 + + + str191char = 'a' + for i in range(190): + str191char = ('%s%d'%(str191char, 1)) + + topicName = str191char + 'a' + + if (192 != len(topicName)): + tdLog.exit("topicName len error") + + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.query(sqlString) + totalTopicNum += 1 + + topicName = str191char + '12' + sqlString = "create topic %s as %s" %(topicName, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.error(sqlString) + totalTopicNum += 0 + + # topicName = str192char + '12' + # sqlString = "create topic %s as %s" %(topicName, queryString) + # tdLog.info("create topic sql: %s"%sqlString) + # tdSql.error(sqlString) + # totalTopicNum += 0 + + # check topic count + tdSql.query('show topics;') + topicNum = tdSql.queryRows + tdLog.info(" topic count: %d"%(topicNum)) + if topicNum != totalTopicNum: + tdLog.exit("show topics %d not equal expect num: %d"%(topicNum, totalTopicNum)) + + + tdLog.printNoPrefix("======== test case 2 end ...... ") + def run(self): self.prepareTestEnv() self.tmqCase1() + self.tmqCase2() def stop(self): tdSql.close() From e812a659a84da9664c52b5722f8064baaf6203b8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 18 Jul 2023 19:30:40 +0800 Subject: [PATCH 052/100] fix:add tmq_position() interface & optimize commit logic --- include/util/taoserror.h | 1 + source/client/src/clientTmq.c | 496 +++++++++++++++++++++------------- source/util/src/terror.c | 1 + 3 files changed, 306 insertions(+), 192 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ff5d37bf00..d6f44f4489 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -778,6 +778,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x4007) #define TSDB_CODE_TMQ_INVALID_VGID TAOS_DEF_ERROR_CODE(0, 0x4008) #define TSDB_CODE_TMQ_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x4009) +#define TSDB_CODE_TMQ_NEED_INITIALIZED TAOS_DEF_ERROR_CODE(0, 0x4010) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 6bbcbe62be..f7c24c6776 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -139,8 +139,8 @@ enum { typedef struct SVgOffsetInfo { STqOffsetVal committedOffset; - STqOffsetVal currentOffset; - STqOffsetVal seekOffset; // the first version in block for seek operation + STqOffsetVal endOffset; // the last version in TAOS_RES + 1 + STqOffsetVal beginOffset; // the first version in TAOS_RES int64_t walVerBegin; int64_t walVerEnd; } SVgOffsetInfo; @@ -255,8 +255,7 @@ typedef struct SSyncCommitInfo { static int32_t doAskEp(tmq_t* tmq); static int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg); static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet); -static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicName, SMqCommitCbParamSet* pParamSet, - int32_t index, int32_t totalVgroups, int32_t type); +static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, SMqCommitCbParamSet* pParamSet); static void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId); static void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param); static void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param); @@ -429,69 +428,10 @@ char** tmq_list_to_c_array(const tmq_list_t* list) { return container->pData; } -//static SMqClientVg* foundClientVg(SArray* pTopicList, const char* pName, int32_t vgId, int32_t* index, -// int32_t* numOfVgroups) { -// int32_t numOfTopics = taosArrayGetSize(pTopicList); -// *index = -1; -// *numOfVgroups = 0; -// -// for (int32_t i = 0; i < numOfTopics; ++i) { -// SMqClientTopic* pTopic = taosArrayGet(pTopicList, i); -// if (strcmp(pTopic->topicName, pName) != 0) { -// continue; -// } -// -// *numOfVgroups = taosArrayGetSize(pTopic->vgs); -// for (int32_t j = 0; j < (*numOfVgroups); ++j) { -// SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); -// if (pClientVg->vgId == vgId) { -// *index = j; -// return pClientVg; -// } -// } -// } -// -// return NULL; -//} - -// Two problems do not need to be addressed here -// 1. update to of epset. the response of poll request will automatically handle this problem -// 2. commit failure. This one needs to be resolved. static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; - // if (code != TSDB_CODE_SUCCESS) { // if commit offset failed, let's try again - // taosThreadMutexLock(&pParam->pTmq->lock); - // int32_t numOfVgroups, index; - // SMqClientVg* pVg = foundClientVg(pParam->pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, - // &numOfVgroups); if (pVg == NULL) { - // tscDebug("consumer:0x%" PRIx64 - // " subKey:%s vgId:%d commit failed, code:%s has been transferred to other consumer, no need retry - // ordinal:%d/%d", pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, tstrerror(code), - // index + 1, numOfVgroups); - // } else { // let's retry the commit - // int32_t code1 = doSendCommitMsg(pParam->pTmq, pVg, pParam->topicName, pParamSet, index, numOfVgroups); - // if (code1 != TSDB_CODE_SUCCESS) { // retry failed. - // tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 - // " retry failed, ignore this commit. code:%s ordinal:%d/%d", - // pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->offsetInfo.committedOffset.version, - // tstrerror(terrno), index + 1, numOfVgroups); - // } - // } - // - // taosThreadMutexUnlock(&pParam->pTmq->lock); - // - // taosMemoryFree(pParam->pOffset); - // taosMemoryFree(pBuf->pData); - // taosMemoryFree(pBuf->pEpSet); - // - // commitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); - // return 0; - // } - // - // // todo replace the pTmq with refId - taosMemoryFree(pParam->pOffset); taosMemoryFree(pBuf->pData); taosMemoryFree(pBuf->pEpSet); @@ -500,15 +440,14 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { return 0; } -static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicName, SMqCommitCbParamSet* pParamSet, - int32_t index, int32_t totalVgroups, int32_t type) { +static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, SMqCommitCbParamSet* pParamSet) { SMqVgOffset* pOffset = taosMemoryCalloc(1, sizeof(SMqVgOffset)); if (pOffset == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } pOffset->consumerId = tmq->consumerId; - pOffset->offset.val = pVg->offsetInfo.currentOffset; + pOffset->offset.val = *offset; int32_t groupLen = strlen(tmq->groupId); memcpy(pOffset->offset.subKey, tmq->groupId, groupLen); @@ -519,6 +458,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN int32_t code = 0; tEncodeSize(tEncodeMqVgOffset, pOffset, len, code); if (code < 0) { + taosMemoryFree(pOffset); return TSDB_CODE_INVALID_PARA; } @@ -528,7 +468,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN return TSDB_CODE_OUT_OF_MEMORY; } - ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); + ((SMsgHead*)buf)->vgId = htonl(vgId); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); @@ -547,7 +487,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN pParam->params = pParamSet; pParam->pOffset = pOffset; - pParam->vgId = pVg->vgId; + pParam->vgId = vgId; pParam->pTmq = tmq; tstrncpy(pParam->topicName, pTopicName, tListLen(pParam->topicName)); @@ -568,23 +508,16 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = tmqCommitCb; - pMsgSendInfo->msgType = type; + pMsgSendInfo->msgType = TDMT_VND_TMQ_COMMIT_OFFSET; atomic_add_fetch_32(&pParamSet->waitingRspNum, 1); atomic_add_fetch_32(&pParamSet->totalRspNum, 1); - SEp* pEp = GET_ACTIVE_EP(&pVg->epSet); - char offsetBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffset->offset.val); + SEp* pEp = GET_ACTIVE_EP(epSet); - char commitBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); - tscInfo("consumer:0x%" PRIx64 " topic:%s on vgId:%d send offset:%s prev:%s, ep:%s:%d, ordinal:%d/%d, req:0x%" PRIx64, - tmq->consumerId, pOffset->offset.subKey, pVg->vgId, offsetBuf, commitBuf, pEp->fqdn, pEp->port, index + 1, - totalVgroups, pMsgSendInfo->requestId); int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, pMsgSendInfo); return TSDB_CODE_SUCCESS; } @@ -604,57 +537,28 @@ static SMqClientTopic* getTopicByName(tmq_t* tmq, const char* pTopicName) { return NULL; } -static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tmq_commit_cb* pCommitFp, void* userParam) { - char* pTopicName = NULL; - int32_t vgId = 0; - int32_t code = 0; - - if (pRes == NULL || tmq == NULL) { - pCommitFp(tmq, TSDB_CODE_INVALID_PARA, userParam); - return; - } - - if (TD_RES_TMQ(pRes)) { - SMqRspObj* pRspObj = (SMqRspObj*)pRes; - pTopicName = pRspObj->topic; - vgId = pRspObj->vgId; - } else if (TD_RES_TMQ_META(pRes)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes; - pTopicName = pMetaRspObj->topic; - vgId = pMetaRspObj->vgId; - } else if (TD_RES_TMQ_METADATA(pRes)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes; - pTopicName = pRspObj->topic; - vgId = pRspObj->vgId; - } else { - pCommitFp(tmq, TSDB_CODE_TMQ_INVALID_MSG, userParam); - return; - } - +static SMqCommitCbParamSet* prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, int32_t rspNum){ SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam); - return; + return NULL; } pParamSet->refId = tmq->refId; pParamSet->epoch = tmq->epoch; pParamSet->callbackFn = pCommitFp; pParamSet->userParam = userParam; + pParamSet->waitingRspNum = rspNum; - taosRLockLatch(&tmq->lock); - int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - - tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); + return pParamSet; +} +static SMqClientVg* getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId){ SMqClientTopic* pTopic = getTopicByName(tmq, pTopicName); if (pTopic == NULL) { - tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId, - pTopicName, numOfTopics); - taosMemoryFree(pParamSet); - pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); - taosRUnLockLatch(&tmq->lock); - return; + tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s", tmq->consumerId, pTopicName); + + return NULL; } int32_t j = 0; @@ -669,89 +573,150 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tm if (j == numOfVgroups) { tscWarn("consumer:0x%" PRIx64 " failed to find the specified vgId:%d, total Vgs:%d, topic:%s", tmq->consumerId, vgId, numOfVgroups, pTopicName); - taosMemoryFree(pParamSet); - pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); - taosRUnLockLatch(&tmq->lock); - return; + return NULL; } SMqClientVg* pVg = (SMqClientVg*)taosArrayGet(pTopic->vgs, j); - if (pVg->offsetInfo.currentOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.currentOffset, &pVg->offsetInfo.committedOffset)) { - code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups, type); + return pVg; +} - // failed to commit, callback user function directly. - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(pParamSet); - pCommitFp(tmq, code, userParam); +static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STqOffsetVal* offsetVal, tmq_commit_cb* pCommitFp, void* userParam) { + int32_t code = 0; + tscInfo("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); + taosRLockLatch(&tmq->lock); + SMqClientVg* pVg = getClientVg(tmq, pTopicName, vgId); + if(pVg == NULL){ + code = TSDB_CODE_TMQ_INVALID_VGID; + goto end; + } + if (offsetVal->type > 0 && !tOffsetEqual(offsetVal, &pVg->offsetInfo.committedOffset)) { + char offsetBuf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(offsetBuf, tListLen(offsetBuf), offsetVal); + + char commitBuf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); + + SMqCommitCbParamSet* pParamSet = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 0); + if (pParamSet == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; } - // update the offset value. - pVg->offsetInfo.committedOffset = pVg->offsetInfo.currentOffset; - } else { // do not perform commit, callback user function directly. - taosMemoryFree(pParamSet); + code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, &pVg->offsetInfo.endOffset, pTopicName, pParamSet); + if (code != TSDB_CODE_SUCCESS) { + tscError("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s", + tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno)); + taosMemoryFree(pParamSet); + goto end; + } + + tscInfo("consumer:0x%" PRIx64 " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s", + tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf); + pVg->offsetInfo.committedOffset = *offsetVal; + } + +end: + taosRUnLockLatch(&tmq->lock); + return code; +} + +static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* pCommitFp, void* userParam){ + char* pTopicName = NULL; + int32_t vgId = 0; + STqOffsetVal offsetVal = {0}; + int32_t code = 0; + + if (pRes == NULL || tmq == NULL) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + if (TD_RES_TMQ(pRes)) { + SMqRspObj* pRspObj = (SMqRspObj*)pRes; + pTopicName = pRspObj->topic; + vgId = pRspObj->vgId; + offsetVal = pRspObj->rsp.rspOffset; + } else if (TD_RES_TMQ_META(pRes)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes; + pTopicName = pMetaRspObj->topic; + vgId = pMetaRspObj->vgId; + offsetVal = pMetaRspObj->metaRsp.rspOffset; + } else if (TD_RES_TMQ_METADATA(pRes)) { + SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes; + pTopicName = pRspObj->topic; + vgId = pRspObj->vgId; + offsetVal = pRspObj->rsp.rspOffset; + } else { + code = TSDB_CODE_TMQ_INVALID_MSG; + goto end; + } + + code = asyncCommitOffset(tmq, pTopicName, vgId, &offsetVal, pCommitFp, userParam); + +end: + if(code != TSDB_CODE_SUCCESS){ pCommitFp(tmq, code, userParam); } - taosRUnLockLatch(&tmq->lock); } static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { - SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); - if (pParamSet == NULL) { - pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam); - return; - } - - pParamSet->refId = tmq->refId; - pParamSet->epoch = tmq->epoch; - pParamSet->callbackFn = pCommitFp; - pParamSet->userParam = userParam; - + int32_t code = 0; // init as 1 to prevent concurrency issue - pParamSet->waitingRspNum = 1; + SMqCommitCbParamSet* pParamSet = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 1); + if (pParamSet == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } taosRLockLatch(&tmq->lock); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); + tscInfo("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); for (int32_t i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); - tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, - numOfVgroups); + tscInfo("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, numOfVgroups); for (int32_t j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->offsetInfo.currentOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.currentOffset, &pVg->offsetInfo.committedOffset)) { - int32_t code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups, TDMT_VND_TMQ_COMMIT_OFFSET); + if (pVg->offsetInfo.endOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.endOffset, &pVg->offsetInfo.committedOffset)) { + char offsetBuf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(offsetBuf, tListLen(offsetBuf), &pVg->offsetInfo.endOffset); + + char commitBuf[TSDB_OFFSET_LEN] = {0}; + tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); + + code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, &pVg->offsetInfo.endOffset, pTopic->topicName, pParamSet); if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 " failed, code:%s ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.committedOffset.version, tstrerror(terrno), - j + 1, numOfVgroups); + tscError("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno), j + 1, numOfVgroups); continue; } - // update the offset value. - pVg->offsetInfo.committedOffset = pVg->offsetInfo.currentOffset; + tscInfo("consumer:0x%" PRIx64 " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, j + 1, numOfVgroups); + pVg->offsetInfo.committedOffset = pVg->offsetInfo.endOffset; } else { - tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.currentOffset.version, j + 1, numOfVgroups); + tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); } } } taosRUnLockLatch(&tmq->lock); - tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, - numOfTopics); + tscInfo("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, numOfTopics); - // no request is sent - if (pParamSet->totalRspNum == 0) { - taosMemoryFree(pParamSet); - pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); + // request is sent + if (pParamSet->totalRspNum != 0) { + // count down since waiting rsp num init as 1 + commitRspCountDown(pParamSet, tmq->consumerId, "", 0); return; } - // count down since waiting rsp num init as 1 - commitRspCountDown(pParamSet, tmq->consumerId, "", 0); +end: + taosMemoryFree(pParamSet); + pCommitFp(tmq, code, userParam); + return; } static void generateTimedTask(int64_t refId, int32_t type) { @@ -827,7 +792,7 @@ void tmqSendHbReq(void* param, void* tmrId) { OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); offRows->vgId = pVg->vgId; offRows->rows = pVg->numOfRows; - offRows->offset = pVg->offsetInfo.seekOffset; + offRows->offset = pVg->offsetInfo.beginOffset; char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); tscInfo("consumer:0x%" PRIx64 ",report offset: vgId:%d, offset:%s, rows:%"PRId64, tmq->consumerId, offRows->vgId, buf, offRows->rows); @@ -1523,9 +1488,9 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic .numOfRows = pInfo ? pInfo->numOfRows : 0, }; - clientVg.offsetInfo.currentOffset = pInfo ? pInfo->currentOffset : offsetNew; + clientVg.offsetInfo.endOffset = pInfo ? pInfo->currentOffset : offsetNew; clientVg.offsetInfo.committedOffset = pInfo ? pInfo->commitOffset : offsetNew; - clientVg.offsetInfo.seekOffset = pInfo ? pInfo->seekOffset : offsetNew; + clientVg.offsetInfo.beginOffset = pInfo ? pInfo->seekOffset : offsetNew; clientVg.offsetInfo.walVerBegin = -1; clientVg.offsetInfo.walVerEnd = -1; clientVg.seekUpdated = false; @@ -1581,11 +1546,11 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) makeTopicVgroupKey(vgKey, pTopicCur->topicName, pVgCur->vgId); char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.currentOffset); + tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.endOffset); tscInfo("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); - SVgroupSaveInfo info = {.currentOffset = pVgCur->offsetInfo.currentOffset, .seekOffset = pVgCur->offsetInfo.seekOffset, .commitOffset = pVgCur->offsetInfo.committedOffset, .numOfRows = pVgCur->numOfRows}; + SVgroupSaveInfo info = {.currentOffset = pVgCur->offsetInfo.endOffset, .seekOffset = pVgCur->offsetInfo.beginOffset, .commitOffset = pVgCur->offsetInfo.committedOffset, .numOfRows = pVgCur->numOfRows}; taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)); } } @@ -1682,7 +1647,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->consumerId = tmq->consumerId; pReq->timeout = timeout; pReq->epoch = tmq->epoch; - pReq->reqOffset = pVg->offsetInfo.currentOffset; + pReq->reqOffset = pVg->offsetInfo.endOffset; pReq->head.vgId = pVg->vgId; pReq->useSnapshot = tmq->useSnapshot; pReq->reqId = generateRequestId(); @@ -1809,7 +1774,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p int64_t transporterId = 0; char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.currentOffset); + tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset); tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId, pTopic->topicName, pVg->vgId, pTmq->epoch, offsetFormatBuf, req.reqId); @@ -1890,8 +1855,8 @@ static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* p static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal* rspOffset, int64_t sver, int64_t ever, int64_t consumerId){ if (!pVg->seekUpdated) { tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", consumerId); - pVg->offsetInfo.seekOffset = *reqOffset; - pVg->offsetInfo.currentOffset = *rspOffset; + pVg->offsetInfo.beginOffset = *reqOffset; + pVg->offsetInfo.endOffset = *rspOffset; } else { tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", consumerId); } @@ -2053,7 +2018,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tmq->totalRows += numOfRows; char buf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.currentOffset); + tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows, @@ -2315,7 +2280,7 @@ void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* if (pRes == NULL) { // here needs to commit all offsets. asyncCommitAllOffsets(tmq, cb, param); } else { // only commit one offset - asyncCommitOffset(tmq, pRes, TDMT_VND_TMQ_COMMIT_OFFSET, cb, param); + asyncCommitFromResult(tmq, pRes, cb, param); } } @@ -2335,7 +2300,7 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { if (pRes == NULL) { asyncCommitAllOffsets(tmq, commitCallBackFn, pInfo); } else { - asyncCommitOffset(tmq, pRes, TDMT_VND_TMQ_COMMIT_OFFSET, commitCallBackFn, pInfo); + asyncCommitFromResult(tmq, pRes, commitCallBackFn, pInfo); } tsem_wait(&pInfo->sem); @@ -2348,6 +2313,87 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { return code; } +// wal range will be ok after calling tmq_get_topic_assignment or poll interface +static bool isWalRangeOk(SVgOffsetInfo* offset){ + if (offset->walVerBegin != -1 && offset->walVerEnd != -1) { + return true; + } + return false; +} + +int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset){ + if (tmq == NULL || pTopicName == NULL) { + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } + + int32_t accId = tmq->pTscObj->acctId; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + taosWLockLatch(&tmq->lock); + SMqClientTopic* pTopic = getTopicByName(tmq, tname); + if (pTopic == NULL) { + tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_INVALID_TOPIC; + } + + SMqClientVg* pVg = NULL; + int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); + for (int32_t i = 0; i < numOfVgs; ++i) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); + if (pClientVg->vgId == vgId) { + pVg = pClientVg; + break; + } + } + + if (pVg == NULL) { + tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_INVALID_VGID; + } + + SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; + if (!isWalRangeOk(pOffsetInfo)) { + tscError("consumer:0x%" PRIx64 " Assignment or poll interface need to be called first", tmq->consumerId); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_NEED_INITIALIZED; + } + + if (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd) { + tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", + tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE; + } + taosWUnLockLatch(&tmq->lock); + + STqOffsetVal offsetVal = {.type = TMQ_OFFSET__LOG, .version = offset}; + + SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); + if (pInfo == NULL) { + tscError("consumer:0x%"PRIx64" failed to prepare seek operation", tmq->consumerId); + return TSDB_CODE_OUT_OF_MEMORY; + } + + tsem_init(&pInfo->sem, 0, 0); + pInfo->code = 0; + + asyncCommitOffset(tmq, tname, vgId, &offsetVal, commitCallBackFn, pInfo); + + tsem_wait(&pInfo->sem); + int32_t code = pInfo->code; + + tsem_destroy(&pInfo->sem); + taosMemoryFree(pInfo); + + tscInfo("consumer:0x%" PRIx64 " send seek to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); + + return code; +} + void updateEpCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) { SAskEpInfo* pInfo = param; pInfo->code = code; @@ -2490,12 +2536,10 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); if (waitingRspNum == 0) { - tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, - vgId); + tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, vgId); tmqCommitDone(pParamSet); } else { - tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, - waitingRspNum); + tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, waitingRspNum); } } @@ -2578,6 +2622,69 @@ static bool isInSnapshotMode(int8_t type, bool useSnapshot){ return false; } +int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId){ + if (tmq == NULL) { + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } + + int32_t accId = tmq->pTscObj->acctId; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + taosWLockLatch(&tmq->lock); + SMqClientTopic* pTopic = getTopicByName(tmq, tname); + if (pTopic == NULL) { + tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_INVALID_TOPIC; + } + + SMqClientVg* pVg = NULL; + int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); + for (int32_t i = 0; i < numOfVgs; ++i) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); + if (pClientVg->vgId == vgId) { + pVg = pClientVg; + break; + } + } + + if (pVg == NULL) { + tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_INVALID_VGID; + } + + int32_t type = pVg->offsetInfo.endOffset.type; + if (isInSnapshotMode(type, tmq->useSnapshot)) { + tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, position error", tmq->consumerId, type); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_SNAPSHOT_ERROR; + } + + if (!isWalRangeOk(&pVg->offsetInfo)) { + tscError("consumer:0x%" PRIx64 " Assignment or poll interface need to be called first", tmq->consumerId); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_NEED_INITIALIZED; + } + + int64_t position = 0; + STqOffsetVal* pOffsetInfo = &pVg->offsetInfo.endOffset; + if(type == TMQ_OFFSET__LOG){ + position = pOffsetInfo->version; + }else if(type == TMQ_OFFSET__RESET_EARLIEST){ + position = pVg->offsetInfo.walVerBegin; + }else if(type == TMQ_OFFSET__RESET_LATEST){ + position = pVg->offsetInfo.walVerEnd; + }else{ + tscError("consumer:0x%" PRIx64 " offset type:%d can not be reach here", tmq->consumerId, type); + } + taosWUnLockLatch(&tmq->lock); + + return position; +} + int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment, int32_t* numOfAssignment) { *numOfAssignment = 0; @@ -2585,7 +2692,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a SMqVgCommon* pCommon = NULL; int32_t accId = tmq->pTscObj->acctId; - char tname[128] = {0}; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; sprintf(tname, "%d.%s", accId, pTopicName); int32_t code = TSDB_CODE_SUCCESS; @@ -2600,7 +2707,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a *numOfAssignment = taosArrayGetSize(pTopic->vgs); for (int32_t j = 0; j < (*numOfAssignment); ++j) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); - int32_t type = pClientVg->offsetInfo.seekOffset.type; + int32_t type = pClientVg->offsetInfo.beginOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type); code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; @@ -2620,13 +2727,13 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a for (int32_t j = 0; j < (*numOfAssignment); ++j) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); - if (pClientVg->offsetInfo.seekOffset.type != TMQ_OFFSET__LOG) { + if (pClientVg->offsetInfo.beginOffset.type != TMQ_OFFSET__LOG) { needFetch = true; break; } tmq_topic_assignment* pAssignment = &(*assignment)[j]; - pAssignment->currentOffset = pClientVg->offsetInfo.seekOffset.version; + pAssignment->currentOffset = pClientVg->offsetInfo.beginOffset.version; pAssignment->begin = pClientVg->offsetInfo.walVerBegin; pAssignment->end = pClientVg->offsetInfo.walVerEnd; pAssignment->vgId = pClientVg->vgId; @@ -2665,7 +2772,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a SMqPollReq req = {0}; tmqBuildConsumeReqImpl(&req, tmq, 10, pTopic, pClientVg); - req.reqOffset = pClientVg->offsetInfo.seekOffset; + req.reqOffset = pClientVg->offsetInfo.beginOffset; int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req); if (msgSize < 0) { @@ -2705,7 +2812,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a int64_t transporterId = 0; char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; - tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.seekOffset); + tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.beginOffset); tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); @@ -2780,7 +2887,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ } int32_t accId = tmq->pTscObj->acctId; - char tname[128] = {0}; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; sprintf(tname, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); @@ -2809,14 +2916,20 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; - int32_t type = pOffsetInfo->currentOffset.type; + int32_t type = pOffsetInfo->endOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } - if (type == TMQ_OFFSET__LOG && (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd)) { + if (!isWalRangeOk(&pVg->offsetInfo)) { + tscError("consumer:0x%" PRIx64 " Assignment or poll interface need to be called first", tmq->consumerId); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_NEED_INITIALIZED; + } + + if (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd) { tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd); taosWUnLockLatch(&tmq->lock); @@ -2824,10 +2937,9 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ } // update the offset, and then commit to vnode - pOffsetInfo->currentOffset.type = TMQ_OFFSET__LOG; - pOffsetInfo->currentOffset.version = offset; - pOffsetInfo->seekOffset = pOffsetInfo->currentOffset; -// pOffsetInfo->committedOffset.version = INT64_MIN; + pOffsetInfo->endOffset.type = TMQ_OFFSET__LOG; + pOffsetInfo->endOffset.version = offset; + pOffsetInfo->beginOffset = pOffsetInfo->endOffset; pVg->seekUpdated = true; tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, vgId); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 8231fad3a7..c36480a63e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -631,6 +631,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SCALAR_CONVERT_ERROR, "Cannot convert to s //tmq TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NEED_INITIALIZED, "Assignment or poll interface need to be called first") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_SNAPSHOT_ERROR, "Can not operate in snapshot mode") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE, "Offset out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_VGID, "VgId does not belong to this consumer") From de4efca56f283e7ce0488410eed590649e2f0ff8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 18 Jul 2023 12:12:56 +0000 Subject: [PATCH 053/100] update ip --- source/libs/transport/src/transCli.c | 53 ++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8062a0618b..01223a2be9 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -73,7 +73,7 @@ typedef struct SCliConn { SDelayTask* task; - char* ip; + char* dstAddr; char src[32]; char dst[32]; @@ -196,6 +196,7 @@ static FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp); static FORCE_INLINE uint32_t cliGetIpFromFqdnCache(SHashObj* cache, char* fqdn); static FORCE_INLINE void cliUpdateFqdnCache(SHashObj* cache, char* fqdn); +static FORCE_INLINE void cliMayUpdateFqdnCache(SHashObj* cache, char* dst); // process data read from server, add decompress etc later static void cliHandleResp(SCliConn* conn); // handle except about conn @@ -543,6 +544,7 @@ void cliConnTimeout(uv_timer_t* handle) { taosArrayPush(pThrd->timerList, &conn->timer); conn->timer = NULL; + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); cliHandleFastFail(conn, UV_ECANCELED); } void cliReadTimeoutCb(uv_timer_t* handle) { @@ -719,7 +721,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { cliDestroyConnMsgs(conn, false); if (conn->list == NULL) { - conn->list = taosHashGet((SHashObj*)pool, conn->ip, strlen(conn->ip) + 1); + conn->list = taosHashGet((SHashObj*)pool, conn->dstAddr, strlen(conn->dstAddr) + 1); } SConnList* pList = conn->list; @@ -878,7 +880,7 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { connList->list->numOfConn--; connList->size--; } else { - SConnList* connList = taosHashGet((SHashObj*)pThrd->pool, conn->ip, strlen(conn->ip) + 1); + SConnList* connList = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr) + 1); if (connList != NULL) connList->list->numOfConn--; } conn->list = NULL; @@ -923,7 +925,7 @@ static void cliDestroy(uv_handle_t* handle) { transReleaseExHandle(transGetRefMgt(), conn->refId); transRemoveExHandle(transGetRefMgt(), conn->refId); - taosMemoryFree(conn->ip); + taosMemoryFree(conn->dstAddr); taosMemoryFree(conn->stream); cliDestroyConnMsgs(conn, true); @@ -1168,7 +1170,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { if (conn == NULL) { conn = cliCreateConn(pThrd); conn->pBatch = pBatch; - conn->ip = taosStrdup(pList->dst); + conn->dstAddr = taosStrdup(pList->dst); uint32_t ipaddr = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, pList->ip); if (ipaddr == 0xffffffff) { @@ -1213,6 +1215,8 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { conn->timer->data = NULL; taosArrayPush(pThrd->timerList, &conn->timer); conn->timer = NULL; + + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); cliHandleFastFail(conn, -1); return; } @@ -1271,11 +1275,11 @@ static void cliHandleFastFail(SCliConn* pConn, int status) { STraceId* trace = &pMsg->msg.info.traceId; tGError("%s msg %s failed to send, conn %p failed to connect to %s, reason: %s", CONN_GET_INST_LABEL(pConn), - TMSG_INFO(pMsg->msg.msgType), pConn, pConn->ip, uv_strerror(status)); + TMSG_INFO(pMsg->msg.msgType), pConn, pConn->dstAddr, uv_strerror(status)); if (pMsg != NULL && REQUEST_NO_RESP(&pMsg->msg) && (pTransInst->failFastFp != NULL && pTransInst->failFastFp(pMsg->msg.msgType))) { - SFailFastItem* item = taosHashGet(pThrd->failFastCache, pConn->ip, strlen(pConn->ip) + 1); + SFailFastItem* item = taosHashGet(pThrd->failFastCache, pConn->dstAddr, strlen(pConn->dstAddr) + 1); int64_t cTimestamp = taosGetTimestampMs(); if (item != NULL) { int32_t elapse = cTimestamp - item->timestamp; @@ -1287,12 +1291,12 @@ static void cliHandleFastFail(SCliConn* pConn, int status) { } } else { SFailFastItem item = {.count = 1, .timestamp = cTimestamp}; - taosHashPut(pThrd->failFastCache, pConn->ip, strlen(pConn->ip) + 1, &item, sizeof(SFailFastItem)); + taosHashPut(pThrd->failFastCache, pConn->dstAddr, strlen(pConn->dstAddr) + 1, &item, sizeof(SFailFastItem)); } } } else { tError("%s batch msg failed to send, conn %p failed to connect to %s, reason: %s", CONN_GET_INST_LABEL(pConn), - pConn, pConn->ip, uv_strerror(status)); + pConn, pConn->dstAddr, uv_strerror(status)); cliDestroyBatch(pConn->pBatch); pConn->pBatch = NULL; } @@ -1314,6 +1318,7 @@ void cliConnCb(uv_connect_t* req, int status) { } if (status != 0) { + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, pConn->dstAddr); if (timeout == false) { cliHandleFastFail(pConn, status); } else if (timeout == true) { @@ -1483,9 +1488,34 @@ static FORCE_INLINE uint32_t cliGetIpFromFqdnCache(SHashObj* cache, char* fqdn) } static FORCE_INLINE void cliUpdateFqdnCache(SHashObj* cache, char* fqdn) { // impl later + uint32_t addr = taosGetIpv4FromFqdn(fqdn); + if (addr != 0xffffffff) { + uint32_t* v = taosHashGet(cache, fqdn, strlen(fqdn) + 1); + if (addr != *v) { + char old[64] = {0}, new[64] = {0}; + tinet_ntoa(old, *v); + tinet_ntoa(new, addr); + tWarn("update ip of fqdn:%s, old: %s, new: %s", fqdn, old, new); + taosHashPut(cache, fqdn, strlen(fqdn) + 1, &addr, sizeof(addr)); + } + } return; } +static void cliMayUpdateFqdnCache(SHashObj* cache, char* dst) { + if (dst == NULL) return; + + int16_t i = 0, len = strlen(dst); + for (i = len - 1; i >= 0; i--) { + if (dst[i] == ':') break; + } + if (i > 0) { + char fqdn[TSDB_FQDN_LEN + 1] = {0}; + memcpy(fqdn, dst, i); + cliUpdateFqdnCache(cache, fqdn); + } +} + static void doFreeTimeoutMsg(void* param) { STaskArg* arg = param; SCliMsg* pMsg = arg->param1; @@ -1560,7 +1590,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { transCtxMerge(&conn->ctx, &pMsg->ctx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); - conn->ip = taosStrdup(addr); + conn->dstAddr = taosStrdup(addr); uint32_t ipaddr = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, fqdn); if (ipaddr == 0xffffffff) { @@ -1578,7 +1608,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { addr.sin_addr.s_addr = ipaddr; addr.sin_port = (uint16_t)htons(port); - tGTrace("%s conn %p try to connect to %s", pTransInst->label, conn, conn->ip); + tGTrace("%s conn %p try to connect to %s", pTransInst->label, conn, conn->dstAddr); pThrd->newConnCount++; int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT * 4); if (fd == -1) { @@ -1608,6 +1638,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { taosArrayPush(pThrd->timerList, &conn->timer); conn->timer = NULL; + cliMayUpdateFqdnCache(pThrd->fqdn2ipCache, conn->dstAddr); cliHandleFastFail(conn, ret); return; } From 79620c99a594ea392ab3c1730095c221d124285a Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 18 Jul 2023 20:18:43 +0800 Subject: [PATCH 054/100] test: del db and max topics invalid --- tests/system-test/7-tmq/tmqMaxTopic.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/system-test/7-tmq/tmqMaxTopic.py b/tests/system-test/7-tmq/tmqMaxTopic.py index 2371d53cc5..1bf20dfbd8 100644 --- a/tests/system-test/7-tmq/tmqMaxTopic.py +++ b/tests/system-test/7-tmq/tmqMaxTopic.py @@ -216,6 +216,32 @@ class TDTestCase: tdLog.info("create topic sql: %s"%sqlString) tdSql.error(sqlString) + tdLog.info("drop database when there are topic") + sqlString = "drop database %s" %(paraDict['dbName']) + tdLog.info("drop database sql: %s"%sqlString) + tdSql.error(sqlString) + + tdLog.info("drop all topic for re-create") + tdSql.query('show topics;') + topicNum = tdSql.queryRows + tdLog.info(" topic count: %d"%(topicNum)) + for i in range(topicNum): + sqlString = "drop topic %s" %(tdSql.getData(i, 0)) + tdLog.info("drop topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + time.sleep(1) + + tdLog.info("re-create topics") + topicNamePrefix = 'newTopic_' + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + for i in range(topicNum): + sqlString = "create topic %s%d as %s" %(topicNamePrefix, i, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + #=================================================# + tdLog.info("drop all topic for testcase2") tdSql.query('show topics;') topicNum = tdSql.queryRows tdLog.info(" topic count: %d"%(topicNum)) From 8f464bd21f380ce7e492e60a900678565472d215 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Wed, 19 Jul 2023 09:42:10 +0800 Subject: [PATCH 055/100] docs: correct the description of auto.offset.reset --- docs/en/07-develop/07-tmq.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 506a8dcc46..ccf39ef581 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -298,7 +298,7 @@ You configure the following parameters when creating a consumer: | `td.connect.port` | string | Port of the server side | | | `group.id` | string | Consumer group ID; consumers with the same ID are in the same group | **Required**. Maximum length: 192. Each topic can create up to 100 consumer groups. | | `client.id` | string | Client ID | Maximum length: 192. | -| `auto.offset.reset` | enum | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) | +| `auto.offset.reset` | enum | Initial offset for the consumer group | `earliest`: subscribe from the earliest data, this is the default behavior; `latest`: subscribe from the latest data; or `none`: can't subscribe without committed offset| | `enable.auto.commit` | boolean | Commit automatically; true: user application doesn't need to explicitly commit; false: user application need to handle commit by itself | Default value is true | | `auto.commit.interval.ms` | integer | Interval for automatic commits, in milliseconds | | `msg.with.table.name` | boolean | Specify whether to deserialize table names from messages | default value: false From 4cdfd0e1e8be1d6ed556fe3638efeb542a7c5eb0 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 19 Jul 2023 10:50:46 +0800 Subject: [PATCH 056/100] conflict --- source/dnode/mnode/impl/src/mndUser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 65393399d5..06523c7c9b 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -620,7 +620,7 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - if (createReq.user[0] == 0) { + if (createReq.user[0] == 0 || strlen(createReq.pass) >= TSDB_PASSWORD_LEN) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; goto _OVER; } From 8f709ea7829173345c7cccc505643327f927d022 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 19 Jul 2023 13:06:59 +0800 Subject: [PATCH 057/100] fix test case --- tests/system-test/1-insert/boundary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/boundary.py b/tests/system-test/1-insert/boundary.py index 29dcbc7c46..4476236ca6 100644 --- a/tests/system-test/1-insert/boundary.py +++ b/tests/system-test/1-insert/boundary.py @@ -33,7 +33,7 @@ class TDTestCase: self.colname_length_boundary = self.boundary.COL_KEY_MAX_LENGTH self.tagname_length_boundary = self.boundary.TAG_KEY_MAX_LENGTH self.username_length_boundary = 23 - self.password_length_boundary = 128 + self.password_length_boundary = 31 def dbname_length_check(self): dbname_length = randint(1,self.dbname_length_boundary-1) for dbname in [tdCom.get_long_name(self.dbname_length_boundary),tdCom.get_long_name(dbname_length)]: From 69fae6200387df5d79f970864658e9d5373405be Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 19 Jul 2023 15:04:39 +0800 Subject: [PATCH 058/100] error message --- source/dnode/mnode/impl/src/mndUser.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 06523c7c9b..cbde56a860 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -620,7 +620,7 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - if (createReq.user[0] == 0 || strlen(createReq.pass) >= TSDB_PASSWORD_LEN) { + if (createReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; goto _OVER; } @@ -630,6 +630,11 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } + if (strlen(createReq.pass) >= TSDB_PASSWORD_LEN){ + terrno = TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG; + goto _OVER; + } + pUser = mndAcquireUser(pMnode, createReq.user); if (pUser != NULL) { terrno = TSDB_CODE_MND_USER_ALREADY_EXIST; From 5cb35f2fa6389e8003287d35a0a02421da1c6ebc Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 19 Jul 2023 17:29:39 +0800 Subject: [PATCH 059/100] feat:add committed & position & commite_offset interface --- include/client/taos.h | 2 +- include/common/tmsgdef.h | 1 + include/util/taoserror.h | 1 + source/client/src/clientRawBlockWrite.c | 7 + source/client/src/clientTmq.c | 468 +++++++++++++------- source/client/test/clientTests.cpp | 83 ++++ source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tq/tq.c | 43 ++ source/dnode/vnode/src/vnd/vnodeSvr.c | 4 +- source/util/src/terror.c | 1 + 11 files changed, 461 insertions(+), 151 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 3cc2d907ab..5ea1510e44 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -288,7 +288,7 @@ DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param); DLL_EXPORT int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset); -DLL_EXPORT void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param); +DLL_EXPORT int32_t tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param); DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment, int32_t *numOfAssignment); DLL_EXPORT void tmq_free_assignment(tmq_topic_assignment* pAssignment); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 3f4335af94..aa23442291 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -312,6 +312,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME, "vnode-tmq-consume", SMqPollReq, SMqDataBlkRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME_PUSH, "vnode-tmq-consume-push", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committed-walinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index d6f44f4489..f37402c18c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -779,6 +779,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TMQ_INVALID_VGID TAOS_DEF_ERROR_CODE(0, 0x4008) #define TSDB_CODE_TMQ_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x4009) #define TSDB_CODE_TMQ_NEED_INITIALIZED TAOS_DEF_ERROR_CODE(0, 0x4010) +#define TSDB_CODE_TMQ_NO_COMMITTED TAOS_DEF_ERROR_CODE(0, 0x4011) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 90b10e0920..dd311db126 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1327,6 +1327,9 @@ end: int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields, int numFields) { + if (!taos || !pData || !tbname) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; @@ -1413,6 +1416,9 @@ end: } int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) { + if (!taos || !pData || !tbname) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; @@ -1812,6 +1818,7 @@ end: } char* tmq_get_json_meta(TAOS_RES* res) { + if (res == NULL) return NULL; uDebug("tmq_get_json_meta called"); if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res)) { return NULL; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 071b5d6b0f..f2ea7309e4 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -220,6 +220,12 @@ typedef struct SMqSeekParam { int32_t code; } SMqSeekParam; +typedef struct SMqCommittedParam { + tsem_t sem; + int32_t code; + SMqVgOffset vgOffset; +} SMqCommittedParam; + typedef struct SMqVgWalInfoParam { int32_t vgId; int32_t epoch; @@ -241,7 +247,7 @@ typedef struct { typedef struct { SMqCommitCbParamSet* params; - SMqVgOffset* pOffset; +// SMqVgOffset* pOffset; char topicName[TSDB_TOPIC_FNAME_LEN]; int32_t vgId; tmq_t* pTmq; @@ -292,6 +298,9 @@ void tmq_conf_destroy(tmq_conf_t* conf) { } tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { + if (conf == NULL || key == NULL || value == NULL){ + return TMQ_CONF_INVALID; + } if (strcasecmp(key, "group.id") == 0) { tstrncpy(conf->groupId, value, TSDB_CGROUP_LEN); return TMQ_CONF_OK; @@ -406,6 +415,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value tmq_list_t* tmq_list_new() { return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); } int32_t tmq_list_append(tmq_list_t* list, const char* src) { + if(list == NULL) return -1; SArray* container = &list->container; if (src == NULL || src[0] == 0) return -1; char* topic = taosStrdup(src); @@ -414,16 +424,19 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { } void tmq_list_destroy(tmq_list_t* list) { + if(list == NULL) return; SArray* container = &list->container; taosArrayDestroyP(container, taosMemoryFree); } int32_t tmq_list_get_size(const tmq_list_t* list) { + if(list == NULL) return -1; const SArray* container = &list->container; return taosArrayGetSize(container); } char** tmq_list_to_c_array(const tmq_list_t* list) { + if(list == NULL) return NULL; const SArray* container = &list->container; return container->pData; } @@ -432,7 +445,7 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; - taosMemoryFree(pParam->pOffset); +// taosMemoryFree(pParam->pOffset); taosMemoryFree(pBuf->pData); taosMemoryFree(pBuf->pEpSet); @@ -441,30 +454,25 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { } static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, SMqCommitCbParamSet* pParamSet) { - SMqVgOffset* pOffset = taosMemoryCalloc(1, sizeof(SMqVgOffset)); - if (pOffset == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + SMqVgOffset pOffset = {0}; - pOffset->consumerId = tmq->consumerId; - pOffset->offset.val = *offset; + pOffset.consumerId = tmq->consumerId; + pOffset.offset.val = *offset; int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset->offset.subKey, tmq->groupId, groupLen); - pOffset->offset.subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset->offset.subKey + groupLen + 1, pTopicName); + memcpy(pOffset.offset.subKey, tmq->groupId, groupLen); + pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR; + strcpy(pOffset.offset.subKey + groupLen + 1, pTopicName); int32_t len = 0; int32_t code = 0; - tEncodeSize(tEncodeMqVgOffset, pOffset, len, code); + tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code); if (code < 0) { - taosMemoryFree(pOffset); return TSDB_CODE_INVALID_PARA; } void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); if (buf == NULL) { - taosMemoryFree(pOffset); return TSDB_CODE_OUT_OF_MEMORY; } @@ -474,19 +482,18 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse SEncoder encoder; tEncoderInit(&encoder, abuf, len); - tEncodeMqVgOffset(&encoder, pOffset); + tEncodeMqVgOffset(&encoder, &pOffset); tEncoderClear(&encoder); // build param SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); if (pParam == NULL) { - taosMemoryFree(pOffset); taosMemoryFree(buf); return TSDB_CODE_OUT_OF_MEMORY; } pParam->params = pParamSet; - pParam->pOffset = pOffset; +// pParam->pOffset = pOffset; pParam->vgId = vgId; pParam->pTmq = tmq; @@ -495,7 +502,6 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse // build send info SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { - taosMemoryFree(pOffset); taosMemoryFree(buf); taosMemoryFree(pParam); return TSDB_CODE_OUT_OF_MEMORY; @@ -553,40 +559,34 @@ static SMqCommitCbParamSet* prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* p return pParamSet; } -static SMqClientVg* getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId){ + + +static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClientVg** pVg){ SMqClientTopic* pTopic = getTopicByName(tmq, pTopicName); if (pTopic == NULL) { - tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s", tmq->consumerId, pTopicName); - - return NULL; + tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); + return TSDB_CODE_TMQ_INVALID_TOPIC; } - int32_t j = 0; - int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); - for (j = 0; j < numOfVgroups; j++) { - SMqClientVg* pVg = (SMqClientVg*)taosArrayGet(pTopic->vgs, j); - if (pVg->vgId == vgId) { + int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); + for (int32_t i = 0; i < numOfVgs; ++i) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); + if (pClientVg->vgId == vgId) { + *pVg = pClientVg; break; } } - if (j == numOfVgroups) { - tscWarn("consumer:0x%" PRIx64 " failed to find the specified vgId:%d, total Vgs:%d, topic:%s", tmq->consumerId, - vgId, numOfVgroups, pTopicName); - return NULL; - } - - SMqClientVg* pVg = (SMqClientVg*)taosArrayGet(pTopic->vgs, j); - return pVg; + return *pVg == NULL ? TSDB_CODE_TMQ_INVALID_VGID : TSDB_CODE_SUCCESS; } static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STqOffsetVal* offsetVal, tmq_commit_cb* pCommitFp, void* userParam) { int32_t code = 0; tscInfo("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); taosRLockLatch(&tmq->lock); - SMqClientVg* pVg = getClientVg(tmq, pTopicName, vgId); - if(pVg == NULL){ - code = TSDB_CODE_TMQ_INVALID_VGID; + SMqClientVg* pVg = NULL; + code = getClientVg(tmq, pTopicName, vgId, &pVg); + if(code != 0){ goto end; } if (offsetVal->type > 0 && !tOffsetEqual(offsetVal, &pVg->offsetInfo.committedOffset)) { @@ -601,7 +601,7 @@ static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STq code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, &pVg->offsetInfo.endOffset, pTopicName, pParamSet); + code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, offsetVal, pTopicName, pParamSet); if (code != TSDB_CODE_SUCCESS) { tscError("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s", tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno)); @@ -964,6 +964,7 @@ int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { } int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { + if(tmq == NULL) return TSDB_CODE_INVALID_PARA; if (*topics == NULL) { *topics = tmq_list_new(); } @@ -977,6 +978,7 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { } int32_t tmq_unsubscribe(tmq_t* tmq) { + if(tmq == NULL) return TSDB_CODE_INVALID_PARA; if (tmq->autoCommit) { int32_t rsp = tmq_commit_sync(tmq, NULL); if (rsp != 0) { @@ -1047,6 +1049,7 @@ static void tmqMgmtInit(void) { } tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { + if(conf == NULL) return NULL; taosThreadOnce(&tmqInit, tmqMgmtInit); if (tmqInitRes != 0) { terrno = tmqInitRes; @@ -1140,6 +1143,7 @@ _failed: } int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { + if(tmq == NULL) return TSDB_CODE_INVALID_PARA; const int32_t MAX_RETRY_COUNT = 120 * 2; // let's wait for 2 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); @@ -1264,6 +1268,7 @@ FAIL: } void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* param) { + if(conf == NULL) return; conf->commitCb = cb; conf->commitCbUserParam = param; } @@ -2050,6 +2055,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { + if(tmq == NULL) return NULL; + void* rspObj; int64_t startTime = taosGetTimestampMs(); @@ -2129,6 +2136,8 @@ static void displayConsumeStatistics(tmq_t* pTmq) { } int32_t tmq_consumer_close(tmq_t* tmq) { + if(tmq == NULL) return TSDB_CODE_INVALID_PARA; + tscInfo("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); displayConsumeStatistics(tmq); @@ -2174,6 +2183,9 @@ const char* tmq_err2str(int32_t err) { } tmq_res_t tmq_get_res_type(TAOS_RES* res) { + if (res == NULL){ + return TMQ_RES_INVALID; + } if (TD_RES_TMQ(res)) { return TMQ_RES_DATA; } else if (TD_RES_TMQ_META(res)) { @@ -2186,6 +2198,9 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) { } const char* tmq_get_topic_name(TAOS_RES* res) { + if (res == NULL){ + return NULL; + } if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; return strchr(pRspObj->topic, '.') + 1; @@ -2201,6 +2216,10 @@ const char* tmq_get_topic_name(TAOS_RES* res) { } const char* tmq_get_db_name(TAOS_RES* res) { + if (res == NULL){ + return NULL; + } + if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; return strchr(pRspObj->db, '.') + 1; @@ -2216,6 +2235,9 @@ const char* tmq_get_db_name(TAOS_RES* res) { } int32_t tmq_get_vgroup_id(TAOS_RES* res) { + if (res == NULL){ + return -1; + } if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; return pRspObj->vgId; @@ -2231,6 +2253,9 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { } int64_t tmq_get_vgroup_offset(TAOS_RES* res) { + if (res == NULL){ + return TSDB_CODE_INVALID_PARA; + } if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*) res; STqOffsetVal* pOffset = &pRspObj->rsp.reqOffset; @@ -2254,10 +2279,13 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { } // data from tsdb, no valid offset info - return -1; + return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } const char* tmq_get_table_name(TAOS_RES* res) { + if (res == NULL){ + return NULL; + } if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 || @@ -2277,6 +2305,10 @@ const char* tmq_get_table_name(TAOS_RES* res) { } void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* param) { + if (tmq == NULL) { + tscError("invalid tmq handle, null"); + return; + } if (pRes == NULL) { // here needs to commit all offsets. asyncCommitAllOffsets(tmq, cb, param); } else { // only commit one offset @@ -2291,6 +2323,11 @@ static void commitCallBackFn(tmq_t *UNUSED_PARAM(tmq), int32_t code, void* param } int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { + if (tmq == NULL) { + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } + int32_t code = 0; SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); @@ -2314,11 +2351,18 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { } // wal range will be ok after calling tmq_get_topic_assignment or poll interface -static bool isWalRangeOk(SVgOffsetInfo* offset){ - if (offset->walVerBegin != -1 && offset->walVerEnd != -1) { - return true; +static int32_t checkWalRange(SVgOffsetInfo* offset, int64_t value){ + if (offset->walVerBegin == -1 || offset->walVerEnd == -1) { + tscError("Assignment or poll interface need to be called first"); + return TSDB_CODE_TMQ_NEED_INITIALIZED; } - return false; + + if (value != -1 && (value < offset->walVerBegin || value > offset->walVerEnd)) { + tscError("invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", value, offset->walVerBegin, offset->walVerEnd); + return TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE; + } + + return 0; } int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset){ @@ -2332,41 +2376,18 @@ int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, sprintf(tname, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); - SMqClientTopic* pTopic = getTopicByName(tmq, tname); - if (pTopic == NULL) { - tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); - taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_TMQ_INVALID_TOPIC; - } - SMqClientVg* pVg = NULL; - int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); - for (int32_t i = 0; i < numOfVgs; ++i) { - SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); - if (pClientVg->vgId == vgId) { - pVg = pClientVg; - break; - } - } - - if (pVg == NULL) { - tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId); + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if(code != 0){ taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_TMQ_INVALID_VGID; + return code; } SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; - if (!isWalRangeOk(pOffsetInfo)) { - tscError("consumer:0x%" PRIx64 " Assignment or poll interface need to be called first", tmq->consumerId); + code = checkWalRange(pOffsetInfo, offset); + if (code != 0) { taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_TMQ_NEED_INITIALIZED; - } - - if (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd) { - tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", - tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd); - taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE; + return code; } taosWUnLockLatch(&tmq->lock); @@ -2384,7 +2405,7 @@ int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, asyncCommitOffset(tmq, tname, vgId, &offsetVal, commitCallBackFn, pInfo); tsem_wait(&pInfo->sem); - int32_t code = pInfo->code; + code = pInfo->code; tsem_destroy(&pInfo->sem); taosMemoryFree(pInfo); @@ -2394,6 +2415,41 @@ int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, return code; } +int32_t tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param){ + if (tmq == NULL || pTopicName == NULL) { + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } + + int32_t accId = tmq->pTscObj->acctId; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = NULL; + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if(code != 0){ + taosWUnLockLatch(&tmq->lock); + return code; + } + + SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; + code = checkWalRange(pOffsetInfo, offset); + if (code != 0) { + taosWUnLockLatch(&tmq->lock); + return code; + } + taosWUnLockLatch(&tmq->lock); + + STqOffsetVal offsetVal = {.type = TMQ_OFFSET__LOG, .version = offset}; + + code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, cb, param); + + tscInfo("consumer:0x%" PRIx64 " send seek to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); + + return code; +} + void updateEpCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) { SAskEpInfo* pInfo = param; pInfo->code = code; @@ -2525,7 +2581,10 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { } // if no more waiting rsp - pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); + if(pParamSet->callbackFn != NULL){ + pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); + } + taosMemoryFree(pParamSet); // tmq->needReportOffsetRows = true; @@ -2622,8 +2681,104 @@ static bool isInSnapshotMode(int8_t type, bool useSnapshot){ return false; } +static int32_t tmCommittedCb(void* param, SDataBuf* pMsg, int32_t code) { + SMqCommittedParam* pParam = param; + + if (code != 0){ + goto end; + } + if (pMsg) { + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)pMsg->pData, pMsg->len); + if (tDecodeMqVgOffset(&decoder, &pParam->vgOffset) < 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + tDecoderClear(&decoder); + } + + end: + if(pMsg){ + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + } + pParam->code = code; + tsem_post(&pParam->sem); + return 0; +} + +int64_t getCommittedFromServer(tmq_t *tmq, char* tname, int32_t vgId, SEpSet* epSet){ + int32_t code = 0; + SMqVgOffset pOffset = {0}; + + pOffset.consumerId = tmq->consumerId; + + int32_t groupLen = strlen(tmq->groupId); + memcpy(pOffset.offset.subKey, tmq->groupId, groupLen); + pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR; + strcpy(pOffset.offset.subKey + groupLen + 1, tname); + + int32_t len = 0; + tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code); + if (code < 0) { + return TSDB_CODE_INVALID_PARA; + } + + void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + ((SMsgHead*)buf)->vgId = htonl(vgId); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + tEncodeMqVgOffset(&encoder, &pOffset); + tEncoderClear(&encoder); + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(buf); + return TSDB_CODE_OUT_OF_MEMORY; + } + + SMqCommittedParam* pParam = taosMemoryMalloc(sizeof(SMqCommittedParam)); + if (pParam == NULL) { + taosMemoryFree(buf); + taosMemoryFree(sendInfo); + return TSDB_CODE_OUT_OF_MEMORY; + } + tsem_init(&pParam->sem, 0, 0); + + sendInfo->msgInfo = (SDataBuf){.pData = buf, .len = sizeof(SMsgHead) + len, .handle = NULL}; + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmCommittedCb; + sendInfo->msgType = TDMT_VND_TMQ_VG_COMMITTEDINFO; + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, sendInfo); + + tsem_wait(&pParam->sem); + code = pParam->code; + if(code == TSDB_CODE_SUCCESS){ + if(pParam->vgOffset.offset.val.type == TMQ_OFFSET__LOG){ + code = pParam->vgOffset.offset.val.version; + }else{ + code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; + } + } + tsem_destroy(&pParam->sem); + taosMemoryFree(pParam); + + return code; +} + int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId){ - if (tmq == NULL) { + if (tmq == NULL || pTopicName == NULL) { tscError("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -2633,60 +2788,103 @@ int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId){ sprintf(tname, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); - SMqClientTopic* pTopic = getTopicByName(tmq, tname); - if (pTopic == NULL) { - tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); - taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_TMQ_INVALID_TOPIC; - } SMqClientVg* pVg = NULL; - int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); - for (int32_t i = 0; i < numOfVgs; ++i) { - SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); - if (pClientVg->vgId == vgId) { - pVg = pClientVg; - break; - } - } - - if (pVg == NULL) { - tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId); + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if(code != 0){ taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_TMQ_INVALID_VGID; + return code; } - int32_t type = pVg->offsetInfo.endOffset.type; + SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; + int32_t type = pOffsetInfo->endOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, position error", tmq->consumerId, type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } - if (!isWalRangeOk(&pVg->offsetInfo)) { - tscError("consumer:0x%" PRIx64 " Assignment or poll interface need to be called first", tmq->consumerId); + code = checkWalRange(pOffsetInfo, -1); + if (code != 0) { taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_TMQ_NEED_INITIALIZED; + return code; } + SEpSet epSet = pVg->epSet; + int64_t begin = pVg->offsetInfo.walVerBegin; + int64_t end = pVg->offsetInfo.walVerEnd; + taosWUnLockLatch(&tmq->lock); int64_t position = 0; - STqOffsetVal* pOffsetInfo = &pVg->offsetInfo.endOffset; if(type == TMQ_OFFSET__LOG){ - position = pOffsetInfo->version; - }else if(type == TMQ_OFFSET__RESET_EARLIEST){ - position = pVg->offsetInfo.walVerBegin; - }else if(type == TMQ_OFFSET__RESET_LATEST){ - position = pVg->offsetInfo.walVerEnd; + position = pOffsetInfo->endOffset.version; + }else if(type == TMQ_OFFSET__RESET_EARLIEST || type == TMQ_OFFSET__RESET_LATEST){ + code = getCommittedFromServer(tmq, tname, vgId, &epSet); + if(code == TSDB_CODE_TMQ_NO_COMMITTED){ + if(type == TMQ_OFFSET__RESET_EARLIEST){ + position = begin; + } else if(type == TMQ_OFFSET__RESET_LATEST){ + position = end; + } + }else{ + position = code; + } }else{ tscError("consumer:0x%" PRIx64 " offset type:%d can not be reach here", tmq->consumerId, type); } - taosWUnLockLatch(&tmq->lock); return position; } +int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId){ + if (tmq == NULL || pTopicName == NULL) { + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } + + int32_t accId = tmq->pTscObj->acctId; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + taosWLockLatch(&tmq->lock); + + SMqClientVg* pVg = NULL; + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if(code != 0){ + taosWUnLockLatch(&tmq->lock); + return code; + } + + SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; + if (isInSnapshotMode(pOffsetInfo->endOffset.type, tmq->useSnapshot)) { + tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, pOffsetInfo->endOffset.type); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_SNAPSHOT_ERROR; + } + + if (isInSnapshotMode(pOffsetInfo->committedOffset.type, tmq->useSnapshot)) { + tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, pOffsetInfo->committedOffset.type); + taosWUnLockLatch(&tmq->lock); + return TSDB_CODE_TMQ_SNAPSHOT_ERROR; + } + + int64_t committed = 0; + if(pOffsetInfo->committedOffset.type == TMQ_OFFSET__LOG){ + committed = pOffsetInfo->committedOffset.version; + taosWUnLockLatch(&tmq->lock); + return committed; + } + SEpSet epSet = pVg->epSet; + taosWUnLockLatch(&tmq->lock); + + return getCommittedFromServer(tmq, tname, vgId, &epSet); +} + int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment, int32_t* numOfAssignment) { + if(tmq == NULL || pTopicName == NULL || assignment == NULL || numOfAssignment == NULL){ + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } *numOfAssignment = 0; *assignment = NULL; SMqVgCommon* pCommon = NULL; @@ -2881,7 +3079,7 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { } int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { - if (tmq == NULL) { + if (tmq == NULL || pTopicName == NULL) { tscError("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -2891,27 +3089,12 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ sprintf(tname, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); - SMqClientTopic* pTopic = getTopicByName(tmq, tname); - if (pTopic == NULL) { - tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); - taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_TMQ_INVALID_TOPIC; - } SMqClientVg* pVg = NULL; - int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); - for (int32_t i = 0; i < numOfVgs; ++i) { - SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); - if (pClientVg->vgId == vgId) { - pVg = pClientVg; - break; - } - } - - if (pVg == NULL) { - tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId); + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if(code != 0){ taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_TMQ_INVALID_VGID; + return code; } SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; @@ -2923,53 +3106,44 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } - if (!isWalRangeOk(&pVg->offsetInfo)) { - tscError("consumer:0x%" PRIx64 " Assignment or poll interface need to be called first", tmq->consumerId); + code = checkWalRange(pOffsetInfo, -1); + if (code != 0) { taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_TMQ_NEED_INITIALIZED; - } - - if (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd) { - tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", - tmq->consumerId, offset, pOffsetInfo->walVerBegin, pOffsetInfo->walVerEnd); - taosWUnLockLatch(&tmq->lock); - return TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE; + return code; } + tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, vgId); // update the offset, and then commit to vnode pOffsetInfo->endOffset.type = TMQ_OFFSET__LOG; pOffsetInfo->endOffset.version = offset; pOffsetInfo->beginOffset = pOffsetInfo->endOffset; pVg->seekUpdated = true; - tscInfo("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, vgId); + SEpSet epSet = pVg->epSet; + taosWUnLockLatch(&tmq->lock); SMqSeekReq req = {0}; - snprintf(req.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s:%s", tmq->groupId, pTopic->topicName); - req.head.vgId = pVg->vgId; + snprintf(req.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s:%s", tmq->groupId, tname); + req.head.vgId = vgId; req.consumerId = tmq->consumerId; int32_t msgSize = tSerializeSMqSeekReq(NULL, 0, &req); if (msgSize < 0) { - taosWUnLockLatch(&tmq->lock); return TSDB_CODE_PAR_INTERNAL_ERROR; } char* msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { - taosWUnLockLatch(&tmq->lock); return TSDB_CODE_OUT_OF_MEMORY; } if (tSerializeSMqSeekReq(msg, msgSize, &req) < 0) { taosMemoryFree(msg); - taosWUnLockLatch(&tmq->lock); return TSDB_CODE_PAR_INTERNAL_ERROR; } SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { taosMemoryFree(msg); - taosWUnLockLatch(&tmq->lock); return TSDB_CODE_OUT_OF_MEMORY; } @@ -2977,7 +3151,6 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ if (pParam == NULL) { taosMemoryFree(msg); taosMemoryFree(sendInfo); - taosWUnLockLatch(&tmq->lock); return TSDB_CODE_OUT_OF_MEMORY; } tsem_init(&pParam->sem, 0, 0); @@ -2991,18 +3164,15 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ int64_t transporterId = 0; tscInfo("consumer:0x%" PRIx64 " %s send seek info vgId:%d, epoch %d" PRIx64, - tmq->consumerId, pTopic->topicName, vgId, tmq->epoch); - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - taosWUnLockLatch(&tmq->lock); + tmq->consumerId, tname, vgId, tmq->epoch); + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); tsem_wait(&pParam->sem); - int32_t code = pParam->code; + code = pParam->code; tsem_destroy(&pParam->sem); taosMemoryFree(pParam); - if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 " failed to send seek to vgId:%d, code:%s", tmq->consumerId, vgId, tstrerror(code)); - } + tscInfo("consumer:0x%" PRIx64 "send seek to vgId:%d, return code:%s", tmq->consumerId, vgId, tstrerror(code)); return code; } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 6aeb2152d5..bfd6908e16 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -1075,6 +1075,89 @@ TEST(clientCase, sub_db_test) { fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); } +TEST(clientCase, tmq_commit) { +// taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); + + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + tmq_conf_t* conf = tmq_conf_new(); + + tmq_conf_set(conf, "enable.auto.commit", "false"); + tmq_conf_set(conf, "auto.commit.interval.ms", "2000"); + tmq_conf_set(conf, "group.id", "group_id_2"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + tmq_conf_destroy(conf); + + char topicName[128] = "tp"; + // 创建订阅 topics 列表 + tmq_list_t* topicList = tmq_list_new(); + tmq_list_append(topicList, topicName); + + // 启动订阅 + tmq_subscribe(tmq, topicList); + tmq_list_destroy(topicList); + + int32_t totalRows = 0; + int32_t msgCnt = 0; + int32_t timeout = 2000; + + tmq_topic_assignment* pAssign = NULL; + int32_t numOfAssign = 0; + + int32_t code = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_free_assignment(pAssign); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } + + for(int i = 0; i < numOfAssign; i++){ + printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end); + + int64_t position = tmq_position(tmq, topicName, pAssign[i].vgId); + printf("position vgId:%d, position:%lld\n", pAssign[i].vgId, position); + tmq_offset_seek(tmq, topicName, pAssign[i].vgId, 1); + position = tmq_position(tmq, topicName, pAssign[i].vgId); + printf("after seek 100, position vgId:%d, position:%lld\n", pAssign[i].vgId, position); + } + + while (1) { + printf("start to poll\n"); + TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); + if (pRes) { + printSubResults(pRes, &totalRows); + } else { + break; + } + + tmq_commit_sync(tmq, pRes); + for(int i = 0; i < numOfAssign; i++) { + int64_t committed = tmq_committed(tmq, topicName, pAssign[i].vgId); + printf("committed vgId:%d, committed:%lld\n", pAssign[i].vgId, committed); + } + if (pRes != NULL) { + taos_free_result(pRes); + } + +// tmq_offset_seek(tmq, "tp", pAssign[0].vgId, pAssign[0].begin); + } + + tmq_free_assignment(pAssign); + + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); +} + TEST(clientCase, td_25129) { // taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 738b7db46a..8a5a4e5079 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -732,6 +732,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME_PUSH, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_VG_WALINFO, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_VG_COMMITTEDINFO, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index b5a7e5fc6b..2a5cdbe555 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -232,6 +232,7 @@ int32_t tqProcessSeekReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg); // tq-stream int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0b10b62267..bf0067b128 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -578,6 +578,49 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return code; } +int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) { + void* data = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + + SMqVgOffset vgOffset = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)data, len); + if (tDecodeMqVgOffset(&decoder, &vgOffset) < 0) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + tDecoderClear(&decoder); + + STqOffset* pOffset = &vgOffset.offset; + STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); + if (pSavedOffset == NULL) { + return TSDB_CODE_TMQ_NO_COMMITTED; + } + vgOffset.offset = *pSavedOffset; + + int32_t code = 0; + tEncodeSize(tEncodeMqVgOffset, &vgOffset, len, code); + if (code < 0) { + return TSDB_CODE_INVALID_PARA; + } + + void* buf = taosMemoryCalloc(1, len); + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SEncoder encoder; + tEncoderInit(&encoder, buf, len); + tEncodeMqVgOffset(&encoder, &vgOffset); + tEncoderClear(&encoder); + + SRpcMsg rsp = {.info = pMsg->info, .pCont = buf, .contLen = len, .code = 0}; + + tmsgSendRsp(&rsp); + + return 0; +} + int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { SMqPollReq req = {0}; if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0d9c478c1b..204107ee3c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -462,7 +462,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg } break; case TDMT_VND_TMQ_COMMIT_OFFSET: - if (tqProcessOffsetCommitReq(pVnode->pTq, ver, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { + if (tqProcessOffsetCommitReq(pVnode->pTq, ver, pReq, len) < 0) { goto _err; } break; @@ -638,6 +638,8 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { // return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_VND_TMQ_VG_WALINFO: return tqProcessVgWalInfoReq(pVnode->pTq, pMsg); + case TDMT_VND_TMQ_VG_COMMITTEDINFO: + return tqProcessVgCommittedInfoReq(pVnode->pTq, pMsg); case TDMT_VND_TMQ_SEEK: return tqProcessSeekReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_RUN: diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c36480a63e..83a50f7051 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -633,6 +633,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SCALAR_CONVERT_ERROR, "Cannot convert to s TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NEED_INITIALIZED, "Assignment or poll interface need to be called first") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_SNAPSHOT_ERROR, "Can not operate in snapshot mode") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_COMMITTED, "No committed info") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE, "Offset out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_VGID, "VgId does not belong to this consumer") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_TOPIC, "Topic does not belong to this consumer") From d7d81d82a0f5824c5bf9fc63e7a5ba4af031710c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 19 Jul 2023 18:27:19 +0800 Subject: [PATCH 060/100] feat:add committed & position & commite_offset interface --- source/client/test/clientTests.cpp | 8 +++++++- source/dnode/vnode/src/tq/tq.c | 14 +++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index bfd6908e16..b331e68b73 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -1127,7 +1127,7 @@ TEST(clientCase, tmq_commit) { printf("position vgId:%d, position:%lld\n", pAssign[i].vgId, position); tmq_offset_seek(tmq, topicName, pAssign[i].vgId, 1); position = tmq_position(tmq, topicName, pAssign[i].vgId); - printf("after seek 100, position vgId:%d, position:%lld\n", pAssign[i].vgId, position); + printf("after seek 1, position vgId:%d, position:%lld\n", pAssign[i].vgId, position); } while (1) { @@ -1143,6 +1143,12 @@ TEST(clientCase, tmq_commit) { for(int i = 0; i < numOfAssign; i++) { int64_t committed = tmq_committed(tmq, topicName, pAssign[i].vgId); printf("committed vgId:%d, committed:%lld\n", pAssign[i].vgId, committed); + if(committed > 0){ + int32_t code = tmq_commit_offset_sync(tmq, topicName, pAssign[i].vgId, 4); + printf("tmq_commit_offset_sync vgId:%d, offset:4, code:%d\n", pAssign[i].vgId, code); + int64_t committed = tmq_committed(tmq, topicName, pAssign[i].vgId); + printf("after tmq_commit_offset_sync, committed vgId:%d, committed:%lld\n", pAssign[i].vgId, committed); + } } if (pRes != NULL) { taos_free_result(pRes); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index bf0067b128..03d6932578 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -85,9 +85,9 @@ void tqDestroyTqHandle(void* data) { } } -static bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) { +static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) { return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG && - pLeft->val.version <= pRight->val.version; + pLeft->val.version == pRight->val.version; } STQ* tqOpen(const char* path, SVnode* pVnode) { @@ -302,10 +302,10 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t STqOffset* pOffset = &vgOffset.offset; if (pOffset->val.type == TMQ_OFFSET__SNAPSHOT_DATA || pOffset->val.type == TMQ_OFFSET__SNAPSHOT_META) { - tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, + tqInfo("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, pOffset->subKey, vgId, pOffset->val.uid, pOffset->val.ts); } else if (pOffset->val.type == TMQ_OFFSET__LOG) { - tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, pOffset->subKey, vgId, + tqInfo("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, pOffset->subKey, vgId, pOffset->val.version); if (pOffset->val.version + 1 == sversion) { pOffset->val.version += 1; @@ -316,8 +316,8 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t } STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); - if (pSavedOffset != NULL && tqOffsetLessOrEqual(pOffset, pSavedOffset)) { - tqDebug("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64, + if (pSavedOffset != NULL && tqOffsetEqual(pOffset, pSavedOffset)) { + tqInfo("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64, vgId, pOffset->subKey, pOffset->val.version, pSavedOffset->val.version); return 0; // no need to update the offset value } @@ -605,7 +605,7 @@ int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) { return TSDB_CODE_INVALID_PARA; } - void* buf = taosMemoryCalloc(1, len); + void* buf = rpcMallocCont(len); if (buf == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } From 49c87a7cf682a03959d39525a0213d1bcfecf91e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 19 Jul 2023 19:14:18 +0800 Subject: [PATCH 061/100] feat:add committed & position & commite_offset interface --- source/client/test/clientTests.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b331e68b73..02443a696c 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -1183,9 +1183,10 @@ TEST(clientCase, td_25129) { tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); tmq_conf_destroy(conf); + char topicName[128] = "tp"; // 创建订阅 topics 列表 tmq_list_t* topicList = tmq_list_new(); - tmq_list_append(topicList, "tp"); + tmq_list_append(topicList, topicName); // 启动订阅 tmq_subscribe(tmq, topicList); @@ -1203,7 +1204,7 @@ TEST(clientCase, td_25129) { tmq_topic_assignment* pAssign = NULL; int32_t numOfAssign = 0; - int32_t code = tmq_get_topic_assignment(tmq, "tp", &pAssign, &numOfAssign); + int32_t code = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssign); if (code != 0) { printf("error occurs:%s\n", tmq_err2str(code)); tmq_free_assignment(pAssign); @@ -1220,7 +1221,7 @@ TEST(clientCase, td_25129) { // tmq_offset_seek(tmq, "tp", pAssign[0].vgId, 4); tmq_free_assignment(pAssign); - code = tmq_get_topic_assignment(tmq, "tp", &pAssign, &numOfAssign); + code = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssign); if (code != 0) { printf("error occurs:%s\n", tmq_err2str(code)); tmq_free_assignment(pAssign); @@ -1236,7 +1237,7 @@ TEST(clientCase, td_25129) { tmq_free_assignment(pAssign); - code = tmq_get_topic_assignment(tmq, "tp", &pAssign, &numOfAssign); + code = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssign); if (code != 0) { printf("error occurs:%s\n", tmq_err2str(code)); tmq_free_assignment(pAssign); @@ -1266,7 +1267,7 @@ TEST(clientCase, td_25129) { printSubResults(pRes, &totalRows); - code = tmq_get_topic_assignment(tmq, "tp", &pAssign, &numOfAssign); + code = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssign); if (code != 0) { printf("error occurs:%s\n", tmq_err2str(code)); tmq_free_assignment(pAssign); @@ -1280,10 +1281,11 @@ TEST(clientCase, td_25129) { printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end); } } else { - tmq_offset_seek(tmq, "tp", pAssign[0].vgId, pAssign[0].currentOffset); - tmq_offset_seek(tmq, "tp", pAssign[1].vgId, pAssign[1].currentOffset); + for(int i = 0; i < numOfAssign; i++) { + tmq_offset_seek(tmq, topicName, pAssign[i].vgId, pAssign[i].currentOffset); + } tmq_commit_sync(tmq, pRes); - continue; + break; } // tmq_commit_sync(tmq, pRes); From e1c4cca33dc1ecb9df30a73e5b0c7031661ff0bb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Jul 2023 00:07:04 +0800 Subject: [PATCH 062/100] feat:add committed & position & commite_offset interface --- source/client/src/clientTmq.c | 2 +- source/client/test/clientTests.cpp | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index f2ea7309e4..3576df434b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -3106,7 +3106,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } - code = checkWalRange(pOffsetInfo, -1); + code = checkWalRange(pOffsetInfo, offset); if (code != 0) { taosWUnLockLatch(&tmq->lock); return code; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 02443a696c..d88a26cbb2 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -1123,6 +1123,9 @@ TEST(clientCase, tmq_commit) { for(int i = 0; i < numOfAssign; i++){ printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end); + int64_t committed = tmq_committed(tmq, topicName, pAssign[i].vgId); + printf("committed vgId:%d, committed:%lld\n", pAssign[i].vgId, committed); + int64_t position = tmq_position(tmq, topicName, pAssign[i].vgId); printf("position vgId:%d, position:%lld\n", pAssign[i].vgId, position); tmq_offset_seek(tmq, topicName, pAssign[i].vgId, 1); @@ -1317,6 +1320,7 @@ TEST(clientCase, td_25129) { printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end); } + tmq_free_assignment(pAssign); tmq_consumer_close(tmq); taos_close(pConn); fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); From d031a77b58d87184299c1435e51a1333becdfd64 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Jul 2023 15:40:41 +0800 Subject: [PATCH 063/100] fix:[TD-25300]dead loop if getchar return EOF in arm64 --- tools/shell/src/shellCommand.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 0e305f57e9..8c91ff53e2 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -501,7 +501,7 @@ int32_t shellReadCommand(char *command) { while (1) { c = taosGetConsoleChar(); - if (c == EOF) { + if (c == (char)EOF) { return c; } From 99c3ebc2825c755224b9c2d11e66e900763de131 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 20 Jul 2023 18:28:21 +0800 Subject: [PATCH 064/100] fix:optimize log & change return value for async interface --- include/client/taos.h | 2 +- include/common/tmsgdef.h | 2 +- source/client/src/clientTmq.c | 49 +++++++++++++++++++++-------------- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 5ea1510e44..3cc2d907ab 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -288,7 +288,7 @@ DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param); DLL_EXPORT int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset); -DLL_EXPORT int32_t tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param); +DLL_EXPORT void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param); DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment, int32_t *numOfAssignment); DLL_EXPORT void tmq_free_assignment(tmq_topic_assignment* pAssignment); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index aa23442291..7d12c2a1d6 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -312,7 +312,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME, "vnode-tmq-consume", SMqPollReq, SMqDataBlkRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME_PUSH, "vnode-tmq-consume-push", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp) - TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committed-walinfo", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committedinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 3576df434b..96ec88e2e9 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -523,9 +523,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, pMsgSendInfo); - - return TSDB_CODE_SUCCESS; + return asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, pMsgSendInfo); } static SMqClientTopic* getTopicByName(tmq_t* tmq, const char* pTopicName) { @@ -546,7 +544,6 @@ static SMqClientTopic* getTopicByName(tmq_t* tmq, const char* pTopicName) { static SMqCommitCbParamSet* prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, int32_t rspNum){ SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { - pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam); return NULL; } @@ -715,7 +712,9 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us end: taosMemoryFree(pParamSet); - pCommitFp(tmq, code, userParam); + if(pParamSet->callbackFn != NULL) { + pCommitFp(tmq, code, userParam); + } return; } @@ -2307,6 +2306,9 @@ const char* tmq_get_table_name(TAOS_RES* res) { void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* param) { if (tmq == NULL) { tscError("invalid tmq handle, null"); + if(cb != NULL) { + cb(tmq, TSDB_CODE_INVALID_PARA, param); + } return; } if (pRes == NULL) { // here needs to commit all offsets. @@ -2410,15 +2412,17 @@ int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, tsem_destroy(&pInfo->sem); taosMemoryFree(pInfo); - tscInfo("consumer:0x%" PRIx64 " send seek to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); + tscInfo("consumer:0x%" PRIx64 " sync send seek to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); return code; } -int32_t tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param){ +void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param){ + int32_t code = 0; if (tmq == NULL || pTopicName == NULL) { tscError("invalid tmq handle, null"); - return TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_INVALID_PARA; + goto end; } int32_t accId = tmq->pTscObj->acctId; @@ -2427,17 +2431,17 @@ int32_t tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId taosWLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; - int32_t code = getClientVg(tmq, tname, vgId, &pVg); + code = getClientVg(tmq, tname, vgId, &pVg); if(code != 0){ taosWUnLockLatch(&tmq->lock); - return code; + goto end; } SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; code = checkWalRange(pOffsetInfo, offset); if (code != 0) { taosWUnLockLatch(&tmq->lock); - return code; + goto end; } taosWUnLockLatch(&tmq->lock); @@ -2445,9 +2449,12 @@ int32_t tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, cb, param); - tscInfo("consumer:0x%" PRIx64 " send seek to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); + tscInfo("consumer:0x%" PRIx64 " async send seek to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); - return code; +end: + if(code != 0 && cb != NULL){ + cb(tmq, code, param); + } } void updateEpCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) { @@ -2832,6 +2839,7 @@ int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId){ tscError("consumer:0x%" PRIx64 " offset type:%d can not be reach here", tmq->consumerId, type); } + tscInfo("consumer:0x%" PRIx64 " tmq_position vgId:%d position:%" PRId64, tmq->consumerId, vgId, position); return position; } @@ -2871,12 +2879,16 @@ int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId){ if(pOffsetInfo->committedOffset.type == TMQ_OFFSET__LOG){ committed = pOffsetInfo->committedOffset.version; taosWUnLockLatch(&tmq->lock); - return committed; + goto end; } SEpSet epSet = pVg->epSet; taosWUnLockLatch(&tmq->lock); - return getCommittedFromServer(tmq, tname, vgId, &epSet); + committed = getCommittedFromServer(tmq, tname, vgId, &epSet); + +end: + tscInfo("consumer:0x%" PRIx64 " tmq_committed vgId:%d committed:%" PRId64, tmq->consumerId, vgId, committed); + return committed; } int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment, @@ -2897,7 +2909,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a taosWLockLatch(&tmq->lock); SMqClientTopic* pTopic = getTopicByName(tmq, tname); if (pTopic == NULL) { - code = TSDB_CODE_INVALID_PARA; + code = TSDB_CODE_TMQ_INVALID_TOPIC; goto end; } @@ -3040,7 +3052,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } SVgOffsetInfo* pOffsetInfo = &pClientVg->offsetInfo; - tscInfo("vgId:%d offset is update to:%"PRId64, p->vgId, p->currentOffset); + tscInfo("consumer:0x%" PRIx64 " %s vgId:%d offset is update to:%"PRId64, tmq->consumerId, pTopic->topicName, p->vgId, p->currentOffset); pOffsetInfo->walVerBegin = p->begin; pOffsetInfo->walVerEnd = p->end; @@ -3078,6 +3090,7 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { return 0; } +// seek interface have to send msg to server to cancel push handle if needed, because consumer may be in wait status if there is no data to poll int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { if (tmq == NULL || pTopicName == NULL) { tscError("invalid tmq handle, null"); @@ -3163,8 +3176,6 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ sendInfo->msgType = TDMT_VND_TMQ_SEEK; int64_t transporterId = 0; - tscInfo("consumer:0x%" PRIx64 " %s send seek info vgId:%d, epoch %d" PRIx64, - tmq->consumerId, tname, vgId, tmq->epoch); asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); tsem_wait(&pParam->sem); From 10ebf1f43823f0b694bed7ab69f7276f168a9135 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 21 Jul 2023 11:07:34 +0800 Subject: [PATCH 065/100] fix:subscribeStb.py maybe failed --- source/dnode/vnode/src/tq/tq.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 03d6932578..8c9eead414 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -307,9 +307,6 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t } else if (pOffset->val.type == TMQ_OFFSET__LOG) { tqInfo("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, pOffset->subKey, vgId, pOffset->val.version); - if (pOffset->val.version + 1 == sversion) { - pOffset->val.version += 1; - } } else { tqError("invalid commit offset type:%d", pOffset->val.type); return -1; From bb86f5c580a8a7d6eafec36185232a823358162f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 21 Jul 2023 11:13:05 +0800 Subject: [PATCH 066/100] fix:commit offset should not plus 1 --- source/dnode/vnode/src/tq/tq.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 03d6932578..8c9eead414 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -307,9 +307,6 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t } else if (pOffset->val.type == TMQ_OFFSET__LOG) { tqInfo("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, pOffset->subKey, vgId, pOffset->val.version); - if (pOffset->val.version + 1 == sversion) { - pOffset->val.version += 1; - } } else { tqError("invalid commit offset type:%d", pOffset->val.type); return -1; From ca346501dedcec8ba2b93fbfd4d8f18d9bb032b1 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 Jul 2023 13:39:15 +0800 Subject: [PATCH 067/100] docs: update readme to claim does not support cross-compile --- README-CN.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README-CN.md b/README-CN.md index 9f2912ec40..53abc5c006 100644 --- a/README-CN.md +++ b/README-CN.md @@ -39,7 +39,7 @@ TDengine 是一款开源、高性能、云原生的时序数据库 (Time-Series # 构建 -TDengine 目前可以在 Linux、 Windows、macOS 等平台上安装和运行。任何 OS 的应用也可以选择 taosAdapter 的 RESTful 接口连接服务端 taosd。CPU 支持 X64/ARM64，后续会支持 MIPS64、Alpha64、ARM32、RISC-V 等 CPU 架构。 +TDengine 目前可以在 Linux、 Windows、macOS 等平台上安装和运行。任何 OS 的应用也可以选择 taosAdapter 的 RESTful 接口连接服务端 taosd。CPU 支持 X64/ARM64，后续会支持 MIPS64、Alpha64、ARM32、RISC-V 等 CPU 架构。目前不支持使用交叉编译器构建。 用户可根据需求选择通过源码、[容器](https://docs.taosdata.com/get-started/docker/)、[安装包](https://docs.taosdata.com/get-started/package/)或[Kubernetes](https://docs.taosdata.com/deployment/k8s/)来安装。本快速指南仅适用于通过源码安装。 diff --git a/README.md b/README.md index f065eb2685..73df4fb187 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ For user manual, system design and architecture, please refer to [TDengine Docum # Building -At the moment, TDengine server supports running on Linux/Windows/macOS systems. Any application can also choose the RESTful interface provided by taosAdapter to connect the taosd service . TDengine supports X64/ARM64 CPU, and it will support MIPS64, Alpha64, ARM32, RISC-V and other CPU architectures in the future. +At the moment, TDengine server supports running on Linux/Windows/macOS systems. Any application can also choose the RESTful interface provided by taosAdapter to connect the taosd service . TDengine supports X64/ARM64 CPU, and it will support MIPS64, Alpha64, ARM32, RISC-V and other CPU architectures in the future. Right now we don't support build with cross-compiling environment. You can choose to install through source code, [container](https://docs.tdengine.com/get-started/docker/), [installation package](https://docs.tdengine.com/get-started/package/) or [Kubernetes](https://docs.tdengine.com/deployment/k8s/). This quick guide only applies to installing from source. From 282bbfc41b39af87f6a2f4d51fbe2c3ccea95b8a Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 21 Jul 2023 14:46:16 +0800 Subject: [PATCH 068/100] remote web from installation package --- packaging/tools/install.sh | 10 +--------- packaging/tools/make_install.sh | 7 ------- packaging/tools/makepkg.sh | 12 ------------ 3 files changed, 1 insertion(+), 28 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index f311714f3d..5c52710cf0 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -613,12 +613,6 @@ function install_examples() { fi } -function install_web() { - if [ -d "${script_dir}/share" ]; then - ${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share > /dev/null 2>&1 ||: - fi -} - function clean_service_on_sysvinit() { if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then @@ -894,7 +888,6 @@ function updateProduct() { fi install_examples - install_web if [ -z $1 ]; then install_bin install_service @@ -971,8 +964,7 @@ function installProduct() { if [ "$verMode" == "cluster" ]; then install_connector fi - install_examples - install_web + install_examples if [ -z $1 ]; then # install service and client # For installing new diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index c5c70e0aa2..0a5f9d2668 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -432,12 +432,6 @@ function install_examples() { ${csudo}cp -rf ${source_dir}/examples/* ${install_main_dir}/examples || : } -function install_web() { - if [ -d "${binary_dir}/build/share" ]; then - ${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_dir}/share || : - fi -} - function clean_service_on_sysvinit() { if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then ${csudo}service ${serverName} stop || : @@ -592,7 +586,6 @@ function update_TDengine() { install_lib # install_connector install_examples - install_web install_bin install_app diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 6c389502b7..a48d264d5d 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -126,7 +126,6 @@ else fi install_files="${script_dir}/install.sh" -web_dir="${top_dir}/../enterprise/src/plugins/web" init_file_deb=${script_dir}/../deb/taosd init_file_rpm=${script_dir}/../rpm/taosd @@ -320,17 +319,6 @@ if [[ $dbName == "taos" ]]; then mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json fi - # Add web files - if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then - if [ -d "${web_dir}/admin" ] ; then - mkdir -p ${install_dir}/share/ - cp -Rfap ${web_dir}/admin ${install_dir}/share/ - cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png - cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||: - else - echo "directory not found for enterprise release: ${web_dir}/admin" - fi - fi fi # Copy driver From d4b29b9d2e7351ae8f11d997f1bc9dc26d993c1a Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 21 Jul 2023 15:29:53 +0800 Subject: [PATCH 069/100] update installation instruction --- packaging/tools/install.sh | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 5c52710cf0..961631561e 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -900,20 +900,22 @@ function updateProduct() { echo echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" + echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" else echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName2}${NC}" [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${clientName2}adapter &${NC}" fi + + echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}: sudo systemctl enable ${clientName2}keeper &${NC}" if [ ${openresty_work} = 'true' ]; then echo -e "${GREEN_DARK}To access ${productName2} ${NC}: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}" @@ -927,6 +929,7 @@ function updateProduct() { fi echo echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}" + echo -e "\033[44;32;1mTo manage ${productName2} instance, view documentation and explorer features, you need to install ${clientName2}Explorer ${NC}" else install_bin install_config @@ -981,21 +984,23 @@ function installProduct() { echo echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" + echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" else echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName2}${NC}" [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${clientName2}adapter &${NC}" fi + echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}: sudo systemctl enable ${clientName2}keeper &${NC}" + if [ ! -z "$firstEp" ]; then tmpFqdn=${firstEp%%:*} substr=":" @@ -1017,6 +1022,7 @@ function installProduct() { fi echo -e "\033[44;32;1m${productName2} is installed successfully!${NC}" + echo -e "\033[44;32;1mTo manage ${productName2} instance, view documentation and explorer features, you need to install ${clientName2}Explorer ${NC}" echo else # Only install client install_bin From 095271734ee236bd3a11ca76bec9560cb15fb5d2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 Jul 2023 17:06:22 +0800 Subject: [PATCH 070/100] docs: refine platform support matrix --- docs/en/14-reference/09-support-platform/index.md | 2 +- docs/zh/14-reference/09-support-platform/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/09-support-platform/index.md b/docs/en/14-reference/09-support-platform/index.md index 7dfa8ac93a..21fe6fc1dc 100644 --- a/docs/en/14-reference/09-support-platform/index.md +++ b/docs/en/14-reference/09-support-platform/index.md @@ -5,7 +5,7 @@ description: This document describes the supported platforms for the TDengine se ## List of supported platforms for TDengine server -| | **Windows Server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **macOS** | +| | **Windows Server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 or later** | **macOS** | | ------------ | ---------------------------- | ----------------- | ---------------- | ---------------- | --------- | | X64 | ● | ● | ● | ● | ● | | ARM64 | | | ● | | ● | diff --git a/docs/zh/14-reference/09-support-platform/index.md b/docs/zh/14-reference/09-support-platform/index.md index 500eeeb14c..c54cbe12e6 100644 --- a/docs/zh/14-reference/09-support-platform/index.md +++ b/docs/zh/14-reference/09-support-platform/index.md @@ -5,7 +5,7 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" ## TDengine 服务端支持的平台列表 -| | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** | +| | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 以上** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** | | ------------ | ---------------------------- | ----------------- | ---------------- | ---------------- | ------------ | ----------------- | ---------------- | --------- | | X64 | ● | ● | ● | ● | ● | ● | ● | ● | | 树莓派 ARM64 | | | ● | | | | | | From 401bf5ee840e1527cfcbb5d837978a434cbed21b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 21 Jul 2023 17:08:19 +0800 Subject: [PATCH 071/100] docs: refine platform support matrix (#22144) --- docs/en/14-reference/09-support-platform/index.md | 2 +- docs/zh/14-reference/09-support-platform/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/09-support-platform/index.md b/docs/en/14-reference/09-support-platform/index.md index 7dfa8ac93a..21fe6fc1dc 100644 --- a/docs/en/14-reference/09-support-platform/index.md +++ b/docs/en/14-reference/09-support-platform/index.md @@ -5,7 +5,7 @@ description: This document describes the supported platforms for the TDengine se ## List of supported platforms for TDengine server -| | **Windows Server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **macOS** | +| | **Windows Server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 or later** | **macOS** | | ------------ | ---------------------------- | ----------------- | ---------------- | ---------------- | --------- | | X64 | ● | ● | ● | ● | ● | | ARM64 | | | ● | | ● | diff --git a/docs/zh/14-reference/09-support-platform/index.md b/docs/zh/14-reference/09-support-platform/index.md index 500eeeb14c..c54cbe12e6 100644 --- a/docs/zh/14-reference/09-support-platform/index.md +++ b/docs/zh/14-reference/09-support-platform/index.md @@ -5,7 +5,7 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" ## TDengine 服务端支持的平台列表 -| | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** | +| | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 以上** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** | | ------------ | ---------------------------- | ----------------- | ---------------- | ---------------- | ------------ | ----------------- | ---------------- | --------- | | X64 | ● | ● | ● | ● | ● | ● | ● | ● | | 树莓派 ARM64 | | | ● | | | | | | From 8f15795cc16f98a4f2e47c1f65579527f1a127c9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 21 Jul 2023 19:47:31 +0800 Subject: [PATCH 072/100] fix:judge if pTopic is NULL --- source/dnode/mnode/impl/src/mndConsumer.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 2b538eccc9..f23201a062 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -515,7 +515,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); // txn guarantees pSub is created - + if(pSub == NULL) continue; taosRLockLatch(&pSub->lock); SMqSubTopicEp topicEp = {0}; @@ -523,6 +523,11 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { // 2.1 fetch topic schema SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); + if(pTopic == NULL) { + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + continue; + } taosRLockLatch(&pTopic->lock); tstrncpy(topicEp.db, pTopic->db, TSDB_DB_FNAME_LEN); topicEp.schema.nCols = pTopic->schema.nCols; From 8b7b9f8855df745d25f6960dc87a5275f581d10b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 23 Jul 2023 00:34:07 +0800 Subject: [PATCH 073/100] fix: use latest version of jdbc connector (#22148) --- examples/JDBC/mybatisplus-demo/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/JDBC/mybatisplus-demo/pom.xml b/examples/JDBC/mybatisplus-demo/pom.xml index 5555145958..f792946c96 100644 --- a/examples/JDBC/mybatisplus-demo/pom.xml +++ b/examples/JDBC/mybatisplus-demo/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.2.4 From 0558ee0512e30e154f6a28c1537c70cd02979456 Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Mon, 24 Jul 2023 00:37:19 +0800 Subject: [PATCH 074/100] Update 05-taosbenchmark.md --- docs/zh/14-reference/05-taosbenchmark.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 319046ba8f..425eb50bc0 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -362,6 +362,8 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **max** : 数据类型的 列/标签 的最大值。生成的值将小于最小值。 +- **fun** : 此列数据以函数填充，目前只支持 sin 和 cos 两函数，输入参数为角度，0~360度，同时支持系数调节，随机波动因子调节，以固定格式的表达式展现，如 fun=“10*sin(x)+100*random(5)” , x 表示角度，会从 0 增长至 360度，然后再恢复至 0 继续增长至 360 度循环不断，每次增长步长为 1 度。10 表示乘的系数，100 表示加或减的系数，5 表示波动幅度在 5% 的随机范围内。目前支持的数据类型为 int , bigint, float , double 四种数据类型。注意：表达式为固定模式，不可前后颠倒，中间不能有空格，否则会解析失败。 + - **values** : nchar/binary 列/标签的值域，将从值中随机选择。 - **sma**: 将该列加入 SMA 中，值为 "yes" 或者 "no"，默认为 "no"。 From 8639e1fc17025c05b399d4560593d766cfd1dd56 Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Mon, 24 Jul 2023 00:44:29 +0800 Subject: [PATCH 075/100] Update 05-taosbenchmark.md --- docs/en/14-reference/05-taosbenchmark.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index 2348810d9e..441b70bb88 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -364,6 +364,7 @@ The configuration parameters for specifying super table tag columns and data col - **min**: The minimum value of the column/label of the data type. The generated value will equal or large than the minimum value. - **max**: The maximum value of the column/label of the data type. The generated value will less than the maximum value. +- **fun**: This column of data is filled with functions. Currently, only the sin and cos functions are supported, with input parameters of angle ranging from 0 to 360 degrees. It also supports coefficient adjustment and random fluctuation factor adjustment, and is presented in a fixed format expression. For example, fun="10*sin(x)+100*random(5)", where x represents the angle, which will increase from 0 to 360 degrees, then recover to 0 and continue to increase to 360 degrees, with each increase step size of 1 degree. 10 represents the coefficient of multiplication, 100 represents the coefficient of addition or subtraction, and 5 represents the fluctuation range within a random range of 5%. The currently supported data type is int32_ T, int64_ t. There are four data types: float and double. Note: The expression is in a fixed mode and cannot be reversed. There must be no spaces in the middle, otherwise parsing will fail. - **values**: The value field of the nchar/binary column/label, which will be chosen randomly from the values. From 1a9161771591173ccf0d6b8f91033b8395695c81 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Mon, 24 Jul 2023 09:11:05 +0800 Subject: [PATCH 076/100] Update 03-table.md --- docs/zh/12-taos-sql/03-table.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 2e66ac4002..fa4cfd1c6f 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -44,9 +44,9 @@ table_option: { 1. 表的第一个字段必须是 TIMESTAMP，并且系统自动将其设为主键； 2. 表名最大长度为 192； 3. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） -4. 子表名只能由字母、数字和下划线组成，且不能以数字开头，不区分大小写 +4. 表名，超级表名，以及子表名只能由字母、数字和下划线组成，且不能以数字开头，不区分大小写 5. 使用数据类型 binary 或 nchar，需指定其最长的字节数，如 binary(20)，表示 20 字节； -6. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 +6. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"。 如果不加转义符，表名会被默认转换成小组；加上转义符可以保留表名中的大小写属性。 例如：\`aBc\` 和 \`abc\` 是不同的表名，但是 abc 和 aBc 是相同的表名。 需要注意的是转义字符中的内容必须是可打印字符。 From e41653e89c2738b2d06e336bd001be6be5418432 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Mon, 24 Jul 2023 09:11:59 +0800 Subject: [PATCH 077/100] Update 19-limit.md --- docs/zh/12-taos-sql/19-limit.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/zh/12-taos-sql/19-limit.md b/docs/zh/12-taos-sql/19-limit.md index 6c815fc5f0..b23a6b783d 100644 --- a/docs/zh/12-taos-sql/19-limit.md +++ b/docs/zh/12-taos-sql/19-limit.md @@ -10,8 +10,7 @@ description: 合法字符集和命名中的限制规则 2. 允许英文字符或下划线开头，不允许以数字开头 3. 不区分大小写 4. 转义后表（列）名规则： - 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。可用让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查 - 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一 + 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。使用转义字符以后，不再对转义字符中的内容进行大小写统一，即可以保留用户指定表名中的大小写属性。 例如：\`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 需要注意的是转义字符中的内容必须是可打印字符。 From 2edb925870350175fcb9f102bd3bcbdac404c8db Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Mon, 24 Jul 2023 09:13:48 +0800 Subject: [PATCH 078/100] Update 03-table.md --- docs/zh/12-taos-sql/03-table.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index fa4cfd1c6f..8ef07c313a 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -48,7 +48,6 @@ table_option: { 5. 使用数据类型 binary 或 nchar，需指定其最长的字节数，如 binary(20)，表示 20 字节； 6. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"。 如果不加转义符，表名会被默认转换成小组；加上转义符可以保留表名中的大小写属性。 例如：\`aBc\` 和 \`abc\` 是不同的表名，但是 abc 和 aBc 是相同的表名。 - 需要注意的是转义字符中的内容必须是可打印字符。 **参数说明** From 3ab7a74f7aa6daa126b28c84bc173a8362e01fb6 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Mon, 24 Jul 2023 09:14:31 +0800 Subject: [PATCH 079/100] Update 19-limit.md --- docs/zh/12-taos-sql/19-limit.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/zh/12-taos-sql/19-limit.md b/docs/zh/12-taos-sql/19-limit.md index b23a6b783d..646aa505f1 100644 --- a/docs/zh/12-taos-sql/19-limit.md +++ b/docs/zh/12-taos-sql/19-limit.md @@ -13,7 +13,6 @@ description: 合法字符集和命名中的限制规则 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。使用转义字符以后，不再对转义字符中的内容进行大小写统一，即可以保留用户指定表名中的大小写属性。 例如：\`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 - 需要注意的是转义字符中的内容必须是可打印字符。 ## 密码合法字符集 From 8798954af45d067a16fcc03d2411295eedeaf006 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Mon, 24 Jul 2023 09:15:55 +0800 Subject: [PATCH 080/100] Update 19-limit.md --- docs/zh/12-taos-sql/19-limit.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/12-taos-sql/19-limit.md b/docs/zh/12-taos-sql/19-limit.md index 646aa505f1..73107bf3ba 100644 --- a/docs/zh/12-taos-sql/19-limit.md +++ b/docs/zh/12-taos-sql/19-limit.md @@ -46,13 +46,13 @@ description: 合法字符集和命名中的限制规则 ### 转义后表（列）名规则： -为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"，可以避免表名与关键词的冲突，同时不受限于上述表名合法性约束检查，转义符不计入表名的长度。 +为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"，可以避免表名与关键词的冲突，转义符不计入表名的长度。 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 例如： \`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 :::note -转义字符中的内容必须是可打印字符。 +转义字符中的内容必须符合命名规则中的字符约束。 ::: From 42f3340f81ea58a34af92df16314d62dc6d03b86 Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Mon, 24 Jul 2023 10:28:09 +0800 Subject: [PATCH 081/100] Update 02-database.md --- docs/zh/12-taos-sql/02-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index d79a985089..75e01b9ab5 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -85,7 +85,7 @@ create database if not exists db vgroups 10 buffer 10 ``` -以上示例创建了一个有 10 个 vgroup 名为 db 的数据库， 其中每个 vnode 分配也 10MB 的写入缓存 +以上示例创建了一个有 10 个 vgroup 名为 db 的数据库， 其中每个 vnode 分配 10MB 的写入缓存 ### 使用数据库 From 7255d32769993e8ded2ae0215ce68798a97dbef2 Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Mon, 24 Jul 2023 12:48:12 +0800 Subject: [PATCH 082/100] Update 05-taosbenchmark.md --- docs/zh/14-reference/05-taosbenchmark.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 425eb50bc0..e4c3efba17 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -362,7 +362,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **max** : 数据类型的 列/标签 的最大值。生成的值将小于最小值。 -- **fun** : 此列数据以函数填充，目前只支持 sin 和 cos 两函数，输入参数为角度，0~360度，同时支持系数调节，随机波动因子调节，以固定格式的表达式展现，如 fun=“10*sin(x)+100*random(5)” , x 表示角度，会从 0 增长至 360度，然后再恢复至 0 继续增长至 360 度循环不断，每次增长步长为 1 度。10 表示乘的系数，100 表示加或减的系数，5 表示波动幅度在 5% 的随机范围内。目前支持的数据类型为 int , bigint, float , double 四种数据类型。注意：表达式为固定模式，不可前后颠倒，中间不能有空格，否则会解析失败。 +- **fun** : 此列数据以函数填充，目前只支持 sin 和 cos 两函数，输入参数为时间戳换算成角度值，换算公式： 角度 x = 输入的时间列ts值 % 360。同时支持系数调节，随机波动因子调节，以固定格式的表达式展现，如 fun=“10\*sin(x)+100\*random(5)” , x 表示角度，取值 0 ~ 360度，增长步长与时间列步长一致。10 表示乘的系数，100 表示加或减的系数，5 表示波动幅度在 5% 的随机范围内。目前支持的数据类型为 int, bigint, float, double 四种数据类型。注意：表达式为固定模式，不可前后颠倒。 - **values** : nchar/binary 列/标签的值域，将从值中随机选择。 From f1d79f0ba3725fe62d483bd2407516be780c00fa Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Mon, 24 Jul 2023 12:52:13 +0800 Subject: [PATCH 083/100] Update 05-taosbenchmark.md --- docs/en/14-reference/05-taosbenchmark.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index 441b70bb88..38a8048a21 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -364,7 +364,7 @@ The configuration parameters for specifying super table tag columns and data col - **min**: The minimum value of the column/label of the data type. The generated value will equal or large than the minimum value. - **max**: The maximum value of the column/label of the data type. The generated value will less than the maximum value. -- **fun**: This column of data is filled with functions. Currently, only the sin and cos functions are supported, with input parameters of angle ranging from 0 to 360 degrees. It also supports coefficient adjustment and random fluctuation factor adjustment, and is presented in a fixed format expression. For example, fun="10*sin(x)+100*random(5)", where x represents the angle, which will increase from 0 to 360 degrees, then recover to 0 and continue to increase to 360 degrees, with each increase step size of 1 degree. 10 represents the coefficient of multiplication, 100 represents the coefficient of addition or subtraction, and 5 represents the fluctuation range within a random range of 5%. The currently supported data type is int32_ T, int64_ t. There are four data types: float and double. Note: The expression is in a fixed mode and cannot be reversed. There must be no spaces in the middle, otherwise parsing will fail. +- **fun**: This column of data is filled with functions. Currently, only the sin and cos functions are supported. The input parameter is the timestamp and converted to an angle value. The conversion formula is: angle x=input time column ts value % 360. At the same time, it supports coefficient adjustment and random fluctuation factor adjustment, presented in a fixed format expression, such as fun="10\*sin(x)+100\*random(5)", where x represents the angle, ranging from 0 to 360 degrees, and the growth step size is consistent with the time column step size. 10 represents the coefficient of multiplication, 100 represents the coefficient of addition or subtraction, and 5 represents the fluctuation range within a random range of 5%. The currently supported data types are int, bigint, float, and double. Note: The expression is fixed and cannot be reversed. - **values**: The value field of the nchar/binary column/label, which will be chosen randomly from the values. From 90f4684918b872932e10dcc63018cf8403ba88f5 Mon Sep 17 00:00:00 2001 From: liuyuan <2805658706@qq.com> Date: Mon, 24 Jul 2023 14:36:23 +0800 Subject: [PATCH 084/100] docs: optimization k8s deploy --- docs/en/10-deployment/03-k8s.md | 514 +++++++++++++++++++++----------- docs/zh/10-deployment/03-k8s.md | 482 +++++++++++++++++++----------- 2 files changed, 648 insertions(+), 348 deletions(-) diff --git a/docs/en/10-deployment/03-k8s.md b/docs/en/10-deployment/03-k8s.md index 49e61caafc..c72234b336 100644 --- a/docs/en/10-deployment/03-k8s.md +++ b/docs/en/10-deployment/03-k8s.md @@ -4,15 +4,23 @@ sidebar_label: Kubernetes description: This document describes how to deploy TDengine on Kubernetes. --- -TDengine is a cloud-native time-series database that can be deployed on Kubernetes. This document gives a step-by-step description of how you can use YAML files to create a TDengine cluster and introduces common operations for TDengine in a Kubernetes environment. +## Overview + +As a time series database for Cloud Native architecture design, TDengine supports Kubernetes deployment. Here we introduce how to use YAML files to create a highly available TDengine cluster from scratch step by step for production use, and highlight the common operations of TDengine in Kubernetes environment. + +To meet [high availability ](https://docs.taosdata.com/tdinternal/high-availability/)requirements, clusters need to meet the following requirements: + +- 3 or more dnodes: The vnodes in the vgroup of TDengine are not allowed to be distributed in one dnode at the same time, so if you create a database with 3 copies, the number of dnodes is greater than or equal to 3 +- 3 mnodes: nmode is responsible for the management of the entire cluster. TDengine defaults to an mnode. At this time, if the dnode where the mnode is located is dropped, the entire cluster is unavailable at this time +- There are 3 copies of the database, and the copy configuration of TDengine is DB level, which can be satisfied with 3 copies. In a 3-node cluster, any dnode goes offline, which does not affect the normal use of the cluster. **If the number of offline is 2, the cluster is unavailable at this time, and RAFT cannot complete the election** , (Enterprise Edition: In the disaster recovery scenario, any node data file is damaged, which can be restored by pulling up the dnode again) ## Prerequisites Before deploying TDengine on Kubernetes, perform the following: -* Current steps are compatible with Kubernetes v1.5 and later version. -* Install and configure minikube, kubectl, and helm. -* Install and deploy Kubernetes and ensure that it can be accessed and used normally. Update any container registries or other services as necessary. +- Current steps are compatible with Kubernetes v1.5 and later version. +- Install and configure minikube, kubectl, and helm. +- Install and deploy Kubernetes and ensure that it can be accessed and used normally. Update any container registries or other services as necessary. You can download the configuration files in this document from [GitHub](https://github.com/taosdata/TDengine-Operator/tree/3.0/src/tdengine). @@ -20,7 +28,7 @@ You can download the configuration files in this document from [GitHub](https:// Create a service configuration file named `taosd-service.yaml`. Record the value of `metadata.name` (in this example, `taos`) for use in the next step. Add the ports required by TDengine: -```yaml +```YAML --- apiVersion: v1 kind: Service @@ -31,10 +39,10 @@ metadata: spec: ports: - name: tcp6030 - - protocol: "TCP" + protocol: "TCP" port: 6030 - name: tcp6041 - - protocol: "TCP" + protocol: "TCP" port: 6041 selector: app: "tdengine" @@ -42,10 +50,11 @@ spec: ## Configure the service as StatefulSet -Configure the TDengine service as a StatefulSet. -Create the `tdengine.yaml` file and set `replicas` to 3. In this example, the region is set to Asia/Shanghai and 10 GB of standard storage are allocated per node. You can change the configuration based on your environment and business requirements. +According to Kubernetes instructions for various deployments, we will use StatefulSet as the service type of TDengine. Create the file `tdengine.yaml `, where replicas defines the number of cluster nodes as 3. The node time zone is China (Asia/Shanghai), and each node is allocated 5G standard storage (refer to the [Storage Classes ](https://kubernetes.io/docs/concepts/storage/storage-classes/)configuration storage class). You can also modify accordingly according to the actual situation. -```yaml +You need to pay attention to the configuration of startupProbe. After the dnode is disconnected for a period of time, restart, and the newly launched dnode will be temporarily unavailable. If the startupProbe configuration is too small, Kubernetes will think that the pod is in an abnormal state and will try to pull the pod again. At this time, dnode will restart frequently and never recover. Refer to [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) + +```YAML --- apiVersion: apps/v1 kind: StatefulSet @@ -69,14 +78,14 @@ spec: spec: containers: - name: "tdengine" - image: "tdengine/tdengine:3.0.0.0" + image: "tdengine/tdengine:3.0.7.1" imagePullPolicy: "IfNotPresent" ports: - name: tcp6030 - - protocol: "TCP" + protocol: "TCP" containerPort: 6030 - name: tcp6041 - - protocol: "TCP" + protocol: "TCP" containerPort: 6041 env: # POD_NAME for FQDN config @@ -102,12 +111,18 @@ spec: # Must set if you want a cluster. - name: TAOS_FIRST_EP value: "$(STS_NAME)-0.$(SERVICE_NAME).$(STS_NAMESPACE).svc.cluster.local:$(TAOS_SERVER_PORT)" - # TAOS_FQDN should always be set in k8s env. + # TAOS_FQND should always be set in k8s env. - name: TAOS_FQDN value: "$(POD_NAME).$(SERVICE_NAME).$(STS_NAMESPACE).svc.cluster.local" volumeMounts: - name: taosdata mountPath: /var/lib/taos + startupProbe: + exec: + command: + - taos-check + failureThreshold: 360 + periodSeconds: 10 readinessProbe: exec: command: @@ -129,266 +144,401 @@ spec: storageClassName: "standard" resources: requests: - storage: "10Gi" + storage: "5Gi" ``` ## Use kubectl to deploy TDengine -Run the following commands: +Execute the following commands in sequence, and you need to create the corresponding namespace in advance. -```bash -kubectl apply -f taosd-service.yaml -kubectl apply -f tdengine.yaml +```Bash +kubectl apply -f taosd-service.yaml -n tdengine-test +kubectl apply -f tdengine.yaml -n tdengine-test ``` -The preceding configuration generates a TDengine cluster with three nodes in which dnodes are automatically configured. You can run the `show dnodes` command to query the nodes in the cluster: +The above configuration will generate a three-node TDengine cluster, dnode is automatically configured, you can use the show dnodes command to view the nodes of the current cluster: -```bash -kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" -kubectl exec -i -t tdengine-1 -- taos -s "show dnodes" -kubectl exec -i -t tdengine-2 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show dnodes" +kubectl exec -it tdengine-2 -n tdengine-test -- taos -s "show dnodes" ``` The output is as follows: -``` +```Bash taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | -Query OK, 3 rows in database (0.003655s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 0 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-19 17:54:18.469 | | | | + 2 | tdengine-1.ta... | 0 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-19 17:54:38.698 | | | | + 3 | tdengine-2.ta... | 0 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-19 17:55:02.039 | | | | +Query OK, 3 row(s) in set (0.001853s) +``` + +View the current mnode + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-19 17:54:18.559 +reboot_time: 2023-07-19 17:54:19.520 +Query OK, 1 row(s) in set (0.001282s) +``` + +## Create mnode + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 2" +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 3" +``` + +View mnode + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" + +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-19 17:54:18.559 +reboot_time: 2023-07-20 09:19:36.060 +*************************** 2.row *************************** + id: 2 + endpoint: tdengine-1.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:05.600 +reboot_time: 2023-07-20 09:22:12.838 +*************************** 3.row *************************** + id: 3 + endpoint: tdengine-2.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:20.042 +reboot_time: 2023-07-20 09:22:23.271 +Query OK, 3 row(s) in set (0.003108s) ``` ## Enable port forwarding The kubectl port forwarding feature allows applications to access the TDengine cluster running on Kubernetes. -``` -kubectl port-forward tdengine-0 6041:6041 & +```bash +kubectl port-forward -n tdengine-test tdengine-0 6041:6041 & ``` Use curl to verify that the TDengine REST API is working on port 6041: -``` -$ curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql -Handling connection for 6041 -{"code":0,"column_meta":[["name","VARCHAR",64],["create_time","TIMESTAMP",8],["vgroups","SMALLINT",2],["ntables","BIGINT",8],["replica","TINYINT",1],["strict","VARCHAR",4],["duration","VARCHAR",10],["keep","VARCHAR",32],["buffer","INT",4],["pagesize","INT",4],["pages","INT",4],["minrows","INT",4],["maxrows","INT",4],["comp","TINYINT",1],["precision","VARCHAR",2],["status","VARCHAR",10],["retention","VARCHAR",60],["single_stable","BOOL",1],["cachemodel","VARCHAR",11],["cachesize","INT",4],["wal_level","TINYINT",1],["wal_fsync_period","INT",4],["wal_retention_period","INT",4],["wal_retention_size","BIGINT",8],["wal_roll_period","INT",4],["wal_segment_size","BIGINT",8]],"data":[["information_schema",null,null,16,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null],["performance_schema",null,null,10,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null]],"rows":2} +```bash +curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql +{"code":0,"column_meta":[["name","VARCHAR",64]],"data":[["information_schema"],["performance_schema"],["test"],["test1"]],"rows":4} ``` -## Enable the dashboard for visualization +## Test cluster - The minikube dashboard command enables visualized cluster management. +### Data preparation -``` -$ minikube dashboard -* Verifying dashboard health ... -* Launching proxy ... -* Verifying proxy health ... -* Opening http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ in your default browser... -http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ +#### taosBenchmark + +Create a 3 replica database with taosBenchmark, write 100 million data at the same time, and view the data at the same time + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taosBenchmark -I stmt -d test -n 10000 -t 10000 -a 3 + +# query data +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "select count(*) from test.meters;" + +taos> select count(*) from test.meters; + count(*) | +======================== + 100000000 | +Query OK, 1 row(s) in set (0.103537s) ``` -In some public clouds, minikube cannot be remotely accessed if it is bound to 127.0.0.1. In this case, use the kubectl proxy command to map the port to 0.0.0.0. Then, you can access the dashboard by using a web browser to open the dashboard URL above on the public IP address and port of the virtual machine. +View vnode distribution by showing dnodes +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" + +taos> show dnodes + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 8 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-19 17:54:18.469 | | | | + 2 | tdengine-1.ta... | 8 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-19 17:54:38.698 | | | | + 3 | tdengine-2.ta... | 8 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-19 17:55:02.039 | | | | +Query OK, 3 row(s) in set (0.001357s) ``` -$ kubectl proxy --accept-hosts='^.*$' --address='0.0.0.0' + +View xnode distribution by showing vgroup + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test.vgroups" + +taos> show test.vgroups + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | v4_status | cacheload | cacheelements | tsma | +============================================================================================================================================================================================== + 2 | test | 1267 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 3 | test | 1215 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 4 | test | 1215 | 1 | leader | 2 | follower | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 5 | test | 1307 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 6 | test | 1245 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 7 | test | 1275 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 8 | test | 1231 | 1 | leader | 2 | follower | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 9 | test | 1245 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | +Query OK, 8 row(s) in set (0.001488s) ``` +#### Manually created + +Common a three-copy test1, and create a table, write 2 pieces of data + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- \ + taos -s \ + "create database if not exists test1 replica 3; + use test1; + create table if not exists t1(ts timestamp, n int); + insert into t1 values(now, 1)(now+1s, 2);" +``` + +View xnode distribution by showing test1.vgroup + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test1.vgroups" + +taos> show test1.vgroups + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | v4_status | cacheload | cacheelements | tsma | +============================================================================================================================================================================================== + 10 | test1 | 1 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 11 | test1 | 0 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | +Query OK, 2 row(s) in set (0.001489s) +``` + +### Test fault tolerance + +The dnode where the Mnode leader is located is offline, dnode1 + +```Bash +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 0/1 ErrImagePull 2 (2s ago) 20m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (6m48s ago) 20m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 21m 10.244.1.223 node85 +``` + +At this time, the cluster mnode has a re-election, and the monde on dnode1 becomes the leader. + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" +Welcome to the TDengine Command Line Interface, Client Version:3.0.7.1.202307190706 +Copyright (c) 2022 by TDengine, all rights reserved. + +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: offline + status: offline +create_time: 2023-07-19 17:54:18.559 +reboot_time: 1970-01-01 08:00:00.000 +*************************** 2.row *************************** + id: 2 + endpoint: tdengine-1.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-20 09:22:05.600 +reboot_time: 2023-07-20 09:32:00.227 +*************************** 3.row *************************** + id: 3 + endpoint: tdengine-2.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:20.042 +reboot_time: 2023-07-20 09:32:00.026 +Query OK, 3 row(s) in set (0.001513s) +``` + +Cluster can read and write normally + +```Bash +# insert +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "insert into test1.t1 values(now, 1)(now+1s, 2);" + +taos> insert into test1.t1 values(now, 1)(now+1s, 2); +Insert OK, 2 row(s) affected (0.002098s) + +# select +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "select *from test1.t1" + +taos> select *from test1.t1 + ts | n | +======================================== + 2023-07-19 18:04:58.104 | 1 | + 2023-07-19 18:04:59.104 | 2 | + 2023-07-19 18:06:00.303 | 1 | + 2023-07-19 18:06:01.303 | 2 | +Query OK, 4 row(s) in set (0.001994s) +``` + +In the same way, as for the mnode dropped by the non-leader, reading and writing can of course be performed normally, so there will be no too much display here. + ## Scaling Out Your Cluster -TDengine clusters can scale automatically: +TDengine cluster supports automatic expansion: -```bash +```Bash kubectl scale statefulsets tdengine --replicas=4 ``` -The preceding command increases the number of replicas to 4. After running this command, query the pod status: +The parameter `--replica = 4 `in the above command line indicates that you want to expand the TDengine cluster to 4 nodes. After execution, first check the status of the POD: -```bash -kubectl get pods -l app=tdengine +```Bash +kubectl get pod -l app=tdengine -n tdengine-test -o wide ``` The output is as follows: -``` -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 161m -tdengine-1 1/1 Running 0 161m -tdengine-2 1/1 Running 0 32m -tdengine-3 1/1 Running 0 32m +```Plain +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h26m ago) 6h53m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (6h39m ago) 6h53m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h16m 10.244.1.224 node85 +tdengine-3 1/1 Running 0 3m24s 10.244.2.76 node86 ``` -The status of all pods is Running. Once the pod status changes to Ready, you can check the dnode status: +At this time, the state of the POD is still Running, and the dnode state in the TDengine cluster can only be seen after the POD state is `ready `: -```bash -kubectl exec -i -t tdengine-3 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-3 -n tdengine-test -- taos -s "show dnodes" ``` -The following output shows that the TDengine cluster has been expanded to 4 replicas: +The dnode list of the expanded four-node TDengine cluster: -``` +```Plain taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | - 4 | tdengine-3.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:33:16.039 | | -Query OK, 4 rows in database (0.008377s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 4 | tdengine-3.ta... | 0 | 16 | ready | 2023-07-20 16:01:44.007 | 2023-07-20 16:01:44.889 | | | | +Query OK, 4 row(s) in set (0.003628s) ``` ## Scaling In Your Cluster -When you scale in a TDengine cluster, your data is migrated to different nodes. You must run the drop dnodes command in TDengine to remove dnodes before scaling in your Kubernetes environment. +Since the TDengine cluster will migrate data between nodes during volume expansion and contraction, using the kubectl command to reduce the volume requires first using the "drop dnodes" command ( **If there are 3 replicas of db in the cluster, the number of dnodes after reduction must also be greater than or equal to 3, otherwise the drop dnode operation will be aborted** ), the node deletion is completed before Kubernetes cluster reduction. -Note: In a Kubernetes StatefulSet service, the newest pods are always removed first. For this reason, when you scale in your TDengine cluster, ensure that you drop the newest dnodes. +Note: Since Kubernetes Pods in the Statefulset can only be removed in reverse order of creation, the TDengine drop dnode also needs to be removed in reverse order of creation, otherwise the Pod will be in an error state. -``` -$ kubectl exec -i -t tdengine-0 -- taos -s "drop dnode 4" -``` - -```bash -$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "drop dnode 4" +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | -Query OK, 3 rows in database (0.004861s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | +Query OK, 3 row(s) in set (0.003324s) ``` -Verify that the dnode have been successfully removed by running the `kubectl exec -i -t tdengine-0 -- taos -s "show dnodes"` command. Then run the following command to remove the pod: +After confirming that the removal is successful (use kubectl exec -i -t tdengine-0 --taos -s "show dnodes" to view and confirm the dnode list), use the kubectl command to remove the POD: -``` -kubectl scale statefulsets tdengine --replicas=3 +```Plain +kubectl scale statefulsets tdengine --replicas=3 -n tdengine-test ``` -The newest pod in the deployment is removed. Run the `kubectl get pods -l app=tdengine` command to query the pod status: +The last POD will be deleted. Use the command kubectl get pods -l app = tdengine to check the POD status: -``` -$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 4m7s -tdengine-1 1/1 Running 0 3m55s -tdengine-2 1/1 Running 0 2m28s +```Plain +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h55m ago) 7h22m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (7h9m ago) 7h23m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h45m 10.244.1.224 node85 ``` -After the pod has been removed, manually delete the PersistentVolumeClaim (PVC). Otherwise, future scale-outs will attempt to use existing data. +After the POD is deleted, the PVC needs to be deleted manually, otherwise the previous data will continue to be used in the next expansion, resulting in the inability to join the cluster normally. -```bash -$ kubectl delete pvc taosdata-tdengine-3 +```Bash +kubectl delete pvc aosdata-tdengine-3 -n tdengine-test ``` -Your cluster has now been safely scaled in, and you can scale it out again as necessary. +The cluster state at this time is safe and can be scaled up again if needed. -```bash -$ kubectl scale statefulsets tdengine --replicas=4 +```Bash +kubectl scale statefulsets tdengine --replicas=4 -n tdengine-test statefulset.apps/tdengine scaled -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 35m -tdengine-1 1/1 Running 0 34m -tdengine-2 1/1 Running 0 12m -tdengine-3 0/1 ContainerCreating 0 4s -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 35m -tdengine-1 1/1 Running 0 34m -tdengine-2 1/1 Running 0 12m -tdengine-3 0/1 Running 0 7s -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" + +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h59m ago) 7h27m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (7h13m ago) 7h27m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h49m 10.244.1.224 node85 +tdengine-3 1/1 Running 0 20s 10.244.2.77 node86 + +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes -id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | -====================================================================================================================================== -1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | -2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | -5 | tdengine-2.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:01:36.479 | | -6 | tdengine-3.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:13:54.411 | | -Query OK, 4 row(s) in set (0.001348s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 5 | tdengine-3.ta... | 0 | 16 | ready | 2023-07-20 16:31:34.092 | 2023-07-20 16:38:17.419 | | | | +Query OK, 4 row(s) in set (0.003881s) ``` ## Remove a TDengine Cluster -To fully remove a TDengine cluster, you must delete its statefulset, svc, configmap, and pvc entries: +> **When deleting the pvc, you need to pay attention to the pv persistentVolumeReclaimPolicy policy. It is recommended to change to Delete, so that the pv will be automatically cleaned up when the pvc is deleted, and the underlying csi storage resources will be cleaned up at the same time. If the policy of deleting the pvc to automatically clean up the pv is not configured, and then after deleting the pvc, when manually cleaning up the pv, the csi storage resources corresponding to the pv may not be released.** -```bash -kubectl delete statefulset -l app=tdengine -kubectl delete svc -l app=tdengine -kubectl delete pvc -l app=tdengine -kubectl delete configmap taoscfg +Complete removal of TDengine cluster, need to clean statefulset, svc, configmap, pvc respectively. +```Bash +kubectl delete statefulset -l app=tdengine -n tdengine-test +kubectl delete svc -l app=tdengine -n tdengine-test +kubectl delete pvc -l app=tdengine -n tdengine-test +kubectl delete configmap taoscfg -n tdengine-test ``` ## Troubleshooting ### Error 1 -If you remove a pod without first running `drop dnode`, some TDengine nodes will go offline. +No "drop dnode" is directly reduced. Since the TDengine has not deleted the node, the reduced pod causes some nodes in the TDengine cluster to be offline. -``` -$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" +```Plain +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes -id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | -====================================================================================================================================== -1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | -2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | -5 | tdengine-2.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:01:36.479 | status msg timeout | -6 | tdengine-3.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:13:54.411 | status msg timeout | -Query OK, 4 row(s) in set (0.001323s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 5 | tdengine-3.ta... | 0 | 16 | offline | 2023-07-20 16:31:34.092 | 2023-07-20 16:38:17.419 | status msg timeout | | | +Query OK, 4 row(s) in set (0.003862s) ``` -### Error 2 +## Finally -If the number of nodes after a scale-in is less than the value of the replica parameter, the cluster will go down: +For the high availability and high reliability of TDengine in the k8s environment, for hardware damage and disaster recovery, it is divided into two levels: -Create a database with replica set to 2 and add data. +1. The disaster recovery capability of the underlying distributed Block Storage, the multi-replica of Block Storage, the current popular distributed Block Storage such as ceph, has the multi-replica capability, extending the storage replica to different racks, cabinets, computer rooms, Data center (or directly use the Block Storage service provided by Public Cloud vendors) +2. TDengine disaster recovery, in TDengine Enterprise, itself has when a dnode permanently offline (TCE-metal disk damage, data sorting loss), re-pull a blank dnode to restore the original dnode work. -```bash -kubectl exec -i -t tdengine-0 -- \ - taos -s \ - "create database if not exists test replica 2; - use test; - create table if not exists t1(ts timestamp, n int); - insert into t1 values(now, 1)(now+1s, 2);" +Finally, welcome to [TDengine Cloud ](https://cloud.tdengine.com/)to experience the one-stop fully managed TDengine Cloud as a Service. - -``` - -Scale in to one node: - -```bash -kubectl scale statefulsets tdengine --replicas=1 - -``` - -In the TDengine CLI, you can see that no database operations succeed: - -``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 2 | 40 | ready | any | 2021-06-01 15:55:52.562 | | - 2 | tdengine-1.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 15:56:07.212 | status msg timeout | -Query OK, 2 row(s) in set (0.000845s) - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 2 | 40 | ready | any | 2021-06-01 15:55:52.562 | | - 2 | tdengine-1.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 15:56:07.212 | status msg timeout | -Query OK, 2 row(s) in set (0.000837s) - -taos> use test; -Database changed. - -taos> insert into t1 values(now, 3); - -DB error: Unable to resolve FQDN (0.013874s) - -``` +> TDengine Cloud is a minimalist fully managed time series data processing Cloud as a Service platform developed based on the open source time series database TDengine. In addition to high-performance time series database, it also has system functions such as caching, subscription and stream computing, and provides convenient and secure data sharing, as well as numerous enterprise-level functions. It allows enterprises in the fields of Internet of Things, Industrial Internet, Finance, IT operation and maintenance monitoring to significantly reduce labor costs and operating costs in the management of time series data. diff --git a/docs/zh/10-deployment/03-k8s.md b/docs/zh/10-deployment/03-k8s.md index 39ca56f3d9..7769b076bc 100644 --- a/docs/zh/10-deployment/03-k8s.md +++ b/docs/zh/10-deployment/03-k8s.md @@ -4,15 +4,23 @@ title: 在 Kubernetes 上部署 TDengine 集群 description: 利用 Kubernetes 部署 TDengine 集群的详细指南 --- -作为面向云原生架构设计的时序数据库，TDengine 支持 Kubernetes 部署。这里介绍如何使用 YAML 文件一步一步从头创建一个 TDengine 集群，并重点介绍 Kubernetes 环境下 TDengine 的常用操作。 +## 概述 + +作为面向云原生架构设计的时序数据库，TDengine 支持 Kubernetes 部署。这里介绍如何使用 YAML 文件一步一步从头创建一个可用于生产使用的高可用TDengine 集群，并重点介绍 Kubernetes 环境下 TDengine 的常用操作。 + +为了满足[高可用](https://docs.taosdata.com/tdinternal/high-availability/)的需求，集群需要满足如下要求： + +- 3个及以上dnode：TDengine的vgroup中的vnode，不允许同时分布在一个dnode，所以如果创建3副本的数据库，则dnode数大于等于3 +- 3个mnode：nmode负责整个集群的管理工作，TDengine默认是一个mnode，此时如果mnode所在的dnode掉线，则此时整个集群不可用 +- 数据库3副本，TDengine的副本配置是DB级别，3副本可满足，在3节点的集群中，任意一个dnode下线，都不影响集群的正常使用，**如果下线个数为2时，此时集群不可用，RAFT无法完成选举**，（企业版：在灾难恢复场景，任一节点数据文件损坏，都可以通过重新拉起dnode进行恢复） ## 前置条件 要使用 Kubernetes 部署管理 TDengine 集群，需要做好如下准备工作。 -* 本文适用 Kubernetes v1.5 以上版本 -* 本文和下一章使用 minikube、kubectl 和 helm 等工具进行安装部署，请提前安装好相应软件 -* Kubernetes 已经安装部署并能正常访问使用或更新必要的容器仓库或其他服务 +- 本文适用 Kubernetes v1.5 以上版本 +- 本文和下一章使用 minikube、kubectl 和 helm 等工具进行安装部署，请提前安装好相应软件 +- Kubernetes 已经安装部署并能正常访问使用或更新必要的容器仓库或其他服务 以下配置文件也可以从 [GitHub 仓库](https://github.com/taosdata/TDengine-Operator/tree/3.0/src/tdengine) 下载。 @@ -20,7 +28,7 @@ description: 利用 Kubernetes 部署 TDengine 集群的详细指南 创建一个 Service 配置文件：`taosd-service.yaml`，服务名称 `metadata.name` (此处为 "taosd") 将在下一步中使用到。添加 TDengine 所用到的端口： -```yaml +```YAML --- apiVersion: v1 kind: Service @@ -42,10 +50,11 @@ spec: ## 有状态服务 StatefulSet -根据 Kubernetes 对各类部署的说明，我们将使用 StatefulSet 作为 TDengine 的服务类型。 -创建文件 `tdengine.yaml`，其中 replicas 定义集群节点的数量为 3。节点时区为中国（Asia/Shanghai），每个节点分配 10G 标准（standard）存储。你也可以根据实际情况进行相应修改。 +根据 Kubernetes 对各类部署的说明，我们将使用 StatefulSet 作为 TDengine 的服务类型。 创建文件 `tdengine.yaml`，其中 replicas 定义集群节点的数量为 3。节点时区为中国（Asia/Shanghai），每个节点分配 5G 标准（standard）存储（参考[Storage Classes](https://kubernetes.io/docs/concepts/storage/storage-classes/) 配置storage class）。你也可以根据实际情况进行相应修改。 -```yaml +需要注意startupProbe的配置，在dnode 掉线一段时间后，重新启动，新上线的dnode会短暂不可用，如果startupProbe配置过小，Kubernetes会认为pod处于不正常的状态，会尝试重新拉起pod，此时，dnode会频繁重启，始终无法恢复。参考 [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) + +```YAML --- apiVersion: apps/v1 kind: StatefulSet @@ -69,7 +78,7 @@ spec: spec: containers: - name: "tdengine" - image: "tdengine/tdengine:3.0.0.0" + image: "tdengine/tdengine:3.0.7.1" imagePullPolicy: "IfNotPresent" ports: - name: tcp6030 @@ -108,6 +117,12 @@ spec: volumeMounts: - name: taosdata mountPath: /var/lib/taos + startupProbe: + exec: + command: + - taos-check + failureThreshold: 360 + periodSeconds: 10 readinessProbe: exec: command: @@ -129,199 +144,373 @@ spec: storageClassName: "standard" resources: requests: - storage: "10Gi" + storage: "5Gi" ``` ## 使用 kubectl 命令部署 TDengine 集群 -顺序执行以下命令。 +顺序执行以下命令，需要提前创建对应的namespace。 -```bash -kubectl apply -f taosd-service.yaml -kubectl apply -f tdengine.yaml +```Bash +kubectl apply -f taosd-service.yaml -n tdengine-test +kubectl apply -f tdengine.yaml -n tdengine-test ``` 上面的配置将生成一个三节点的 TDengine 集群，dnode 为自动配置，可以使用 show dnodes 命令查看当前集群的节点： -```bash -kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" -kubectl exec -i -t tdengine-1 -- taos -s "show dnodes" -kubectl exec -i -t tdengine-2 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show dnodes" +kubectl exec -it tdengine-2 -n tdengine-test -- taos -s "show dnodes" ``` 输出如下： -``` +```Bash taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | -Query OK, 3 rows in database (0.003655s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 0 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-19 17:54:18.469 | | | | + 2 | tdengine-1.ta... | 0 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-19 17:54:38.698 | | | | + 3 | tdengine-2.ta... | 0 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-19 17:55:02.039 | | | | +Query OK, 3 row(s) in set (0.001853s) +``` + +查看当前mnode + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-19 17:54:18.559 +reboot_time: 2023-07-19 17:54:19.520 +Query OK, 1 row(s) in set (0.001282s) +``` + +## 创建mnode + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 2" +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "create mnode on dnode 3" +``` + +查看mnode + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" + +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-19 17:54:18.559 +reboot_time: 2023-07-20 09:19:36.060 +*************************** 2.row *************************** + id: 2 + endpoint: tdengine-1.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:05.600 +reboot_time: 2023-07-20 09:22:12.838 +*************************** 3.row *************************** + id: 3 + endpoint: tdengine-2.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:20.042 +reboot_time: 2023-07-20 09:22:23.271 +Query OK, 3 row(s) in set (0.003108s) ``` ## 使能端口转发 利用 kubectl 端口转发功能可以使应用可以访问 Kubernetes 环境运行的 TDengine 集群。 -``` -kubectl port-forward tdengine-0 6041:6041 & +```bash +kubectl port-forward -n tdengine-test tdengine-0 6041:6041 & ``` 使用 curl 命令验证 TDengine REST API 使用的 6041 接口。 -``` -$ curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql -Handling connection for 6041 -{"code":0,"column_meta":[["name","VARCHAR",64],["create_time","TIMESTAMP",8],["vgroups","SMALLINT",2],["ntables","BIGINT",8],["replica","TINYINT",1],["strict","VARCHAR",4],["duration","VARCHAR",10],["keep","VARCHAR",32],["buffer","INT",4],["pagesize","INT",4],["pages","INT",4],["minrows","INT",4],["maxrows","INT",4],["comp","TINYINT",1],["precision","VARCHAR",2],["status","VARCHAR",10],["retention","VARCHAR",60],["single_stable","BOOL",1],["cachemodel","VARCHAR",11],["cachesize","INT",4],["wal_level","TINYINT",1],["wal_fsync_period","INT",4],["wal_retention_period","INT",4],["wal_retention_size","BIGINT",8],["wal_roll_period","INT",4],["wal_segment_size","BIGINT",8]],"data":[["information_schema",null,null,16,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null],["performance_schema",null,null,10,null,null,null,null,null,null,null,null,null,null,null,"ready",null,null,null,null,null,null,null,null,null,null]],"rows":2} +```bash +curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql +{"code":0,"column_meta":[["name","VARCHAR",64]],"data":[["information_schema"],["performance_schema"],["test"],["test1"]],"rows":4} ``` -## 使用 dashboard 进行图形化管理 +## 集群测试 - minikube 提供 dashboard 命令支持图形化管理界面。 +### 数据准备 -``` -$ minikube dashboard -* Verifying dashboard health ... -* Launching proxy ... -* Verifying proxy health ... -* Opening http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ in your default browser... -http://127.0.0.1:46617/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ +#### taosBenchmark + +通过taosBenchmark 创建一个3副本的数据库，同时写入1亿条数据，同时查看数据 + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taosBenchmark -I stmt -d test -n 10000 -t 10000 -a 3 + +# query data +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "select count(*) from test.meters;" + +taos> select count(*) from test.meters; + count(*) | +======================== + 100000000 | +Query OK, 1 row(s) in set (0.103537s) ``` -对于某些公有云环境，minikube 绑定在 127.0.0.1 IP 地址上无法通过远程访问，需要使用 kubectl proxy 命令将端口映射到 0.0.0.0 IP 地址上，再通过浏览器访问虚拟机公网 IP 和端口以及相同的 dashboard URL 路径即可远程访问 dashboard。 +查看vnode分布，通过show dnodes +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" + +taos> show dnodes + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 8 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-19 17:54:18.469 | | | | + 2 | tdengine-1.ta... | 8 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-19 17:54:38.698 | | | | + 3 | tdengine-2.ta... | 8 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-19 17:55:02.039 | | | | +Query OK, 3 row(s) in set (0.001357s) ``` -$ kubectl proxy --accept-hosts='^.*$' --address='0.0.0.0' + +通过show vgroup 查看xnode分布情况 + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test.vgroups" + +taos> show test.vgroups + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | v4_status | cacheload | cacheelements | tsma | +============================================================================================================================================================================================== + 2 | test | 1267 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 3 | test | 1215 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 4 | test | 1215 | 1 | leader | 2 | follower | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 5 | test | 1307 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 6 | test | 1245 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 7 | test | 1275 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 8 | test | 1231 | 1 | leader | 2 | follower | 3 | follower | NULL | NULL | 0 | 0 | 0 | + 9 | test | 1245 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | +Query OK, 8 row(s) in set (0.001488s) ``` +#### 手工创建 + +常见一个三副本的test1，并创建一张表，写入2条数据 + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- \ + taos -s \ + "create database if not exists test1 replica 3; + use test1; + create table if not exists t1(ts timestamp, n int); + insert into t1 values(now, 1)(now+1s, 2);" +``` + +通过show test1.vgroup 查看xnode分布情况 + +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test1.vgroups" + +taos> show test1.vgroups + vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | v4_status | cacheload | cacheelements | tsma | +============================================================================================================================================================================================== + 10 | test1 | 1 | 1 | follower | 2 | follower | 3 | leader | NULL | NULL | 0 | 0 | 0 | + 11 | test1 | 0 | 1 | follower | 2 | leader | 3 | follower | NULL | NULL | 0 | 0 | 0 | +Query OK, 2 row(s) in set (0.001489s) +``` + +### 容错测试 + +Mnode leader 所在的dnode掉线，dnode1 + +```Bash +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 0/1 ErrImagePull 2 (2s ago) 20m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (6m48s ago) 20m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 21m 10.244.1.223 node85 +``` + +此时集群mnode发生重新选举，dnode1上的monde 成为leader + +```Bash +kubectl exec -it tdengine-1 -n tdengine-test -- taos -s "show mnodes\G" +Welcome to the TDengine Command Line Interface, Client Version:3.0.7.1.202307190706 +Copyright (c) 2022 by TDengine, all rights reserved. + +taos> show mnodes\G +*************************** 1.row *************************** + id: 1 + endpoint: tdengine-0.taosd.tdengine-test.svc.cluster.local:6030 + role: offline + status: offline +create_time: 2023-07-19 17:54:18.559 +reboot_time: 1970-01-01 08:00:00.000 +*************************** 2.row *************************** + id: 2 + endpoint: tdengine-1.taosd.tdengine-test.svc.cluster.local:6030 + role: leader + status: ready +create_time: 2023-07-20 09:22:05.600 +reboot_time: 2023-07-20 09:32:00.227 +*************************** 3.row *************************** + id: 3 + endpoint: tdengine-2.taosd.tdengine-test.svc.cluster.local:6030 + role: follower + status: ready +create_time: 2023-07-20 09:22:20.042 +reboot_time: 2023-07-20 09:32:00.026 +Query OK, 3 row(s) in set (0.001513s) +``` + +集群可以正常读写 + +```Bash +# insert +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "insert into test1.t1 values(now, 1)(now+1s, 2);" + +taos> insert into test1.t1 values(now, 1)(now+1s, 2); +Insert OK, 2 row(s) affected (0.002098s) + +# select +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "select *from test1.t1" + +taos> select *from test1.t1 + ts | n | +======================================== + 2023-07-19 18:04:58.104 | 1 | + 2023-07-19 18:04:59.104 | 2 | + 2023-07-19 18:06:00.303 | 1 | + 2023-07-19 18:06:01.303 | 2 | +Query OK, 4 row(s) in set (0.001994s) +``` + +同理，至于非leader得mnode掉线，读写当然可以正常进行，这里就不做过多的展示， + ## 集群扩容 TDengine 集群支持自动扩容： -```bash +```Bash kubectl scale statefulsets tdengine --replicas=4 ``` 上面命令行中参数 `--replica=4` 表示要将 TDengine 集群扩容到 4 个节点，执行后首先检查 POD 的状态： -```bash -kubectl get pods -l app=tdengine +```Bash +kubectl get pod -l app=tdengine -n tdengine-test -o wide ``` 输出如下： -``` -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 161m -tdengine-1 1/1 Running 0 161m -tdengine-2 1/1 Running 0 32m -tdengine-3 1/1 Running 0 32m +```Plain +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h26m ago) 6h53m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (6h39m ago) 6h53m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h16m 10.244.1.224 node85 +tdengine-3 1/1 Running 0 3m24s 10.244.2.76 node86 ``` 此时 POD 的状态仍然是 Running，TDengine 集群中的 dnode 状态要等 POD 状态为 `ready` 之后才能看到： -```bash -kubectl exec -i -t tdengine-3 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-3 -n tdengine-test -- taos -s "show dnodes" ``` 扩容后的四节点 TDengine 集群的 dnode 列表: -``` +```Plain taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | - 4 | tdengine-3.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:33:16.039 | | -Query OK, 4 rows in database (0.008377s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 4 | tdengine-3.ta... | 0 | 16 | ready | 2023-07-20 16:01:44.007 | 2023-07-20 16:01:44.889 | | | | +Query OK, 4 row(s) in set (0.003628s) ``` ## 集群缩容 -由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令，节点删除完成后再进行 Kubernetes 集群缩容。 +由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令（**如果集群中存在3副本的db，那么缩容后的dnode个数也要必须大于等于3，否则drop dnode操作会被中止**），节点删除完成后再进行 Kubernetes 集群缩容。 注意：由于 Kubernetes Statefulset 中 Pod 的只能按创建顺序逆序移除，所以 TDengine drop dnode 也需要按照创建顺序逆序移除，否则会导致 Pod 处于错误状态。 -``` -$ kubectl exec -i -t tdengine-0 -- taos -s "drop dnode 4" -``` - -```bash -$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" +```Bash +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "drop dnode 4" +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | note | -============================================================================================================================================ - 1 | tdengine-0.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:14:57.285 | | - 2 | tdengine-1.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:11.302 | | - 3 | tdengine-2.taosd.default.sv... | 0 | 256 | ready | 2022-08-10 13:15:23.290 | | -Query OK, 3 rows in database (0.004861s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | +Query OK, 3 row(s) in set (0.003324s) ``` 确认移除成功后（使用 kubectl exec -i -t tdengine-0 -- taos -s "show dnodes" 查看和确认 dnode 列表），使用 kubectl 命令移除 POD： -``` -kubectl scale statefulsets tdengine --replicas=3 +```Plain +kubectl scale statefulsets tdengine --replicas=3 -n tdengine-test ``` 最后一个 POD 将会被删除。使用命令 kubectl get pods -l app=tdengine 查看POD状态： -``` -$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 4m7s -tdengine-1 1/1 Running 0 3m55s -tdengine-2 1/1 Running 0 2m28s +```Plain +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h55m ago) 7h22m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (7h9m ago) 7h23m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h45m 10.244.1.224 node85 ``` POD删除后，需要手动删除PVC，否则下次扩容时会继续使用以前的数据导致无法正常加入集群。 -```bash -$ kubectl delete pvc taosdata-tdengine-3 +```Bash +kubectl delete pvc aosdata-tdengine-3 -n tdengine-test ``` 此时的集群状态是安全的，需要时还可以再次进行扩容： -```bash -$ kubectl scale statefulsets tdengine --replicas=4 +```Bash +kubectl scale statefulsets tdengine --replicas=4 -n tdengine-test statefulset.apps/tdengine scaled -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 35m -tdengine-1 1/1 Running 0 34m -tdengine-2 1/1 Running 0 12m -tdengine-3 0/1 ContainerCreating 0 4s -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl get pods -l app=tdengine -NAME READY STATUS RESTARTS AGE -tdengine-0 1/1 Running 0 35m -tdengine-1 1/1 Running 0 34m -tdengine-2 1/1 Running 0 12m -tdengine-3 0/1 Running 0 7s -it@k8s-2:~/TDengine-Operator/src/tdengine$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" + +kubectl get pod -l app=tdengine -n tdengine-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +tdengine-0 1/1 Running 4 (6h59m ago) 7h27m 10.244.2.75 node86 +tdengine-1 1/1 Running 1 (7h13m ago) 7h27m 10.244.0.59 node84 +tdengine-2 1/1 Running 0 5h49m 10.244.1.224 node85 +tdengine-3 1/1 Running 0 20s 10.244.2.77 node86 + +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes -id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | -====================================================================================================================================== -1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | -2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | -5 | tdengine-2.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:01:36.479 | | -6 | tdengine-3.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 18:13:54.411 | | -Query OK, 4 row(s) in set (0.001348s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 5 | tdengine-3.ta... | 0 | 16 | ready | 2023-07-20 16:31:34.092 | 2023-07-20 16:38:17.419 | | | | +Query OK, 4 row(s) in set (0.003881s) ``` ## 清理 TDengine 集群 +> **删除pvc时需要注意下pv persistentVolumeReclaimPolicy策略，建议改为Delete，这样在删除pvc时才会自动清理pv，同时会清理底层的csi存储资源，如果没有配置删除pvc自动清理pv的策略，再删除pvc后，在手动清理pv时，pv对应的csi存储资源可能不会被释放。** + 完整移除 TDengine 集群，需要分别清理 statefulset、svc、configmap、pvc。 -```bash -kubectl delete statefulset -l app=tdengine -kubectl delete svc -l app=tdengine -kubectl delete pvc -l app=tdengine -kubectl delete configmap taoscfg - +```Bash +kubectl delete statefulset -l app=tdengine -n tdengine-test +kubectl delete svc -l app=tdengine -n tdengine-test +kubectl delete pvc -l app=tdengine -n tdengine-test +kubectl delete configmap taoscfg -n tdengine-test ``` ## 常见错误 @@ -330,65 +519,26 @@ kubectl delete configmap taoscfg 未进行 "drop dnode" 直接进行缩容，由于 TDengine 尚未删除节点，缩容 pod 导致 TDengine 集群中部分节点处于 offline 状态。 -``` -$ kubectl exec -it tdengine-0 -- taos -s "show dnodes" +```Plain +kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes -id | endpoint | vnodes | support_vnodes | status | create_time | offline reason | -====================================================================================================================================== -1 | tdengine-0.taosd.default.sv... | 0 | 4 | ready | 2022-07-25 17:38:49.012 | | -2 | tdengine-1.taosd.default.sv... | 1 | 4 | ready | 2022-07-25 17:39:01.517 | | -5 | tdengine-2.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:01:36.479 | status msg timeout | -6 | tdengine-3.taosd.default.sv... | 0 | 4 | offline | 2022-07-25 18:13:54.411 | status msg timeout | -Query OK, 4 row(s) in set (0.001323s) + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | +============================================================================================================================================================================================================================================= + 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | + 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | + 3 | tdengine-2.ta... | 10 | 16 | ready | 2023-07-19 17:55:01.141 | 2023-07-20 10:48:43.445 | | | | + 5 | tdengine-3.ta... | 0 | 16 | offline | 2023-07-20 16:31:34.092 | 2023-07-20 16:38:17.419 | status msg timeout | | | +Query OK, 4 row(s) in set (0.003862s) ``` -### 错误二 +## 最后 -TDengine 集群会持有 replica 参数，如果缩容后的节点数小于这个值，集群将无法使用： +对于在k8s环境下TDengine高可用、高可靠来说，对于硬件损坏、灾难恢复，分为两个层面来讲： -创建一个库使用 replica 参数为 2，插入部分数据： +1. 底层的分布式块存储具备的灾难恢复能力，块存储的多副本，当下流行的分布式块存储如ceph，就具备多副本能力，将存储副本扩展到不同的机架、机柜、机房、数据中心（或者直接使用公有云厂商提供的块存储服务） +2. TDengine的灾难恢复，在TDengine Enterprise中，本身具备了当一个dnode永久下线（物理机磁盘损坏，数据分拣丢失）后，重新拉起一个空白的dnode来恢复原dnode的工作。 -```bash -kubectl exec -i -t tdengine-0 -- \ - taos -s \ - "create database if not exists test replica 2; - use test; - create table if not exists t1(ts timestamp, n int); - insert into t1 values(now, 1)(now+1s, 2);" +最后，欢迎使用[TDengine Cloud](https://cloud.taosdata.com/)，来体验一站式全托管的TDengine云服务。 - -``` - -缩容到单节点： - -```bash -kubectl scale statefulsets tdengine --replicas=1 - -``` - -在 TDengine CLI 中的所有数据库操作将无法成功。 - -``` -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 2 | 40 | ready | any | 2021-06-01 15:55:52.562 | | - 2 | tdengine-1.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 15:56:07.212 | status msg timeout | -Query OK, 2 row(s) in set (0.000845s) - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | offline reason | -====================================================================================================================================== - 1 | tdengine-0.taosd.default.sv... | 2 | 40 | ready | any | 2021-06-01 15:55:52.562 | | - 2 | tdengine-1.taosd.default.sv... | 1 | 40 | offline | any | 2021-06-01 15:56:07.212 | status msg timeout | -Query OK, 2 row(s) in set (0.000837s) - -taos> use test; -Database changed. - -taos> insert into t1 values(now, 3); - -DB error: Unable to resolve FQDN (0.013874s) - -``` +> TDengine Cloud 是一个极简的全托管时序数据处理云服务平台，它是基于开源的时序数据库 TDengine 而开发的。除高性能的时序数据库之外，它还具有缓存、订阅和流计算等系统功能，而且提供了便利而又安全的数据分享、以及众多的企业级功能。它可以让物联网、工业互联网、金融、IT 运维监控等领域企业在时序数据的管理上大幅降低人力成本和运营成本。 From 95a1db6e89a77fc3c152567bd4d9167d90256e0d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 24 Jul 2023 07:09:03 +0000 Subject: [PATCH 085/100] fix err while connect invalid fqdn --- source/client/src/clientImpl.c | 23 +++++++++++++++++++---- source/client/src/clientMsgHandler.c | 15 +++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 14d6394fc4..f5dc627dd8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1297,13 +1297,20 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe return -1; } - int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return terrno; } - - mgmtEpSet->numOfEps++; + uint32_t addr = taosGetIpv4FromFqdn(mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn); + if (addr == 0xffffffff) { + int32_t code = TAOS_SYSTEM_ERROR(errno); + tscError("failed to resolve firstEp fqdn: %s, code:%s", mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn, + tstrerror(errno)); + memset(&(mgmtEpSet->eps[mgmtEpSet->numOfEps]), 0, sizeof(mgmtEpSet->eps[mgmtEpSet->numOfEps])); + } else { + mgmtEpSet->numOfEps++; + } } if (secondEp && secondEp[0] != 0) { @@ -1313,7 +1320,15 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe } taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); - mgmtEpSet->numOfEps++; + uint32_t addr = taosGetIpv4FromFqdn(mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn); + if (addr == 0xffffffff) { + int32_t code = TAOS_SYSTEM_ERROR(errno); + tscError("failed to resolve secondEp fqdn: %s, code:%s", mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn, + tstrerror(errno)); + memset(&(mgmtEpSet->eps[mgmtEpSet->numOfEps]), 0, sizeof(mgmtEpSet->eps[mgmtEpSet->numOfEps])); + } else { + mgmtEpSet->numOfEps++; + } } if (mgmtEpSet->numOfEps == 0) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 9ab618cf3a..7455a2c1c8 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -99,13 +99,20 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { goto End; } + int updateEpSet = 1; if (connectRsp.dnodeNum == 1) { SEpSet srcEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); SEpSet dstEpSet = connectRsp.epSet; - rpcSetDefaultAddr(pTscObj->pAppInfo->pTransporter, srcEpSet.eps[srcEpSet.inUse].fqdn, - dstEpSet.eps[dstEpSet.inUse].fqdn); - } else if (connectRsp.dnodeNum > 1 && !isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { - SEpSet* pOrig = &pTscObj->pAppInfo->mgmtEp.epSet; + if (srcEpSet.numOfEps == 1) { + rpcSetDefaultAddr(pTscObj->pAppInfo->pTransporter, srcEpSet.eps[srcEpSet.inUse].fqdn, + dstEpSet.eps[dstEpSet.inUse].fqdn); + updateEpSet = 0; + } + } + if (updateEpSet == 1 && !isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { + SEpSet corEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + SEpSet* pOrig = &corEpSet; SEp* pOrigEp = &pOrig->eps[pOrig->inUse]; SEp* pNewEp = &connectRsp.epSet.eps[connectRsp.epSet.inUse]; tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in connRsp", pOrig->inUse, pOrig->numOfEps, From dc77e783a5e11bb69fc0b062ba9bb2cc228b29df Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Mon, 24 Jul 2023 10:59:30 +0800 Subject: [PATCH 086/100] fix: monitor no longer reports logs --- source/libs/monitor/src/monMain.c | 2 +- tests/system-test/0-others/taosdMonitor.py | 28 ---------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index 949e91198a..8f94bfdb96 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -547,7 +547,7 @@ void monSendReport() { monGenGrantJson(pMonitor); monGenDnodeJson(pMonitor); monGenDiskJson(pMonitor); - monGenLogJson(pMonitor); + //monGenLogJson(pMonitor); // TS-3691 char *pCont = tjsonToString(pMonitor->pJson); // uDebugL("report cont:%s\n", pCont); diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 8094c4e0f5..6c21eb8daa 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -186,33 +186,6 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): tdLog.exit("total is null!") - # log_infos ==================================== - - if "log_infos" not in infoDict or infoDict["log_infos"]== None: - tdLog.exit("log_infos is null!") - - if "logs" not in infoDict["log_infos"] or len(infoDict["log_infos"]["logs"]) < 8:#!= 10: - tdLog.exit("logs is null!") - - if "ts" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 10: - tdLog.exit("ts is null!") - - if "level" not in infoDict["log_infos"]["logs"][0] or infoDict["log_infos"]["logs"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: - tdLog.exit("level is null!") - - if "content" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 1: - tdLog.exit("content is null!") - - if "summary" not in infoDict["log_infos"] or len(infoDict["log_infos"]["summary"])!= 4: - tdLog.exit("summary is null!") - - - if "total" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["total"] < 0 : - tdLog.exit("total is null!") - - if "level" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: - tdLog.exit("level is null!") - def do_GET(self): """ process GET request @@ -315,4 +288,3 @@ class TDTestCase: tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) - From 9628a9f74ebb40d4808b56d5545c1967da859006 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Jul 2023 15:53:02 +0800 Subject: [PATCH 087/100] fix:open info log in tmq & ignore wal apply ver when read wal --- source/client/src/clientTmq.c | 2 +- source/dnode/mnode/impl/src/mndConsumer.c | 16 ++++++++-------- source/dnode/mnode/impl/src/mndSubscribe.c | 4 ++-- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/libs/wal/src/walRead.c | 21 +++++++++++---------- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 96ec88e2e9..fa2e250b2b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -712,7 +712,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us end: taosMemoryFree(pParamSet); - if(pParamSet->callbackFn != NULL) { + if(pCommitFp != NULL) { pCommitFp(tmq, code, userParam); } return; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 2b538eccc9..50115db81c 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -94,7 +94,7 @@ void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId){ bool mndRebTryStart() { int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); - mDebug("tq timer, rebalance counter old val:%d", old); + mInfo("tq timer, rebalance counter old val:%d", old); return old == 0; } @@ -116,7 +116,7 @@ void mndRebCntDec() { int32_t newVal = val - 1; int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal); if (oldVal == val) { - mDebug("rebalance trans end, rebalance counter:%d", newVal); + mInfo("rebalance trans end, rebalance counter:%d", newVal); break; } } @@ -281,7 +281,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { // rebalance cannot be parallel if (!mndRebTryStart()) { - mDebug("mq rebalance already in progress, do nothing"); + mInfo("mq rebalance already in progress, do nothing"); return 0; } @@ -312,7 +312,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); int32_t status = atomic_load_32(&pConsumer->status); - mDebug("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d", + mInfo("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d", pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->createTime, hbStatus); @@ -362,7 +362,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { - mInfo("mq rebalance will be triggered"); + mInfo("mq rebalance will be triggered"); SRpcMsg rpcMsg = { .msgType = TDMT_MND_TMQ_DO_REBALANCE, .pCont = pRebMsg, @@ -416,7 +416,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { for(int i = 0; i < taosArrayGetSize(req.topics); i++){ TopicOffsetRows* data = taosArrayGet(req.topics, i); - mDebug("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName); + mInfo("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName); if(pSub == NULL){ @@ -1104,13 +1104,13 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * } if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { - mDebug("showing consumer:0x%" PRIx64 " no assigned topic, skip", pConsumer->consumerId); + mInfo("showing consumer:0x%" PRIx64 " no assigned topic, skip", pConsumer->consumerId); sdbRelease(pSdb, pConsumer); continue; } taosRLockLatch(&pConsumer->lock); - mDebug("showing consumer:0x%" PRIx64, pConsumer->consumerId); + mInfo("showing consumer:0x%" PRIx64, pConsumer->consumerId); int32_t topicSz = taosArrayGetSize(pConsumer->assignedTopics); bool hasTopic = true; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index f51a61eda3..6bd23c3b90 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -1207,7 +1207,7 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock int32_t numOfRows = 0; SMqSubscribeObj *pSub = NULL; - mDebug("mnd show subscriptions begin"); + mInfo("mnd show subscriptions begin"); while (numOfRows < rowsCapacity) { pShow->pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pShow->pIter, (void **)&pSub); @@ -1247,7 +1247,7 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock sdbRelease(pSdb, pSub); } - mDebug("mnd end show subscriptions"); + mInfo("mnd end show subscriptions"); pShow->numOfRows += numOfRows; return numOfRows; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8c9eead414..89ed3ca1c7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -703,7 +703,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; int32_t vgId = TD_VID(pTq->pVnode); - tqDebug("vgId:%d, tq process delete sub req %s", vgId, pReq->subKey); + tqInfo("vgId:%d, tq process delete sub req %s", vgId, pReq->subKey); int32_t code = 0; taosWLockLatch(&pTq->lock); @@ -784,7 +784,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg return -1; } - tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey, + tqInfo("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey, req.oldConsumerId, req.newConsumerId); STqHandle* pHandle = NULL; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index a839d6cbd8..7ff7fe748e 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -70,17 +70,18 @@ int32_t walNextValidMsg(SWalReader *pReader) { int64_t fetchVer = pReader->curVersion; int64_t lastVer = walGetLastVer(pReader->pWal); int64_t committedVer = walGetCommittedVer(pReader->pWal); - int64_t appliedVer = walGetAppliedVer(pReader->pWal); +// int64_t appliedVer = walGetAppliedVer(pReader->pWal); - if(appliedVer < committedVer){ // wait apply ver equal to commit ver, otherwise may lost data when consume data [TD-24010] - wDebug("vgId:%d, wal apply ver:%"PRId64" smaller than commit ver:%"PRId64, pReader->pWal->cfg.vgId, appliedVer, committedVer); - } +// if(appliedVer < committedVer){ // wait apply ver equal to commit ver, otherwise may lost data when consume data [TD-24010] +// wDebug("vgId:%d, wal apply ver:%"PRId64" smaller than commit ver:%"PRId64, pReader->pWal->cfg.vgId, appliedVer, committedVer); +// } - int64_t endVer = TMIN(appliedVer, committedVer); +// int64_t endVer = TMIN(appliedVer, committedVer); + int64_t endVer = committedVer; wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 - ", applied index:%" PRId64", end index:%" PRId64, - pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer, endVer); + ", end index:%" PRId64, + pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, endVer); if (fetchVer > endVer){ terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; @@ -370,9 +371,9 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { pRead->pWal->vers.appliedVer); // TODO: valid ver - if (ver > pRead->pWal->vers.appliedVer) { - return -1; - } +// if (ver > pRead->pWal->vers.appliedVer) { +// return -1; +// } if (pRead->curVersion != ver) { code = walReaderSeekVer(pRead, ver); From d1de8fe878f9a6a04d88a9b12db472ba5996836c Mon Sep 17 00:00:00 2001 From: liuyuan <2805658706@qq.com> Date: Mon, 24 Jul 2023 16:03:03 +0800 Subject: [PATCH 088/100] docs: optimization k8s deploy --- docs/en/10-deployment/03-k8s.md | 36 +++++++++++++-------------- docs/zh/10-deployment/03-k8s.md | 44 ++++++++++++++++----------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/docs/en/10-deployment/03-k8s.md b/docs/en/10-deployment/03-k8s.md index c72234b336..939dea6f80 100644 --- a/docs/en/10-deployment/03-k8s.md +++ b/docs/en/10-deployment/03-k8s.md @@ -10,17 +10,17 @@ As a time series database for Cloud Native architecture design, TDengine support To meet [high availability ](https://docs.taosdata.com/tdinternal/high-availability/)requirements, clusters need to meet the following requirements: -- 3 or more dnodes: The vnodes in the vgroup of TDengine are not allowed to be distributed in one dnode at the same time, so if you create a database with 3 copies, the number of dnodes is greater than or equal to 3 -- 3 mnodes: nmode is responsible for the management of the entire cluster. TDengine defaults to an mnode. At this time, if the dnode where the mnode is located is dropped, the entire cluster is unavailable at this time -- There are 3 copies of the database, and the copy configuration of TDengine is DB level, which can be satisfied with 3 copies. In a 3-node cluster, any dnode goes offline, which does not affect the normal use of the cluster. **If the number of offline is 2, the cluster is unavailable at this time, and RAFT cannot complete the election** , (Enterprise Edition: In the disaster recovery scenario, any node data file is damaged, which can be restored by pulling up the dnode again) +- 3 or more dnodes : multiple vnodes in the same vgroup of TDengine are not allowed to be distributed in one dnode at the same time, so if you create a database with 3 copies, the number of dnodes is greater than or equal to 3 +- 3 mnodes : m n ode is responsible for the management of the entire cluster, TDengine defaults to an mnode . If the dnode where the mnode is located is dropped, the entire cluster is unavailable . +- Database 3 replicas : TDengine replica configuration is the database level, so database 3 replicas can meet the three dnode cluster, any one dnode offline, does not affect the normal use of the cluster . **If the number of offline** **dnodes** **is 2, then the cluster is not available,** **because** **RAFT can not complete the election** **.** (Enterprise version: in the disaster recovery scenario, any node data file is damaged, can be restored by pulling up the dnode again) ## Prerequisites Before deploying TDengine on Kubernetes, perform the following: -- Current steps are compatible with Kubernetes v1.5 and later version. -- Install and configure minikube, kubectl, and helm. -- Install and deploy Kubernetes and ensure that it can be accessed and used normally. Update any container registries or other services as necessary. +- This article applies Kubernetes 1.19 and above +- This article uses the kubectl tool to install and deploy, please install the corresponding software in advance +- Kubernetes have been installed and deployed and can access or update the necessary container repositories or other services You can download the configuration files in this document from [GitHub](https://github.com/taosdata/TDengine-Operator/tree/3.0/src/tdengine). @@ -52,7 +52,7 @@ spec: According to Kubernetes instructions for various deployments, we will use StatefulSet as the service type of TDengine. Create the file `tdengine.yaml `, where replicas defines the number of cluster nodes as 3. The node time zone is China (Asia/Shanghai), and each node is allocated 5G standard storage (refer to the [Storage Classes ](https://kubernetes.io/docs/concepts/storage/storage-classes/)configuration storage class). You can also modify accordingly according to the actual situation. -You need to pay attention to the configuration of startupProbe. After the dnode is disconnected for a period of time, restart, and the newly launched dnode will be temporarily unavailable. If the startupProbe configuration is too small, Kubernetes will think that the pod is in an abnormal state and will try to pull the pod again. At this time, dnode will restart frequently and never recover. Refer to [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) +Please pay special attention to the startupProbe configuration, after dnode 's pod drops for a period of time, then restart, this time the newly launched dnode will be temporarily unavailable . If the startupProbe configuration is too small, Kubernetes will think that the Pod is in an abnormal state , and try to restart the Pod, the dnode 's Pod will restart frequently and never return to the normal state . Refer to [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) ```YAML --- @@ -149,7 +149,7 @@ spec: ## Use kubectl to deploy TDengine -Execute the following commands in sequence, and you need to create the corresponding namespace in advance. +First create the corresponding namespace, and then execute the following command in sequence : ```Bash kubectl apply -f taosd-service.yaml -n tdengine-test @@ -230,7 +230,7 @@ Query OK, 3 row(s) in set (0.003108s) ## Enable port forwarding -The kubectl port forwarding feature allows applications to access the TDengine cluster running on Kubernetes. +Kubectl port forwarding enables applications to access TDengine clusters running in Kubernetes environments. ```bash kubectl port-forward -n tdengine-test tdengine-0 6041:6041 & @@ -325,7 +325,7 @@ Query OK, 2 row(s) in set (0.001489s) ### Test fault tolerance -The dnode where the Mnode leader is located is offline, dnode1 +The dnode where the mnode leader is located is disconnected, dnode1 ```Bash kubectl get pod -l app=tdengine -n tdengine-test -o wide @@ -389,7 +389,7 @@ taos> select *from test1.t1 Query OK, 4 row(s) in set (0.001994s) ``` -In the same way, as for the mnode dropped by the non-leader, reading and writing can of course be performed normally, so there will be no too much display here. +Similarly, as for the non-leader mnode dropped, read and write can of course be normal, here will not do too much display . ## Scaling Out Your Cluster @@ -425,7 +425,7 @@ The dnode list of the expanded four-node TDengine cluster: ```Plain taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | ============================================================================================================================================================================================================================================= 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | @@ -469,7 +469,7 @@ tdengine-1 1/1 Running 1 (7h9m ago) 7h23m 10.244.0.59 node84 < tdengine-2 1/1 Running 0 5h45m 10.244.1.224 node85 ``` -After the POD is deleted, the PVC needs to be deleted manually, otherwise the previous data will continue to be used in the next expansion, resulting in the inability to join the cluster normally. +After the POD is deleted, the PVC needs to be deleted manually, otherwise the previous data will continue to be used for the next expansion, resulting in the inability to join the cluster normally. ```Bash kubectl delete pvc aosdata-tdengine-3 -n tdengine-test @@ -491,7 +491,7 @@ tdengine-3 1/1 Running 0 20s 10.244.2.77 node86 < kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | ============================================================================================================================================================================================================================================= 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | @@ -504,7 +504,7 @@ Query OK, 4 row(s) in set (0.003881s) > **When deleting the pvc, you need to pay attention to the pv persistentVolumeReclaimPolicy policy. It is recommended to change to Delete, so that the pv will be automatically cleaned up when the pvc is deleted, and the underlying csi storage resources will be cleaned up at the same time. If the policy of deleting the pvc to automatically clean up the pv is not configured, and then after deleting the pvc, when manually cleaning up the pv, the csi storage resources corresponding to the pv may not be released.** -Complete removal of TDengine cluster, need to clean statefulset, svc, configmap, pvc respectively. +Complete removal of TDengine cluster, need to clean up statefulset, svc, configmap, pvc respectively. ```Bash kubectl delete statefulset -l app=tdengine -n tdengine-test @@ -534,10 +534,10 @@ Query OK, 4 row(s) in set (0.003862s) ## Finally -For the high availability and high reliability of TDengine in the k8s environment, for hardware damage and disaster recovery, it is divided into two levels: +For the high availability and high reliability of TDengine in a Kubernetes environment, hardware damage and disaster recovery are divided into two levels: -1. The disaster recovery capability of the underlying distributed Block Storage, the multi-replica of Block Storage, the current popular distributed Block Storage such as ceph, has the multi-replica capability, extending the storage replica to different racks, cabinets, computer rooms, Data center (or directly use the Block Storage service provided by Public Cloud vendors) -2. TDengine disaster recovery, in TDengine Enterprise, itself has when a dnode permanently offline (TCE-metal disk damage, data sorting loss), re-pull a blank dnode to restore the original dnode work. +1. The disaster recovery capability of the underlying distributed Block Storage, the multi-copy of Block Storage, the current popular distributed Block Storage such as CEPH , has the multi-copy capability, extending the storage copy to different racks, cabinets, computer rooms, Data center (or directly use the Block Storage service provided by Public Cloud vendors) +2. TDengine disaster recovery, in TDengine Enterprise, itself has when a dnode permanently offline (TCE-metal disk damage, data sorting loss), re-pull a blank dnode to restore the original dnode work. Finally, welcome to [TDengine Cloud ](https://cloud.tdengine.com/)to experience the one-stop fully managed TDengine Cloud as a Service. diff --git a/docs/zh/10-deployment/03-k8s.md b/docs/zh/10-deployment/03-k8s.md index 7769b076bc..16e2be0dfd 100644 --- a/docs/zh/10-deployment/03-k8s.md +++ b/docs/zh/10-deployment/03-k8s.md @@ -6,27 +6,27 @@ description: 利用 Kubernetes 部署 TDengine 集群的详细指南 ## 概述 -作为面向云原生架构设计的时序数据库，TDengine 支持 Kubernetes 部署。这里介绍如何使用 YAML 文件一步一步从头创建一个可用于生产使用的高可用TDengine 集群，并重点介绍 Kubernetes 环境下 TDengine 的常用操作。 +作为面向云原生架构设计的时序数据库，TDengine 本身就支持 Kubernetes 部署。这里介绍如何使用 YAML 文件从头一步一步创建一个可用于生产使用的高可用 TDengine 集群，并重点介绍 Kubernetes 环境下 TDengine 的常用操作。 为了满足[高可用](https://docs.taosdata.com/tdinternal/high-availability/)的需求，集群需要满足如下要求： -- 3个及以上dnode：TDengine的vgroup中的vnode，不允许同时分布在一个dnode，所以如果创建3副本的数据库，则dnode数大于等于3 -- 3个mnode：nmode负责整个集群的管理工作，TDengine默认是一个mnode，此时如果mnode所在的dnode掉线，则此时整个集群不可用 -- 数据库3副本，TDengine的副本配置是DB级别，3副本可满足，在3节点的集群中，任意一个dnode下线，都不影响集群的正常使用，**如果下线个数为2时，此时集群不可用，RAFT无法完成选举**，（企业版：在灾难恢复场景，任一节点数据文件损坏，都可以通过重新拉起dnode进行恢复） +- 3个及以上 dnode ：TDengine 的同一个 vgroup 中的多个 vnode ，不允许同时分布在一个 dnode ，所以如果创建3副本的数据库，则 dnode 数大于等于3 +- 3个 mnode ：mnode 负责整个集群的管理工作，TDengine 默认是一个 mnode。如果这个 mnode 所在的 dnode 掉线，则整个集群不可用。 +- 数据库的3副本：TDengine 的副本配置是数据库级别，所以数据库3副本可满足在3个 dnode 的集群中，任意一个 dnode 下线，都不影响集群的正常使用。**如果下线** **dnode** **个数为2时，此时集群不可用，****因为****RAFT无法完成选举****。**（企业版：在灾难恢复场景，任一节点数据文件损坏，都可以通过重新拉起dnode进行恢复） ## 前置条件 要使用 Kubernetes 部署管理 TDengine 集群，需要做好如下准备工作。 -- 本文适用 Kubernetes v1.5 以上版本 -- 本文和下一章使用 minikube、kubectl 和 helm 等工具进行安装部署，请提前安装好相应软件 +- 本文适用 Kubernetes v1.19 以上版本 +- 本文使用 kubectl 工具进行安装部署，请提前安装好相应软件 - Kubernetes 已经安装部署并能正常访问使用或更新必要的容器仓库或其他服务 以下配置文件也可以从 [GitHub 仓库](https://github.com/taosdata/TDengine-Operator/tree/3.0/src/tdengine) 下载。 ## 配置 Service 服务 -创建一个 Service 配置文件：`taosd-service.yaml`，服务名称 `metadata.name` (此处为 "taosd") 将在下一步中使用到。添加 TDengine 所用到的端口： +创建一个 Service 配置文件：`taosd-service.yaml`，服务名称 `metadata.name` (此处为 "taosd") 将在下一步中使用到。首先添加 TDengine 所用到的端口，然后在选择器设置确定的标签 app (此处为 “tdengine”)。 ```YAML --- @@ -50,9 +50,9 @@ spec: ## 有状态服务 StatefulSet -根据 Kubernetes 对各类部署的说明，我们将使用 StatefulSet 作为 TDengine 的服务类型。 创建文件 `tdengine.yaml`，其中 replicas 定义集群节点的数量为 3。节点时区为中国（Asia/Shanghai），每个节点分配 5G 标准（standard）存储（参考[Storage Classes](https://kubernetes.io/docs/concepts/storage/storage-classes/) 配置storage class）。你也可以根据实际情况进行相应修改。 +根据 Kubernetes 对各类部署的说明，我们将使用 StatefulSet 作为 TDengine 的部署资源类型。 创建文件 `tdengine.yaml`，其中 replicas 定义集群节点的数量为 3。节点时区为中国（Asia/Shanghai），每个节点分配 5G 标准（standard）存储（参考[Storage Classes](https://kubernetes.io/docs/concepts/storage/storage-classes/) 配置 storage class ）。你也可以根据实际情况进行相应修改。 -需要注意startupProbe的配置，在dnode 掉线一段时间后，重新启动，新上线的dnode会短暂不可用，如果startupProbe配置过小，Kubernetes会认为pod处于不正常的状态，会尝试重新拉起pod，此时，dnode会频繁重启，始终无法恢复。参考 [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) +请特别注意startupProbe的配置，在 dnode 的 Pod 掉线一段时间后，再重新启动，这个时候新上线的 dnode 会短暂不可用。如果startupProbe配置过小，Kubernetes 会认为该 Pod 处于不正常的状态，并尝试重启该 Pod，该 dnode 的 Pod 会频繁重启，始终无法恢复到正常状态。参考 [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) ```YAML --- @@ -149,7 +149,7 @@ spec: ## 使用 kubectl 命令部署 TDengine 集群 -顺序执行以下命令，需要提前创建对应的namespace。 +首先创建对应的 namespace，然后顺序执行以下命令： ```Bash kubectl apply -f taosd-service.yaml -n tdengine-test @@ -168,7 +168,7 @@ kubectl exec -it tdengine-2 -n tdengine-test -- taos -s "show dnodes" ```Bash taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | ============================================================================================================================================================================================================================================= 1 | tdengine-0.ta... | 0 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-19 17:54:18.469 | | | | 2 | tdengine-1.ta... | 0 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-19 17:54:38.698 | | | | @@ -232,13 +232,13 @@ Query OK, 3 row(s) in set (0.003108s) 利用 kubectl 端口转发功能可以使应用可以访问 Kubernetes 环境运行的 TDengine 集群。 -```bash +```Plain kubectl port-forward -n tdengine-test tdengine-0 6041:6041 & ``` 使用 curl 命令验证 TDengine REST API 使用的 6041 接口。 -```bash +```Plain curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql {"code":0,"column_meta":[["name","VARCHAR",64]],"data":[["information_schema"],["performance_schema"],["test"],["test1"]],"rows":4} ``` @@ -278,7 +278,7 @@ taos> show dnodes Query OK, 3 row(s) in set (0.001357s) ``` -通过show vgroup 查看xnode分布情况 +通过show vgroup 查看 vnode 分布情况 ```Bash kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show test.vgroups" @@ -325,7 +325,7 @@ Query OK, 2 row(s) in set (0.001489s) ### 容错测试 -Mnode leader 所在的dnode掉线，dnode1 +Mnode leader 所在的 dnode 掉线，dnode1 ```Bash kubectl get pod -l app=tdengine -n tdengine-test -o wide @@ -389,7 +389,7 @@ taos> select *from test1.t1 Query OK, 4 row(s) in set (0.001994s) ``` -同理，至于非leader得mnode掉线，读写当然可以正常进行，这里就不做过多的展示， +同理，至于非leader得mnode掉线，读写当然可以正常进行，这里就不做过多的展示。 ## 集群扩容 @@ -415,7 +415,7 @@ tdengine-2 1/1 Running 0 5h16m 10.244.1.224 node85 < tdengine-3 1/1 Running 0 3m24s 10.244.2.76 node86 ``` -此时 POD 的状态仍然是 Running，TDengine 集群中的 dnode 状态要等 POD 状态为 `ready` 之后才能看到： +此时 Pod 的状态仍然是 Running，TDengine 集群中的 dnode 状态要等 Pod 状态为 `ready` 之后才能看到： ```Bash kubectl exec -it tdengine-3 -n tdengine-test -- taos -s "show dnodes" @@ -436,7 +436,7 @@ Query OK, 4 row(s) in set (0.003628s) ## 集群缩容 -由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令（**如果集群中存在3副本的db，那么缩容后的dnode个数也要必须大于等于3，否则drop dnode操作会被中止**），节点删除完成后再进行 Kubernetes 集群缩容。 +由于 TDengine 集群在扩缩容时会对数据进行节点间迁移，使用 kubectl 命令进行缩容需要首先使用 "drop dnodes" 命令（**如果集群中存在3副本的db，那么缩容后的** **dnode** **个数也要必须大于等于3，否则drop dnode操作会被中止**），然后再节点删除完成后再进行 Kubernetes 集群缩容。 注意：由于 Kubernetes Statefulset 中 Pod 的只能按创建顺序逆序移除，所以 TDengine drop dnode 也需要按照创建顺序逆序移除，否则会导致 Pod 处于错误状态。 @@ -491,7 +491,7 @@ tdengine-3 1/1 Running 0 20s 10.244.2.77 node86 < kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" taos> show dnodes - id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | + id | endpoint | vnodes | support_vnodes | status | create_time | reboot_time | note | active_code | c_active_code | ============================================================================================================================================================================================================================================= 1 | tdengine-0.ta... | 10 | 16 | ready | 2023-07-19 17:54:18.552 | 2023-07-20 09:39:04.297 | | | | 2 | tdengine-1.ta... | 10 | 16 | ready | 2023-07-19 17:54:37.828 | 2023-07-20 09:28:24.240 | | | | @@ -534,10 +534,10 @@ Query OK, 4 row(s) in set (0.003862s) ## 最后 -对于在k8s环境下TDengine高可用、高可靠来说，对于硬件损坏、灾难恢复，分为两个层面来讲： +对于在 Kubernetes 环境下 TDengine 的高可用和高可靠来说，对于硬件损坏、灾难恢复，分为两个层面来讲： -1. 底层的分布式块存储具备的灾难恢复能力，块存储的多副本，当下流行的分布式块存储如ceph，就具备多副本能力，将存储副本扩展到不同的机架、机柜、机房、数据中心（或者直接使用公有云厂商提供的块存储服务） -2. TDengine的灾难恢复，在TDengine Enterprise中，本身具备了当一个dnode永久下线（物理机磁盘损坏，数据分拣丢失）后，重新拉起一个空白的dnode来恢复原dnode的工作。 +1. 底层的分布式块存储具备的灾难恢复能力，块存储的多副本，当下流行的分布式块存储如 Ceph，就具备多副本能力，将存储副本扩展到不同的机架、机柜、机房、数据中心（或者直接使用公有云厂商提供的块存储服务） +2. TDengine的灾难恢复，在 TDengine Enterprise 中，本身具备了当一个 dnode 永久下线（物理机磁盘损坏，数据分拣丢失）后，重新拉起一个空白的dnode来恢复原dnode的工作。 最后，欢迎使用[TDengine Cloud](https://cloud.taosdata.com/)，来体验一站式全托管的TDengine云服务。 From 72aceb3d98492b98f1b18a412d36cb3a8169ae0b Mon Sep 17 00:00:00 2001 From: Yaqiang Li Date: Mon, 24 Jul 2023 16:11:11 +0800 Subject: [PATCH 089/100] Update 03-k8s.md --- docs/en/10-deployment/03-k8s.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/10-deployment/03-k8s.md b/docs/en/10-deployment/03-k8s.md index 939dea6f80..b4294fadaf 100644 --- a/docs/en/10-deployment/03-k8s.md +++ b/docs/en/10-deployment/03-k8s.md @@ -6,20 +6,20 @@ description: This document describes how to deploy TDengine on Kubernetes. ## Overview -As a time series database for Cloud Native architecture design, TDengine supports Kubernetes deployment. Here we introduce how to use YAML files to create a highly available TDengine cluster from scratch step by step for production use, and highlight the common operations of TDengine in Kubernetes environment. +As a time series database for Cloud Native architecture design, TDengine supports Kubernetes deployment. Firstly we introduce how to use YAML files to create a highly available TDengine cluster from scratch step by step for production usage, and highlight the common operations of TDengine in Kubernetes environment. To meet [high availability ](https://docs.taosdata.com/tdinternal/high-availability/)requirements, clusters need to meet the following requirements: -- 3 or more dnodes : multiple vnodes in the same vgroup of TDengine are not allowed to be distributed in one dnode at the same time, so if you create a database with 3 copies, the number of dnodes is greater than or equal to 3 -- 3 mnodes : m n ode is responsible for the management of the entire cluster, TDengine defaults to an mnode . If the dnode where the mnode is located is dropped, the entire cluster is unavailable . -- Database 3 replicas : TDengine replica configuration is the database level, so database 3 replicas can meet the three dnode cluster, any one dnode offline, does not affect the normal use of the cluster . **If the number of offline** **dnodes** **is 2, then the cluster is not available,** **because** **RAFT can not complete the election** **.** (Enterprise version: in the disaster recovery scenario, any node data file is damaged, can be restored by pulling up the dnode again) +- 3 or more dnodes: multiple vnodes in the same vgroup of TDengine are not allowed to be distributed in one dnode at the same time, so if you create a database with 3 replicas, the number of dnodes is greater than or equal to 3 +- 3 mnodes: mnode is responsible for the management of the entire TDengine cluster. The default number of mnode in TDengine cluster is only one. If the dnode where the mnode located is dropped, the entire cluster is unavailable. +- Database 3 replicas: The TDengine replica configuration is the database level, so 3 replicas for the database must need three dnodes in the cluster. If any one dnode is offline, does not affect the normal usage of the whole cluster. **If the number of offline** **dnodes** **is 2, then the cluster is not available,** **because** ** the cluster can not complete the election based on RAFT** **.** (Enterprise version: in the disaster recovery scenario, any node data file is damaged, can be restored by pulling up the dnode again) ## Prerequisites Before deploying TDengine on Kubernetes, perform the following: - This article applies Kubernetes 1.19 and above -- This article uses the kubectl tool to install and deploy, please install the corresponding software in advance +- This article uses the **kubectl** tool to install and deploy, please install the corresponding software in advance - Kubernetes have been installed and deployed and can access or update the necessary container repositories or other services You can download the configuration files in this document from [GitHub](https://github.com/taosdata/TDengine-Operator/tree/3.0/src/tdengine). From d0fd6e8b47001027f6f4b7100cae9d04117b1cbb Mon Sep 17 00:00:00 2001 From: Yaqiang Li Date: Mon, 24 Jul 2023 16:23:32 +0800 Subject: [PATCH 090/100] Update 03-k8s.md --- docs/en/10-deployment/03-k8s.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/en/10-deployment/03-k8s.md b/docs/en/10-deployment/03-k8s.md index b4294fadaf..10c0341598 100644 --- a/docs/en/10-deployment/03-k8s.md +++ b/docs/en/10-deployment/03-k8s.md @@ -26,7 +26,7 @@ You can download the configuration files in this document from [GitHub](https:// ## Configure the service -Create a service configuration file named `taosd-service.yaml`. Record the value of `metadata.name` (in this example, `taos`) for use in the next step. Add the ports required by TDengine: +Create a service configuration file named `taosd-service.yaml`. Record the value of `metadata.name` (in this example, `taos`) for use in the next step. And then add the ports required by TDengine and record the value of the selector label "app" (in this example, `tdengine`) for use in the next step: ```YAML --- @@ -50,9 +50,9 @@ spec: ## Configure the service as StatefulSet -According to Kubernetes instructions for various deployments, we will use StatefulSet as the service type of TDengine. Create the file `tdengine.yaml `, where replicas defines the number of cluster nodes as 3. The node time zone is China (Asia/Shanghai), and each node is allocated 5G standard storage (refer to the [Storage Classes ](https://kubernetes.io/docs/concepts/storage/storage-classes/)configuration storage class). You can also modify accordingly according to the actual situation. +According to Kubernetes instructions for various deployments, we will use StatefulSet as the deployment resource type of TDengine. Create the file `tdengine.yaml `, where replicas defines the number of cluster nodes as 3. The node time zone is China (Asia/Shanghai), and each node is allocated 5G standard storage (refer to the [Storage Classes ](https://kubernetes.io/docs/concepts/storage/storage-classes/)configuration storage class). You can also modify accordingly according to the actual situation. -Please pay special attention to the startupProbe configuration, after dnode 's pod drops for a period of time, then restart, this time the newly launched dnode will be temporarily unavailable . If the startupProbe configuration is too small, Kubernetes will think that the Pod is in an abnormal state , and try to restart the Pod, the dnode 's Pod will restart frequently and never return to the normal state . Refer to [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) +Please pay special attention to the startupProbe configuration. If dnode's Pod drops for a period of time and then restart, the newly launched dnode Pod will be temporarily unavailable. The reason is the startupProbe configuration is too small, Kubernetes will know that the Pod is in an abnormal state and try to restart it, then the dnode's Pod will restart frequently and never return to the normal status. Refer to [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) ```YAML --- @@ -156,7 +156,7 @@ kubectl apply -f taosd-service.yaml -n tdengine-test kubectl apply -f tdengine.yaml -n tdengine-test ``` -The above configuration will generate a three-node TDengine cluster, dnode is automatically configured, you can use the show dnodes command to view the nodes of the current cluster: +The above configuration will generate a three-node TDengine cluster, dnode is automatically configured, you can use the **show dnodes** command to view the nodes of the current cluster: ```Bash kubectl exec -it tdengine-0 -n tdengine-test -- taos -s "show dnodes" @@ -236,7 +236,7 @@ Kubectl port forwarding enables applications to access TDengine clusters running kubectl port-forward -n tdengine-test tdengine-0 6041:6041 & ``` -Use curl to verify that the TDengine REST API is working on port 6041: +Use **curl** to verify that the TDengine REST API is working on port 6041: ```bash curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql @@ -249,7 +249,7 @@ curl -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql #### taosBenchmark -Create a 3 replica database with taosBenchmark, write 100 million data at the same time, and view the data at the same time +Create a 3 replicas database with taosBenchmark, write 100 million data at the same time, and view the data at the same time ```Bash kubectl exec -it tdengine-0 -n tdengine-test -- taosBenchmark -I stmt -d test -n 10000 -t 10000 -a 3 @@ -399,7 +399,7 @@ TDengine cluster supports automatic expansion: kubectl scale statefulsets tdengine --replicas=4 ``` -The parameter `--replica = 4 `in the above command line indicates that you want to expand the TDengine cluster to 4 nodes. After execution, first check the status of the POD: +The parameter `--replica = 4 `in the above command line indicates that you want to expand the TDengine cluster to 4 nodes. After execution, first check the status of the Pod: ```Bash kubectl get pod -l app=tdengine -n tdengine-test -o wide @@ -415,7 +415,7 @@ tdengine-2 1/1 Running 0 5h16m 10.244.1.224 node85 < tdengine-3 1/1 Running 0 3m24s 10.244.2.76 node86 ``` -At this time, the state of the POD is still Running, and the dnode state in the TDengine cluster can only be seen after the POD state is `ready `: +At this time, the state of the POD is still Running, and the dnode state in the TDengine cluster can only be seen after the Pod status is `ready `: ```Bash kubectl exec -it tdengine-3 -n tdengine-test -- taos -s "show dnodes" @@ -436,7 +436,7 @@ Query OK, 4 row(s) in set (0.003628s) ## Scaling In Your Cluster -Since the TDengine cluster will migrate data between nodes during volume expansion and contraction, using the kubectl command to reduce the volume requires first using the "drop dnodes" command ( **If there are 3 replicas of db in the cluster, the number of dnodes after reduction must also be greater than or equal to 3, otherwise the drop dnode operation will be aborted** ), the node deletion is completed before Kubernetes cluster reduction. +Since the TDengine cluster will migrate data between nodes during volume expansion and contraction, using the **kubectl** command to reduce the volume requires first using the "drop dnodes" command ( **If there are 3 replicas of db in the cluster, the number of dnodes after reduction must also be greater than or equal to 3, otherwise the drop dnode operation will be aborted** ), the node deletion is completed before Kubernetes cluster reduction. Note: Since Kubernetes Pods in the Statefulset can only be removed in reverse order of creation, the TDengine drop dnode also needs to be removed in reverse order of creation, otherwise the Pod will be in an error state. @@ -453,13 +453,13 @@ taos> show dnodes Query OK, 3 row(s) in set (0.003324s) ``` -After confirming that the removal is successful (use kubectl exec -i -t tdengine-0 --taos -s "show dnodes" to view and confirm the dnode list), use the kubectl command to remove the POD: +After confirming that the removal is successful (use kubectl exec -i -t tdengine-0 --taos -s "show dnodes" to view and confirm the dnode list), use the kubectl command to remove the Pod: ```Plain kubectl scale statefulsets tdengine --replicas=3 -n tdengine-test ``` -The last POD will be deleted. Use the command kubectl get pods -l app = tdengine to check the POD status: +The last Pod will be deleted. Use the command kubectl get pods -l app = tdengine to check the Pod status: ```Plain kubectl get pod -l app=tdengine -n tdengine-test -o wide @@ -469,7 +469,7 @@ tdengine-1 1/1 Running 1 (7h9m ago) 7h23m 10.244.0.59 node84 < tdengine-2 1/1 Running 0 5h45m 10.244.1.224 node85 ``` -After the POD is deleted, the PVC needs to be deleted manually, otherwise the previous data will continue to be used for the next expansion, resulting in the inability to join the cluster normally. +After the Pod is deleted, the PVC needs to be deleted manually, otherwise the previous data will continue to be used for the next expansion, resulting in the inability to join the cluster normally. ```Bash kubectl delete pvc aosdata-tdengine-3 -n tdengine-test @@ -502,7 +502,7 @@ Query OK, 4 row(s) in set (0.003881s) ## Remove a TDengine Cluster -> **When deleting the pvc, you need to pay attention to the pv persistentVolumeReclaimPolicy policy. It is recommended to change to Delete, so that the pv will be automatically cleaned up when the pvc is deleted, and the underlying csi storage resources will be cleaned up at the same time. If the policy of deleting the pvc to automatically clean up the pv is not configured, and then after deleting the pvc, when manually cleaning up the pv, the csi storage resources corresponding to the pv may not be released.** +> **When deleting the PVC, you need to pay attention to the pv persistentVolumeReclaimPolicy policy. It is recommended to change to Delete, so that the PV will be automatically cleaned up when the PVC is deleted, and the underlying CSI storage resources will be cleaned up at the same time. If the policy of deleting the PVC to automatically clean up the PV is not configured, and then after deleting the pvc, when manually cleaning up the PV, the CSI storage resources corresponding to the PV may not be released.** Complete removal of TDengine cluster, need to clean up statefulset, svc, configmap, pvc respectively. @@ -534,10 +534,10 @@ Query OK, 4 row(s) in set (0.003862s) ## Finally -For the high availability and high reliability of TDengine in a Kubernetes environment, hardware damage and disaster recovery are divided into two levels: +For the high availability and high reliability of TDengine in a Kubernetes environment, hardware damage and disaster recovery are divided into two levels: -1. The disaster recovery capability of the underlying distributed Block Storage, the multi-copy of Block Storage, the current popular distributed Block Storage such as CEPH , has the multi-copy capability, extending the storage copy to different racks, cabinets, computer rooms, Data center (or directly use the Block Storage service provided by Public Cloud vendors) -2. TDengine disaster recovery, in TDengine Enterprise, itself has when a dnode permanently offline (TCE-metal disk damage, data sorting loss), re-pull a blank dnode to restore the original dnode work. +1. The disaster recovery capability of the underlying distributed Block Storage, the multi-copy of Block Storage, the current popular distributed Block Storage such as Ceph, has the multi-copy capability, extending the storage copy to different racks, cabinets, computer rooms, Data center (or directly use the Block Storage service provided by Public Cloud vendors) +2. TDengine disaster recovery, in TDengine Enterprise, itself has when a dnode permanently offline (TCE-metal disk damage, data sorting loss), re-pull a blank dnode to restore the original dnode work. Finally, welcome to [TDengine Cloud ](https://cloud.tdengine.com/)to experience the one-stop fully managed TDengine Cloud as a Service. From 9e3421e63e08c97f4f0440e9e9e5f50a4a7385ed Mon Sep 17 00:00:00 2001 From: sunpeng Date: Mon, 24 Jul 2023 17:03:57 +0800 Subject: [PATCH 091/100] docs: fix python connector docs --- docs/en/14-reference/03-connector/07-python.mdx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index f0a59842fe..831e79eeb7 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -1007,13 +1007,12 @@ consumer.close() ### Other sample programs | Example program links | Example program content | -| ------------------------------------------------------------------------------------------------------------- | ------------------- ---- | -| [bind_multi.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-multi.py) | parameter binding, -bind multiple rows at once | -| [bind_row.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-row.py) | bind_row.py +|-----------------------|-------------------------| +| [bind_multi.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-multi.py) | parameter binding, bind multiple rows at once | +| [bind_row.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-row.py) | parameter binding, bind one row at once | | [insert_lines.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/insert-lines.py) | InfluxDB line protocol writing | | [json_tag.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/json-tag.py) | Use JSON type tags | -| [tmq.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq.py) | TMQ subscription | +| [tmq_consumer.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq_consumer.py) | TMQ subscription | ## Other notes From 101a3c23b1ab1df9d7023e1d77c3963d7dfd3f48 Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Mon, 24 Jul 2023 17:50:28 +0800 Subject: [PATCH 092/100] docs: fix links in rust connector doc (#22168) --- docs/en/14-reference/03-connector/06-rust.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index 986b5cd104..56f5e20cb4 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -648,12 +648,12 @@ stmt.execute()?; //stmt.execute()?; ``` -For a working example, see [GitHub](https://github.com/taosdata/taos-connector-rust/blob/main/examples/bind.rs). +For a working example, see [GitHub](https://github.com/taosdata/taos-connector-rust/blob/main/taos/examples/bind.rs). For information about other structure APIs, see the [Rust documentation](https://docs.rs/taos). -[taos]: https://github.com/taosdata/rust-connector-taos +[taos]: https://github.com/taosdata/taos-connector-rust [r2d2]: https://crates.io/crates/r2d2 [TaosBuilder]: https://docs.rs/taos/latest/taos/struct.TaosBuilder.html [TaosCfg]: https://docs.rs/taos/latest/taos/struct.TaosCfg.html From 0b707a108e1f989290caeb558f4fa3867cb0c452 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 24 Jul 2023 10:28:52 +0000 Subject: [PATCH 093/100] fix err while connect invalid fqdn --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index f5dc627dd8..6b89739547 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -106,7 +106,7 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas SCorEpSet epSet = {0}; if (ip) { - if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { + if (initEpSetFromCfg(ip, tsSecond, &epSet) < 0) { return NULL; } } else { From 842a0bc8c3d87f85a1ad03449d5aa95a6751d0f0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 24 Jul 2023 18:48:09 +0800 Subject: [PATCH 094/100] fix err while connect invalid fqdn --- source/client/src/clientImpl.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index f5dc627dd8..d448dd1edf 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1304,9 +1304,8 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe } uint32_t addr = taosGetIpv4FromFqdn(mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn); if (addr == 0xffffffff) { - int32_t code = TAOS_SYSTEM_ERROR(errno); tscError("failed to resolve firstEp fqdn: %s, code:%s", mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn, - tstrerror(errno)); + tstrerror(TSDB_CODE_TSC_INVALID_FQDN)); memset(&(mgmtEpSet->eps[mgmtEpSet->numOfEps]), 0, sizeof(mgmtEpSet->eps[mgmtEpSet->numOfEps])); } else { mgmtEpSet->numOfEps++; @@ -1322,9 +1321,8 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); uint32_t addr = taosGetIpv4FromFqdn(mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn); if (addr == 0xffffffff) { - int32_t code = TAOS_SYSTEM_ERROR(errno); tscError("failed to resolve secondEp fqdn: %s, code:%s", mgmtEpSet->eps[mgmtEpSet->numOfEps].fqdn, - tstrerror(errno)); + tstrerror(TSDB_CODE_TSC_INVALID_FQDN)); memset(&(mgmtEpSet->eps[mgmtEpSet->numOfEps]), 0, sizeof(mgmtEpSet->eps[mgmtEpSet->numOfEps])); } else { mgmtEpSet->numOfEps++; @@ -1332,8 +1330,8 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe } if (mgmtEpSet->numOfEps == 0) { - terrno = TSDB_CODE_TSC_INVALID_FQDN; - return -1; + terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + return TSDB_CODE_RPC_NETWORK_UNAVAIL; } return 0; From 6532afa7b84febf668080db1b458691aa49c942d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 24 Jul 2023 18:49:13 +0800 Subject: [PATCH 095/100] fix err while connect invalid fqdn --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index fdce502078..d448dd1edf 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -106,7 +106,7 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas SCorEpSet epSet = {0}; if (ip) { - if (initEpSetFromCfg(ip, tsSecond, &epSet) < 0) { + if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { return NULL; } } else { From 6a0e9941693b0e17b2f2153cf2145703c41e2b29 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Jul 2023 19:24:05 +0800 Subject: [PATCH 096/100] fix:avoid request offset type is 0 --- source/client/src/clientTmq.c | 4 ++-- source/common/src/tmsg.c | 5 ----- source/dnode/vnode/src/tq/tqUtil.c | 4 +++- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 3576df434b..8fb6911f81 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1860,8 +1860,8 @@ static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* p static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal* rspOffset, int64_t sver, int64_t ever, int64_t consumerId){ if (!pVg->seekUpdated) { tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", consumerId); - pVg->offsetInfo.beginOffset = *reqOffset; - pVg->offsetInfo.endOffset = *rspOffset; + if(reqOffset->type != 0) pVg->offsetInfo.beginOffset = *reqOffset; + if(rspOffset->type != 0) pVg->offsetInfo.endOffset = *rspOffset; } else { tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", consumerId); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index fc99202bce..8051f4d0bd 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -7207,11 +7207,6 @@ bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) { return pLeft->uid == pRight->uid && pLeft->ts == pRight->ts; } else if (pLeft->type == TMQ_OFFSET__SNAPSHOT_META) { return pLeft->uid == pRight->uid; - } else { - ASSERT(0); - /*ASSERT(pLeft->type == TMQ_OFFSET__RESET_NONE || pLeft->type == TMQ_OFFSET__RESET_EARLIEST ||*/ - /*pLeft->type == TMQ_OFFSET__RESET_LATEST);*/ - /*return true;*/ } } return false; diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 7d632f44bc..4320995306 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -344,9 +344,11 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ if (blockReturned) { return 0; } - } else { // use the consumer specified offset + } else if(reqOffset.type != 0){ // use the consumer specified offset // the offset value can not be monotonious increase?? offset = reqOffset; + } else { + return TSDB_CODE_TMQ_INVALID_MSG; } // this is a normal subscribe requirement From 29460db6f5cfdb640e8447a1f263d970277d0d29 Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Tue, 25 Jul 2023 00:11:09 +0800 Subject: [PATCH 097/100] docs: change to new product names (#22175) * change to new product names * change product names --- docs/en/05-get-started/03-package.md | 2 +- docs/en/12-taos-sql/02-database.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index 91bf94034c..3e3c04682f 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -18,7 +18,7 @@ The full package of TDengine includes the TDengine Server (`taosd`), TDengine Cl The standard server installation package includes `taos`, `taosd`, `taosAdapter`, `taosBenchmark`, and sample code. You can also download the Lite package that includes only `taosd` and the C/C++ connector. -The TDengine Community Edition is released as Deb and RPM packages. The Deb package can be installed on Debian, Ubuntu, and derivative systems. The RPM package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the Deb or RPM package, download and install taosTools separately. TDengine can also be installed on x64 Windows and x64/m1 macOS. +TDengine OSS is released as Deb and RPM packages. The Deb package can be installed on Debian, Ubuntu, and derivative systems. The RPM package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the Deb or RPM package, download and install taosTools separately. TDengine can also be installed on x64 Windows and x64/m1 macOS. ## Operating environment requirements In the Linux system, the minimum requirements for the operating environment are as follows: diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index 68dba3fc56..e783d61497 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -58,7 +58,7 @@ database_option: { - WAL_FSYNC_PERIOD: specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter takes effect only when the WAL parameter is set to 2. The default value is 3000. Enter a value between 0 and 180000. The value 0 indicates that incoming data is immediately written to disk. - MAXROWS: specifies the maximum number of rows recorded in a block. The default value is 4096. - MINROWS: specifies the minimum number of rows recorded in a block. The default value is 100. -- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The Enterprise Edition supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; the Community Edition does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). +- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. TDengine Enterprise supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; TDengine OSS does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). - PAGES: specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. The default value is 256. The space occupied by metadata storage on each vnode is equal to the product of the values of the PAGESIZE and PAGES parameters. The space occupied by default is 1 MB. - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384. - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. From 4e43b99c94629900fe8e882f3fcd398b98f345a2 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Tue, 25 Jul 2023 11:42:34 +0800 Subject: [PATCH 098/100] docs: delete logs in monitor doc --- docs/en/13-operation/10-monitor.md | 13 ------------- docs/zh/17-operation/10-monitor.md | 13 ------------- 2 files changed, 26 deletions(-) diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index 197dda20ee..c1c6ac3c4c 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -214,19 +214,6 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### logs table - -`logs` table contains login information records. - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|level|VARCHAR||log level| -|content|NCHAR||log content| -|dnode\_id|INT|TAG|dnode id| -|dnode\_ep|NCHAR|TAG|dnode endpoint| -|cluster\_id|NCHAR|TAG|cluster id| - ### log\_summary table `log_summary` table contains log summary information records. diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md index 50da505808..4f8dccc78d 100644 --- a/docs/zh/17-operation/10-monitor.md +++ b/docs/zh/17-operation/10-monitor.md @@ -210,19 +210,6 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### logs 表 - -`logs` 表记录登录信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|level|VARCHAR||log level| -|content|NCHAR||log content，长度不超过1024字节| -|dnode\_id|INT|TAG|dnode id| -|dnode\_ep|NCHAR|TAG|dnode endpoint| -|cluster\_id|NCHAR|TAG|cluster id| - ### log\_summary 表 `log_summary` 记录日志统计信息。 From 30980308c27746932a8d4f8c8c4f204d532f99f1 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 26 Jul 2023 10:57:31 +0800 Subject: [PATCH 099/100] docs: refine zh/21-tdinternal/01-arch.md --- docs/zh/21-tdinternal/01-arch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/21-tdinternal/01-arch.md b/docs/zh/21-tdinternal/01-arch.md index 32d940abc1..e2480b6682 100644 --- a/docs/zh/21-tdinternal/01-arch.md +++ b/docs/zh/21-tdinternal/01-arch.md @@ -112,7 +112,7 @@ TDengine 3.0 采用 hash 一致性算法，确定每张数据表所在的 vnode ### 数据分区 -TDengine 除 vnode 分片之外，还对时序数据按照时间段进行分区。每个数据文件只包含一个时间段的时序数据，时间段的长度由 DB 的配置参数 days 决定。这种按时间段分区的方法还便于高效实现数据的保留策略，只要数据文件超过规定的天数（系统配置参数 keep），将被自动删除。而且不同的时间段可以存放于不同的路径和存储介质，以便于大数据的冷热管理，实现多级存储。 +TDengine 除 vnode 分片之外，还对时序数据按照时间段进行分区。每个数据文件只包含一个时间段的时序数据，时间段的长度由 DB 的配置参数 duration 决定。这种按时间段分区的方法还便于高效实现数据的保留策略，只要数据文件超过规定的天数（系统配置参数 keep），将被自动删除。而且不同的时间段可以存放于不同的路径和存储介质，以便于大数据的冷热管理，实现多级存储。 总的来说，**TDengine 是通过 vnode 以及时间两个维度，对大数据进行切分**，便于并行高效的管理，实现水平扩展。 From 238279157aae00190c4b210051f8963405ea4c22 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 27 Jul 2023 12:09:49 +0800 Subject: [PATCH 100/100] docs: refine python version udf requires --- docs/en/07-develop/09-udf.md | 2 +- docs/zh/07-develop/09-udf.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index 5137e35c0a..7526aba43b 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -403,7 +403,7 @@ In this section we will demonstrate 5 examples of developing UDF in Python langu In the guide, some debugging skills of using Python UDF will be explained too. -We assume you are using Linux system and already have TDengine 3.0.4.0+ and Python 3.x. +We assume you are using Linux system and already have TDengine 3.0.4.0+ and Python 3.7+. Note:**You can't use print() function to output log inside a UDF, you have to write the log to a specific file or use logging module of Python.** diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index ff46437687..bb6a575ccd 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -398,7 +398,7 @@ def finish(buf: bytes) -> output_type: 3. 定义一个标量函数，输入一个时间戳，输出距离这个时间最近的下一个周日。完成这个函数要用到第三方库 moment。我们在这个示例中讲解使用第三方库的注意事项。 4. 定义一个聚合函数，计算某一列最大值和最小值的差, 也就是实现 TDengien 内置的 spread 函数。 同时也包含大量实用的 debug 技巧。 -本文假设你用的是 Linux 系统，且已安装好了 TDengine 3.0.4.0+ 和 Python 3.x。 +本文假设你用的是 Linux 系统，且已安装好了 TDengine 3.0.4.0+ 和 Python 3.7+。 注意：**UDF 内无法通过 print 函数输出日志，需要自己写文件或用 python 内置的 logging 库写文件**。