From 63b40fb29c8b0566e5223abffcb75db0f9c064cd Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Tue, 14 Jun 2022 14:03:51 +0800 Subject: [PATCH 01/41] enh: backup source code if case fails --- tests/parallel_test/collect_cases.sh | 11 +++++++++++ tests/parallel_test/run.sh | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/parallel_test/collect_cases.sh b/tests/parallel_test/collect_cases.sh index c560598c81..8ae83f5618 100755 --- a/tests/parallel_test/collect_cases.sh +++ b/tests/parallel_test/collect_cases.sh @@ -41,5 +41,16 @@ fi cat ../script/jenkins/basic.txt |grep -v "^#"|grep -v "^$"|sed "s/^/,,script,/" >>$case_file grep "^python" ../system-test/fulltest.sh |sed "s/^/,,system-test,/" >>$case_file +# tar source code for run.sh to use +if [ $ent -eq 0 ]; then + cd ../../../ + rm -rf TDengine.tar.gz + tar czf TDengine.tar.gz TDengine taos-connector-python --exclude=TDengine/debug +else + cd ../../../../ + rm -rf TDinternal.tar.gz + tar czf TDinternal.tar.gz TDinternal taos-connector-python --exclude=TDinternal/debug --exclude=TDinternal/community/debug +fi + exit 0 diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index 6417f41fd4..3b0c0da8cd 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -291,6 +291,19 @@ function run_thread() { fi cmd="$scpcmd:${remote_sim_tar} $log_dir/${case_file}.sim.tar.gz" $cmd + # backup source code + source_tar_dir=$log_dir/TDengine_${hosts[index]} + source_tar_file=TDengine.tar.gz + if [ $ent -ne 0 ]; then + source_tar_dir=$log_dir/TDinternal_${hosts[index]} + source_tar_file=TDinternal.tar.gz + fi + mkdir $source_tar_dir 2>/dev/null + if [ $? -eq 0 ]; then + cmd="$scpcmd:${workdirs[index]}/$source_tar_file $source_tar_dir" + echo "$cmd" + $cmd + fi fi done } From c77e7d521afba79c625ffa0bb25eef51ad8d17a5 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Tue, 14 Jun 2022 15:01:45 +0800 Subject: [PATCH 02/41] enh: exclude some more directories when tar source files --- tests/parallel_test/collect_cases.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parallel_test/collect_cases.sh b/tests/parallel_test/collect_cases.sh index 8ae83f5618..2c8c61b0b7 100755 --- a/tests/parallel_test/collect_cases.sh +++ b/tests/parallel_test/collect_cases.sh @@ -45,11 +45,11 @@ grep "^python" ../system-test/fulltest.sh |sed "s/^/,,system-test,/" >>$case_fil if [ $ent -eq 0 ]; then cd ../../../ rm -rf TDengine.tar.gz - tar czf TDengine.tar.gz TDengine taos-connector-python --exclude=TDengine/debug + tar --exclude=TDengine/debug --exclude=TDengine/sim --exclude=TDengine/release -czf TDengine.tar.gz TDengine taos-connector-python else cd ../../../../ rm -rf TDinternal.tar.gz - tar czf TDinternal.tar.gz TDinternal taos-connector-python --exclude=TDinternal/debug --exclude=TDinternal/community/debug + tar --exclude=TDinternal/debug --exclude=TDinternal/sim --exclude=TDinternal/community/debug --exclude=TDinternal/community/release -czf TDinternal.tar.gz TDinternal taos-connector-python fi exit 0 From 3c8fcc7c794723509cdf15fb1230ace631810baf Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Tue, 14 Jun 2022 16:00:05 +0800 Subject: [PATCH 03/41] enh: exclude some more directories --- tests/parallel_test/collect_cases.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/collect_cases.sh b/tests/parallel_test/collect_cases.sh index 2c8c61b0b7..c12413b981 100755 --- a/tests/parallel_test/collect_cases.sh +++ b/tests/parallel_test/collect_cases.sh @@ -49,7 +49,7 @@ if [ $ent -eq 0 ]; then else cd ../../../../ rm -rf TDinternal.tar.gz - tar --exclude=TDinternal/debug --exclude=TDinternal/sim --exclude=TDinternal/community/debug --exclude=TDinternal/community/release -czf TDinternal.tar.gz TDinternal taos-connector-python + tar --exclude=TDinternal/debug --exclude=TDinternal/sim --exclude=TDinternal/community/debug --exclude=TDinternal/community/release --exclude=TDinternal/community/sim -czf TDinternal.tar.gz TDinternal taos-connector-python fi exit 0 From 3acacf163ac29b648d13470ef6cba749013bea7b Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Tue, 14 Jun 2022 16:32:17 +0800 Subject: [PATCH 04/41] enh: show commit id of target branch --- Jenkinsfile2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 950b316bab..a29169eece 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -82,6 +82,7 @@ def pre_test(){ sh ''' cd ${WKC} git pull >/dev/null + git log -5 git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD git log -5 @@ -93,6 +94,7 @@ def pre_test(){ sh ''' cd ${WK} git pull >/dev/null + git log -5 git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD git log -5 From aa5efe5cbe43faf47b465d74410132062b45dce0 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Tue, 14 Jun 2022 18:18:59 +0800 Subject: [PATCH 05/41] enh: show commit id of python connector --- Jenkinsfile2 | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index a29169eece..79a7bab0c3 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -115,6 +115,7 @@ def pre_test(){ cd ${WKPY} git reset --hard git pull + git log -5 ''' return 1 } From 43afdc5ad0564693671c87910841baa8024fe4cd Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 14 Jun 2022 19:31:45 +0800 Subject: [PATCH 06/41] os: add taosd assert kill --- tests/system-test/7-tmq/basic5.py | 4 +++- tests/system-test/test-all.bat | 10 +++++----- tests/system-test/test.py | 15 +++++++++++---- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 3d9efea938..a10eaf1fb5 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -134,7 +134,7 @@ class TDTestCase: parameterDict['cfg'] = cfgPath prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() - time.sleep(2) + prepareEnvThread.join() # wait stb ready while 1: @@ -245,6 +245,7 @@ class TDTestCase: prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() + prepareEnvThread.join() # wait db ready while 1: @@ -371,6 +372,7 @@ class TDTestCase: prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() + prepareEnvThread.join() # wait db ready while 1: diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index 076be6563a..a5a114f6eb 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -61,19 +61,19 @@ goto :eof set tt=%1 set tt=%tt:.= % set tt=%tt::= % -set index=1 +set /a index=1 for %%a in (%tt%) do ( if !index! EQU 1 ( - set /a hh=%%a + set /a hh=%%a || echo 11 %%a )^ else if !index! EQU 2 ( - set /a mm=%%a + set /a mm=%%a || echo 22 %%a )^ else if !index! EQU 3 ( - set /a ss=%%a + set /a ss=%%a || echo 33 %%a ) set /a index=index+1 ) -set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% || echo hh:%hh% mm:%mm% ss:%ss% +set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% || echo 44 hh:%hh% mm:%mm% ss:%ss% goto :eof \ No newline at end of file diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 47e0cefb52..aaa61d8127 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -21,6 +21,7 @@ import base64 import json import platform import socket +import threading from distutils.log import warn as printf from fabric2 import Connection sys.path.append("../pytest") @@ -30,6 +31,13 @@ from util.cases import * import taos +def checkRunTimeError(): + import win32gui + while 1: + time.sleep(1) + hwnd = win32gui.FindWindow(None, "Microsoft Visual C++ Runtime Library") + if hwnd: + os.system("TASKKILL /F /IM taosd.exe") if __name__ == "__main__": @@ -42,9 +50,6 @@ if __name__ == "__main__": logSql = True stop = 0 restart = False - windows = 0 - if platform.system().lower() == 'windows': - windows = 1 updateCfgDict = {} execCmd = "" opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:', [ @@ -159,7 +164,9 @@ if __name__ == "__main__": host = masterIp tdLog.info("Procedures for tdengine deployed in %s" % (host)) - if windows: + if platform.system().lower() == 'windows': + if (masterIp == ""): + threading.Thread(target=checkRunTimeError,daemon=True).start() tdCases.logSql(logSql) tdLog.info("Procedures for testing self-deployment") tdDnodes.init(deployPath, masterIp) From 26114896065b26b0e243d1ed5c9b9b81c5b13c72 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 14 Jun 2022 19:35:28 +0800 Subject: [PATCH 07/41] test:add test case for tmq --- tests/system-test/7-tmq/tmqError.py | 315 ++++++++++++++++++++++++++++ tests/system-test/fulltest.sh | 1 + 2 files changed, 316 insertions(+) create mode 100644 tests/system-test/7-tmq/tmqError.py diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py new file mode 100644 index 0000000000..907d69bb7b --- /dev/null +++ b/tests/system-test/7-tmq/tmqError.py @@ -0,0 +1,315 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class actionType(Enum): + CREATE_DATABASE = 0 + CREATE_STABLE = 1 + CREATE_CTABLE = 2 + INSERT_DATA = 3 + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + # tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + # tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def initConsumerInfoTable(self,cdbName='cdb'): + tdLog.info("drop consumeinfo table") + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, dbName,stbName): + tsql.execute("create table if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(dbName, stbName)) + tdLog.debug("complete to create %s.%s" %(dbName, stbName)) + return + + def create_ctables(self,tsql, dbName,stbName,ctbNum): + tsql.execute("use %s" %dbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfSql = 0 + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + rowsOfSql += 1 + if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsOfSql = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + + if parameterDict["actionType"] == actionType.CREATE_DATABASE: + self.create_database(tsql, parameterDict["dbName"]) + elif parameterDict["actionType"] == actionType.CREATE_STABLE: + self.create_stable(tsql, parameterDict["dbName"], parameterDict["stbName"]) + elif parameterDict["actionType"] == actionType.CREATE_CTABLE: + self.create_ctables(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + elif parameterDict["actionType"] == actionType.INSERT_DATA: + self.insert_data(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],parameterDict["batchNum"]) + else: + tdLog.exit("not support's action: ", parameterDict["actionType"]) + + return + + def tmqCase1(self, cfgPath, buildPath): + ''' + Leave a TMQ process. Stop taosd, delete the data directory, restart taosd, + and restart a consumption process to complete a consumption + ''' + tdLog.printNoPrefix("======== test case 1: ") + + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db3', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 20000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,parameterDict["dbName"],parameterDict["stbName"],parameterDict["ctbNum"],parameterDict["rowsPerTbl"],parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + # expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + expectrowcnt = 90000000000 + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 9000000 # Forever loop + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + time.sleep(3) + tdLog.info("================= stop dnode, and remove data file, then start dnode ===========================") + tdDnodes.stop(1) + # time.sleep(5) + dataPath = buildPath + "/../sim/dnode1/data/*" + shellCmd = 'rm -rf ' + dataPath + tdLog.info(shellCmd) + os.system(shellCmd) + tdDnodes.start(1) + time.sleep(2) + + ######### redo to consume + self.initConsumerTable() + + self.create_database(tdSql, parameterDict["dbName"]) + self.create_stable(tdSql, parameterDict["dbName"], parameterDict["stbName"]) + self.create_ctables(tdSql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + self.insert_data(tdSql,parameterDict["dbName"],parameterDict["stbName"],parameterDict["ctbNum"],parameterDict["rowsPerTbl"],parameterDict["batchNum"]) + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + + tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb1, parameterDict['dbName'], parameterDict['stbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 20 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + if not (totalConsumeRows == expectrowcnt): + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb1) + os.system('pkill tmq_sim') + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + self.tmqCase1(cfgPath, buildPath) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index e04db9dae9..38c3741cda 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -112,3 +112,4 @@ python3 ./test.py -f 7-tmq/subscribeStb2.py python3 ./test.py -f 7-tmq/subscribeStb3.py python3 ./test.py -f 7-tmq/subscribeStb4.py python3 ./test.py -f 7-tmq/db.py +python3 ./test.py -f 7-tmq/tmqError.py From c7f276462547783b1ef9843aaf7c38be15657628 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 14 Jun 2022 20:31:42 +0800 Subject: [PATCH 08/41] os: add taosd assert kill --- tests/system-test/test-all.bat | 9 +++++---- tests/system-test/test.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index a5a114f6eb..0929b1fc6e 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -61,19 +61,20 @@ goto :eof set tt=%1 set tt=%tt:.= % set tt=%tt::= % +set tt=%tt: 0= % set /a index=1 for %%a in (%tt%) do ( if !index! EQU 1 ( - set /a hh=%%a || echo 11 %%a + set /a hh=%%a )^ else if !index! EQU 2 ( - set /a mm=%%a || echo 22 %%a + set /a mm=%%a )^ else if !index! EQU 3 ( - set /a ss=%%a || echo 33 %%a + set /a ss=%%a ) set /a index=index+1 ) -set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% || echo 44 hh:%hh% mm:%mm% ss:%ss% +set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% goto :eof \ No newline at end of file diff --git a/tests/system-test/test.py b/tests/system-test/test.py index aaa61d8127..0022a51329 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -165,7 +165,7 @@ if __name__ == "__main__": tdLog.info("Procedures for tdengine deployed in %s" % (host)) if platform.system().lower() == 'windows': - if (masterIp == ""): + if (masterIp == "" and not fileName[0:3] == "udf"): threading.Thread(target=checkRunTimeError,daemon=True).start() tdCases.logSql(logSql) tdLog.info("Procedures for testing self-deployment") From 435aca2eeb0eb5a9c9a4b1ff2265d32226a8e281 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 15 Jun 2022 09:04:06 +0800 Subject: [PATCH 09/41] os: add taosd assert kill --- tests/system-test/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 0022a51329..5a68f548da 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -165,7 +165,7 @@ if __name__ == "__main__": tdLog.info("Procedures for tdengine deployed in %s" % (host)) if platform.system().lower() == 'windows': - if (masterIp == "" and not fileName[0:3] == "udf"): + if (masterIp == "" and not fileName[0:12] == "0-others\\udf"): threading.Thread(target=checkRunTimeError,daemon=True).start() tdCases.logSql(logSql) tdLog.info("Procedures for testing self-deployment") From c9caaeea38e248b6a3a15cfd5986c255e059b718 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 15 Jun 2022 10:18:05 +0800 Subject: [PATCH 10/41] fix tb index issue --- source/libs/catalog/src/ctgAsync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 159c120dde..77de4409ee 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -870,7 +870,7 @@ _return: int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; SArray* pInfo = NULL; From afe3171edc21aee32c59fab156a2538e0b930052 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 15 Jun 2022 10:40:39 +0800 Subject: [PATCH 11/41] fix free issue --- source/libs/catalog/src/ctgUtil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 1cb2b99a67..e97c34dc26 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -675,7 +675,8 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { for (int32_t i = 0; i < num; ++i) { STableIndexInfo *pInfo = taosArrayGet(pIndex, i); - taosArrayPush(*pRes, pInfo); + pInfo = taosArrayPush(*pRes, pInfo); + pInfo->expr = strdup(pInfo->expr); } return TSDB_CODE_SUCCESS; From b94772e838e357374e6a3e6adbac053d6cb8e820 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Jun 2022 10:56:06 +0800 Subject: [PATCH 12/41] fix: stable column reset after create sma --- source/dnode/mnode/impl/src/mndStb.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3e91bfa926..fcd86a10be 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -323,10 +323,14 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { pOld->smaVer = pNew->smaVer; pOld->nextColId = pNew->nextColId; pOld->ttl = pNew->ttl; - pOld->numOfColumns = pNew->numOfColumns; - pOld->numOfTags = pNew->numOfTags; - memcpy(pOld->pColumns, pNew->pColumns, pOld->numOfColumns * sizeof(SSchema)); - memcpy(pOld->pTags, pNew->pTags, pOld->numOfTags * sizeof(SSchema)); + if (pNew->numOfColumns > 0) { + pOld->numOfColumns = pNew->numOfColumns; + memcpy(pOld->pColumns, pNew->pColumns, pOld->numOfColumns * sizeof(SSchema)); + } + if (pNew->numOfTags > 0) { + pOld->numOfTags = pNew->numOfTags; + memcpy(pOld->pTags, pNew->pTags, pOld->numOfTags * sizeof(SSchema)); + } if (pNew->commentLen != 0) { memcpy(pOld->comment, pNew->comment, pNew->commentLen); } From 6c7072286d4f82dd3e7353af02992a9a9df838f6 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 15 Jun 2022 11:06:06 +0800 Subject: [PATCH 13/41] refactor(sync): if eqmsg error, return --- source/libs/sync/src/syncMain.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 9db9550681..920a0245fd 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1743,16 +1743,26 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); syncRpcMsgLog2((char*)"==syncNodeEqElectTimer==", &rpcMsg); if (pSyncNode->FpEqMsg != NULL) { - pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + if (code != 0) { + sError("vgId:%d sync enqueue elect msg error, code:%d", pSyncNode->vgId, code); + rpcFreeCont(rpcMsg.pCont); + syncTimeoutDestroy(pSyncMsg); + return; + } } else { - sTrace("syncNodeEqElectTimer pSyncNode->FpEqMsg is NULL"); + sTrace("syncNodeEqElectTimer FpEqMsg is NULL"); } syncTimeoutDestroy(pSyncMsg); // reset timer ms - pSyncNode->electTimerMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); - taosTmrReset(syncNodeEqElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pElectTimer); + if (gSyncEnv != NULL) { + pSyncNode->electTimerMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); + taosTmrReset(syncNodeEqElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pElectTimer); + } else { + sError("sync env elect is already stop"); + } } else { sTrace("==syncNodeEqElectTimer== electTimerLogicClock:%" PRIu64 ", electTimerLogicClockUser:%" PRIu64 "", pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); @@ -1774,19 +1784,19 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { if (code != 0) { sError("vgId:%d sync enqueue timer msg error, code:%d", pSyncNode->vgId, code); rpcFreeCont(rpcMsg.pCont); + syncTimeoutDestroy(pSyncMsg); return; } - } else { - sTrace("syncNodeEqHeartbeatTimer pSyncNode->FpEqMsg is NULL"); + sError("syncNodeEqHeartbeatTimer FpEqMsg is NULL"); } syncTimeoutDestroy(pSyncMsg); if (gSyncEnv != NULL) { - taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pHeartbeatTimer); + taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pHeartbeatTimer); } else { - sError("sync env is already stop"); + sError("sync env heartbeat is already stop"); } } else { sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 From c9f6c2041091204289163212a170487d12ac41fa Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 15 Jun 2022 11:42:04 +0800 Subject: [PATCH 14/41] refactor(sync): if eqmsg error, return --- source/libs/sync/src/syncMain.c | 57 +++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 920a0245fd..7b1fd010ab 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -945,9 +945,13 @@ int32_t syncNodePingAll(SSyncNode* pSyncNode) { // timer control -------------- int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { int32_t ret = 0; - taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pPingTimer); - atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); + if (syncEnvIsStart()) { + taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pPingTimer); + atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); + } else { + sError("sync env is stop, syncNodeStartPingTimer"); + } return ret; } @@ -961,10 +965,14 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms) { int32_t ret = 0; - pSyncNode->electTimerMS = ms; - taosTmrReset(pSyncNode->FpElectTimerCB, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pElectTimer); - atomic_store_64(&pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); + if (syncEnvIsStart()) { + pSyncNode->electTimerMS = ms; + taosTmrReset(pSyncNode->FpElectTimerCB, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pElectTimer); + atomic_store_64(&pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); + } else { + sError("sync env is stop, syncNodeStartElectTimer"); + } return ret; } @@ -998,9 +1006,13 @@ int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) { int32_t ret = 0; - taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pHeartbeatTimer); - atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); + if (syncEnvIsStart()) { + taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pHeartbeatTimer); + atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); + } else { + sError("sync env is stop, syncNodeStartHeartbeatTimer"); + } return ret; } @@ -1720,14 +1732,25 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); syncRpcMsgLog2((char*)"==syncNodeEqPingTimer==", &rpcMsg); if (pSyncNode->FpEqMsg != NULL) { - pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + if (code != 0) { + sError("vgId:%d sync enqueue ping msg error, code:%d", pSyncNode->vgId, code); + rpcFreeCont(rpcMsg.pCont); + syncTimeoutDestroy(pSyncMsg); + return; + } } else { sTrace("syncNodeEqPingTimer pSyncNode->FpEqMsg is NULL"); } syncTimeoutDestroy(pSyncMsg); - taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pPingTimer); + if (syncEnvIsStart()) { + taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pPingTimer); + } else { + sError("sync env is stop, syncNodeEqPingTimer"); + } + } else { sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%" PRIu64 ", pingTimerLogicClockUser:%" PRIu64 "", pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); @@ -1756,12 +1779,12 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { syncTimeoutDestroy(pSyncMsg); // reset timer ms - if (gSyncEnv != NULL) { + if (syncEnvIsStart()) { pSyncNode->electTimerMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); taosTmrReset(syncNodeEqElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pElectTimer); } else { - sError("sync env elect is already stop"); + sError("sync env is stop, syncNodeEqElectTimer"); } } else { sTrace("==syncNodeEqElectTimer== electTimerLogicClock:%" PRIu64 ", electTimerLogicClockUser:%" PRIu64 "", @@ -1792,11 +1815,11 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } syncTimeoutDestroy(pSyncMsg); - if (gSyncEnv != NULL) { + if (syncEnvIsStart()) { taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pHeartbeatTimer); } else { - sError("sync env heartbeat is already stop"); + sError("sync env is stop, syncNodeEqHeartbeatTimer"); } } else { sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 From 0c8b9db19ca6ecbfd8ba87df3469a6164f6c4e90 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 15 Jun 2022 12:39:30 +0800 Subject: [PATCH 15/41] test: add test case for tmq --- tests/system-test/7-tmq/schema.py | 195 ++++++++++++++++++++++++++---- tests/system-test/fulltest.sh | 1 + 2 files changed, 171 insertions(+), 25 deletions(-) diff --git a/tests/system-test/7-tmq/schema.py b/tests/system-test/7-tmq/schema.py index 54b6881318..a3462feb14 100644 --- a/tests/system-test/7-tmq/schema.py +++ b/tests/system-test/7-tmq/schema.py @@ -358,8 +358,8 @@ class TDTestCase: tdSql.error("alter table %s.%s modify column c2 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s modify tag t2 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t1 10"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t2 '20'"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s set tag t1=20"%(parameterDict['dbName'], ctbName)) + tdSql.error("alter table %s.%s set tag t2='20'"%(parameterDict['dbName'], ctbName)) tdSql.error("alter table %s.%s rename column c1 c1new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s rename column c2 c2new"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -370,9 +370,9 @@ class TDTestCase: tdSql.query("alter table %s.%s modify column c4 binary(60)"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s modify tag t4 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s set tag t3 30"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s set tag t4 '40'"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s set tag t5 '50'"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s set tag t3=20"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t4='20'"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t5='20'"%(parameterDict['dbName'], ctbName)) tdSql.query("alter table %s.%s rename column c3 c3new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s rename column c4 c4new"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -395,7 +395,7 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 2: ") parameterDict = {'cfg': '', \ 'actionType': 0, \ - 'dbName': 'db1', \ + 'dbName': 'db2', \ 'dropFlag': 1, \ 'vgroups': 4, \ 'replica': 1, \ @@ -407,8 +407,8 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - # tdLog.info("create database, super table, child table, normal table") - # self.create_database(tdSql, parameterDict["dbName"]) + tdLog.info("create database, super table, child table, normal table") + self.create_database(tdSql, parameterDict["dbName"]) ntbName = 'ntb2' tdSql.query("create table %s.%s (ts timestamp, c1 int, c2 binary(32), c3 double, c4 binary(32), c5 nchar(10)) tags (t1 int, t2 binary(32), t3 double, t4 binary(32), t5 nchar(10))"%(parameterDict["dbName"],parameterDict["stbName"])) tdSql.query("create table %s.%s (ts timestamp, c1 int, c2 binary(32), c3 double, c4 binary(32), c5 nchar(10))"%(parameterDict["dbName"],ntbName)) @@ -449,10 +449,10 @@ class TDTestCase: tdSql.query("alter table %s.%s modify column c5 nchar(60)"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s modify tag t5 nchar(60)"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s rename column c5 c5new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c5 c5new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s rename tag t5 t5new"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s drop column c5new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c5"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s drop tag t5new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s add column c5 int"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -508,10 +508,10 @@ class TDTestCase: tdSql.error("alter table %s.%s modify tag t2 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s modify tag t4 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t1 11"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t2 '22'"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t3 33"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t4 '44'"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s set tag t1=20"%(parameterDict['dbName'], ctbName)) + tdSql.error("alter table %s.%s set tag t2='20'"%(parameterDict['dbName'], ctbName)) + tdSql.error("alter table %s.%s set tag t3=20"%(parameterDict['dbName'], ctbName)) + tdSql.error("alter table %s.%s set tag t4='20'"%(parameterDict['dbName'], ctbName)) tdSql.error("alter table %s.%s rename column c1 c1new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s rename column c2 c2new"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -526,7 +526,7 @@ class TDTestCase: tdSql.query("alter table %s.%s modify column c5 nchar(60)"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s modify tag t5 nchar(40)"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.query("alter table %s.%s set tag t5 '50'"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s set tag t5='50'"%(parameterDict['dbName'], ctbName)) tdSql.query("alter table %s.%s rename column c5 c5new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s rename tag t5 t5new"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -543,7 +543,7 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 3: ") parameterDict = {'cfg': '', \ 'actionType': 0, \ - 'dbName': 'db1', \ + 'dbName': 'db3', \ 'dropFlag': 1, \ 'vgroups': 4, \ 'replica': 1, \ @@ -555,7 +555,8 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - # tdLog.info("create database, super table, child table, normal table") + tdLog.info("create database, super table, child table, normal table") + self.create_database(tdSql, parameterDict["dbName"]) ntbName = 'ntb3' tdSql.query("create table %s.%s (ts timestamp, c1 int, c2 binary(32), c3 double, c4 binary(32), c5 nchar(10)) tags (t1 int, t2 binary(32), t3 double, t4 binary(32), t5 nchar(10))"%(parameterDict["dbName"],parameterDict["stbName"])) tdSql.query("create table %s.%s (ts timestamp, c1 int, c2 binary(32), c3 double, c4 binary(32), c5 nchar(10))"%(parameterDict["dbName"],ntbName)) @@ -627,7 +628,7 @@ class TDTestCase: parameterDict['stbName'] = 'stb31' ctbName = 'stb31_0' tdSql.query("create table %s.%s (ts timestamp, c1 int, c2 binary(32), c3 double, c4 binary(32), c5 nchar(10)) tags (t1 int, t2 binary(32), t3 double, t4 binary(32), t5 nchar(10))"%(parameterDict["dbName"],parameterDict['stbName'])) - tdSql.query("create table %s.%s using %s.%s tags (10, 100, '1000')"%(parameterDict["dbName"],ctbName,parameterDict["dbName"],parameterDict['stbName'])) + tdSql.query("create table %s.%s using %s.%s tags (10, '10', 10, '10', '10')"%(parameterDict["dbName"],ctbName,parameterDict["dbName"],parameterDict['stbName'])) tdLog.info("create topics from child table") columnTopicFromCtb = 'column_topic_from_ctb3' @@ -653,11 +654,11 @@ class TDTestCase: tdSql.error("alter table %s.%s modify tag t4 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s modify tag t5 nchar(40)"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t1 10"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t2 '20'"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t3 30"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t4 '40'"%(parameterDict['dbName'], parameterDict['stbName'])) - tdSql.error("alter table %s.%s set tag t5 '50'"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s set tag t1=20"%(parameterDict['dbName'], ctbName)) + tdSql.error("alter table %s.%s set tag t2='20'"%(parameterDict['dbName'], ctbName)) + tdSql.error("alter table %s.%s set tag t3=20"%(parameterDict['dbName'], ctbName)) + tdSql.error("alter table %s.%s set tag t4='20'"%(parameterDict['dbName'], ctbName)) + tdSql.error("alter table %s.%s set tag t5='20'"%(parameterDict['dbName'], ctbName)) tdSql.error("alter table %s.%s rename column c1 c1new"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s rename column c2 c2new"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -676,6 +677,148 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 3 end ...... ") + def tmqCase4(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 4: ") + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db4', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb4', \ + 'ctbPrefix': 'stb4', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 23, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + ctbName = 'stb4_0' + + tdLog.info("create database, super table, child table, normal table") + self.create_database(tdSql, parameterDict["dbName"]) + tdSql.query("create table %s.%s (ts timestamp, c1 int, c2 binary(32), c3 double, c4 binary(32), c5 nchar(10)) tags (t1 int, t2 binary(32), t3 double, t4 binary(32), t5 nchar(10))"%(parameterDict["dbName"],parameterDict["stbName"])) + tdSql.query("create table %s.%s using %s.%s tags (10, '10', 10, '10', '10')"%(parameterDict["dbName"],ctbName,parameterDict["dbName"],parameterDict['stbName'])) + + tdLog.info("create topics from super table") + columnTopicFromStb = 'star_topic_from_stb4' + + tdSql.execute("create topic %s as stable %s.%s" %(columnTopicFromStb, parameterDict['dbName'], parameterDict['stbName'])) + + tdLog.info("======== child table test:") + tdSql.query("alter table %s.%s set tag t1=20"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t2='20'"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t3=20"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t4='20'"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t5='20'"%(parameterDict['dbName'], ctbName)) + + tdLog.info("======== super table test:") + # all alter actions allow + tdSql.query("alter table %s.%s add column c6 int"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s add tag t6 float"%(parameterDict['dbName'], parameterDict['stbName'])) + + tdSql.query("alter table %s.%s modify column c2 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s modify column c4 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s modify column c5 nchar(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s modify tag t2 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s modify tag t4 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s modify tag t5 nchar(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + + tdSql.error("alter table %s.%s rename column c1 c1new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c2 c2new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c3 c3new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c4 c4new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c5 c5new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s rename tag t1 t1new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s rename tag t2 t2new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s rename tag t3 t3new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s rename tag t4 t4new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s rename tag t5 t5new"%(parameterDict['dbName'], parameterDict['stbName'])) + + tdSql.query("alter table %s.%s drop column c1"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c2"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c3"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c4"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c5"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop tag t1new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop tag t2new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop tag t3new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop tag t4new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop tag t5new"%(parameterDict['dbName'], parameterDict['stbName'])) + + tdLog.printNoPrefix("======== test case 4 end ...... ") + + def tmqCase5(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 5: ") + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db5', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb5', \ + 'ctbPrefix': 'stb5', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 23, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + ctbName = 'stb5_0' + + tdLog.info("create database, super table, child table, normal table") + self.create_database(tdSql, parameterDict["dbName"]) + tdSql.query("create table %s.%s (ts timestamp, c1 int, c2 binary(32), c3 double, c4 binary(32), c5 nchar(10)) tags (t1 int, t2 binary(32), t3 double, t4 binary(32), t5 nchar(10))"%(parameterDict["dbName"],parameterDict["stbName"])) + tdSql.query("create table %s.%s using %s.%s tags (10, '10', 10, '10', '10')"%(parameterDict["dbName"],ctbName,parameterDict["dbName"],parameterDict['stbName'])) + + tdLog.info("create topics from super table") + columnTopicFromStb = 'star_topic_from_db5' + + tdSql.execute("create topic %s as database %s" %(columnTopicFromStb, parameterDict['dbName'])) + + tdLog.info("======== child table test:") + tdSql.query("alter table %s.%s set tag t1=20"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t2='20'"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t3=20"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t4='20'"%(parameterDict['dbName'], ctbName)) + tdSql.query("alter table %s.%s set tag t5='20'"%(parameterDict['dbName'], ctbName)) + + tdLog.info("======== super table test:") + # all alter actions allow + tdSql.query("alter table %s.%s add column c6 int"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s add tag t6 float"%(parameterDict['dbName'], parameterDict['stbName'])) + + tdSql.query("alter table %s.%s modify column c2 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s modify column c4 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s modify column c5 nchar(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s modify tag t2 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s modify tag t4 binary(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s modify tag t5 nchar(40)"%(parameterDict['dbName'], parameterDict['stbName'])) + + tdSql.error("alter table %s.%s rename column c1 c1new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c2 c2new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c3 c3new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c4 c4new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.error("alter table %s.%s rename column c5 c5new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s rename tag t1 t1new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s rename tag t2 t2new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s rename tag t3 t3new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s rename tag t4 t4new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s rename tag t5 t5new"%(parameterDict['dbName'], parameterDict['stbName'])) + + tdSql.query("alter table %s.%s drop column c1"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c2"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c3"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c4"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop column c5"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop tag t1new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop tag t2new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop tag t3new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop tag t4new"%(parameterDict['dbName'], parameterDict['stbName'])) + tdSql.query("alter table %s.%s drop tag t5new"%(parameterDict['dbName'], parameterDict['stbName'])) + + tdLog.printNoPrefix("======== test case 5 end ...... ") + def run(self): tdSql.prepare() @@ -687,9 +830,11 @@ class TDTestCase: cfgPath = buildPath + "/../sim/psim/cfg" tdLog.info("cfgPath: %s" % cfgPath) - self.tmqCase1(cfgPath, buildPath) - self.tmqCase2(cfgPath, buildPath) + # self.tmqCase1(cfgPath, buildPath) + # self.tmqCase2(cfgPath, buildPath) # self.tmqCase3(cfgPath, buildPath) + self.tmqCase4(cfgPath, buildPath) + self.tmqCase5(cfgPath, buildPath) def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 676f2da788..07e2137a4e 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -117,3 +117,4 @@ python3 ./test.py -f 7-tmq/subscribeStb3.py python3 ./test.py -f 7-tmq/subscribeStb4.py python3 ./test.py -f 7-tmq/db.py python3 ./test.py -f 7-tmq/tmqError.py +python3 ./test.py -f 7-tmq/schema.py From 8a1d0812fe1fb85deedb34f38038ad981a509b6a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 15 Jun 2022 13:22:36 +0800 Subject: [PATCH 16/41] fix last function crash on windows --- source/libs/function/src/builtinsimpl.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c6ab9f859d..04d7396d12 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2392,6 +2392,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; + int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; pInfo->bytes = bytes; @@ -2428,6 +2429,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { + if (IS_VAR_DATA_TYPE(type)) { + bytes = varDataTLen(data); + pInfo->bytes = bytes; + } memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; pInfo->hasResult = true; @@ -2458,6 +2463,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { + if (IS_VAR_DATA_TYPE(type)) { + bytes = varDataTLen(data); + pInfo->bytes = bytes; + } memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; pInfo->hasResult = true; @@ -2481,6 +2490,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; + int32_t type = pInputCol->info.type; int32_t bytes = pInputCol->info.bytes; pInfo->bytes = bytes; @@ -2508,6 +2518,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { + if (IS_VAR_DATA_TYPE(type)) { + bytes = varDataTLen(data); + pInfo->bytes = bytes; + } memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); @@ -2527,6 +2541,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { + if (IS_VAR_DATA_TYPE(type)) { + bytes = varDataTLen(data); + pInfo->bytes = bytes; + } memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; pInfo->hasResult = true; From 7de08aaadcb1c7c4027b14e85e123e44a120d3f8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 15 Jun 2022 13:26:12 +0800 Subject: [PATCH 17/41] add db vg refresh --- include/common/systable.h | 1 + source/libs/catalog/src/ctgAsync.c | 90 ++++++++++++-------------- source/libs/parser/src/parTranslater.c | 8 +++ 3 files changed, 51 insertions(+), 48 deletions(-) diff --git a/include/common/systable.h b/include/common/systable.h index 8b0bb4a3fb..d2c28941c7 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -52,6 +52,7 @@ extern "C" { #define TSDB_PERFS_TABLE_OFFSETS "offsets" #define TSDB_PERFS_TABLE_TRANS "trans" #define TSDB_PERFS_TABLE_STREAMS "streams" +#define TSDB_PERFS_TABLE_APPS "apps" typedef struct SSysDbTableSchema { const char* name; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 77de4409ee..6e24d632e2 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -261,54 +261,48 @@ int32_t ctgInitGetTbIndexTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { } -int32_t ctgHandleForceUpdate(SCatalog* pCtg, SCtgJob *pJob, const SCatalogReq* pReq) { - int32_t dbNum = pJob->dbCfgNum + pJob->dbVgNum + pJob->dbInfoNum; - if (dbNum > 0) { - if (dbNum > pJob->dbCfgNum && dbNum > pJob->dbVgNum && dbNum > pJob->dbInfoNum) { - SHashObj* pDb = taosHashInit(dbNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (NULL == pDb) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); - } - - for (int32_t i = 0; i < pJob->dbVgNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbVgroup, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - } - - for (int32_t i = 0; i < pJob->dbCfgNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbCfg, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - } - - for (int32_t i = 0; i < pJob->dbInfoNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbInfo, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - } - - char* dbFName = taosHashIterate(pDb, NULL); - while (dbFName) { - ctgDropDbVgroupEnqueue(pCtg, dbFName, true); - dbFName = taosHashIterate(pDb, dbFName); - } - - taosHashCleanup(pDb); - } else { - for (int32_t i = 0; i < pJob->dbVgNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbVgroup, i); - CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); - } - - for (int32_t i = 0; i < pJob->dbCfgNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbCfg, i); - CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); - } - - for (int32_t i = 0; i < pJob->dbInfoNum; ++i) { - char* dbFName = taosArrayGet(pReq->pDbInfo, i); - CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); - } - } +int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, const SCatalogReq* pReq) { + SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pDb) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + + for (int32_t i = 0; i < pJob->dbVgNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbVgroup, i); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } + + for (int32_t i = 0; i < pJob->dbCfgNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbCfg, i); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } + + for (int32_t i = 0; i < pJob->dbInfoNum; ++i) { + char* dbFName = taosArrayGet(pReq->pDbInfo, i); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } + + for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { + SName* name = taosArrayGet(pReq->pTableMeta, i); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(name, dbFName); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } + + for (int32_t i = 0; i < pJob->tbHashNum; ++i) { + SName* name = taosArrayGet(pReq->pTableHash, i); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(name, dbFName); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } + + char* dbFName = taosHashIterate(pDb, NULL); + while (dbFName) { + ctgDropDbVgroupEnqueue(pCtg, dbFName, true); + dbFName = taosHashIterate(pDb, dbFName); + } + + taosHashCleanup(pDb); int32_t tbNum = pJob->tbMetaNum + pJob->tbHashNum; if (tbNum > 0) { @@ -404,7 +398,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint6 } if (pReq->forceUpdate) { - CTG_ERR_JRET(ctgHandleForceUpdate(pCtg, pJob, pReq)); + CTG_ERR_JRET(ctgHandleForceUpdate(pCtg, *taskNum, pJob, pReq)); } int32_t taskIdx = 0; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 52abeadac9..b94a3d200e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4124,12 +4124,14 @@ static const char* getSysDbName(ENodeType type) { case QUERY_NODE_SHOW_SNODES_STMT: case QUERY_NODE_SHOW_LICENCE_STMT: case QUERY_NODE_SHOW_CLUSTER_STMT: + case QUERY_NODE_SHOW_VARIABLE_STMT: return TSDB_INFORMATION_SCHEMA_DB; case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_STREAMS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + case QUERY_NODE_SHOW_APPS_STMT: return TSDB_PERFORMANCE_SCHEMA_DB; default: break; @@ -4179,6 +4181,10 @@ static const char* getSysTableName(ENodeType type) { return TSDB_PERFS_TABLE_TOPICS; case QUERY_NODE_SHOW_TRANSACTIONS_STMT: return TSDB_PERFS_TABLE_TRANS; + case QUERY_NODE_SHOW_VARIABLE_STMT: + return TSDB_INS_TABLE_CONFIGS; + case QUERY_NODE_SHOW_APPS_STMT: + return TSDB_PERFS_TABLE_APPS; default: break; } @@ -5233,6 +5239,8 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_SHOW_CLUSTER_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + case QUERY_NODE_SHOW_VARIABLE_STMT: + case QUERY_NODE_SHOW_APPS_STMT: code = rewriteShow(pCxt, pQuery); break; case QUERY_NODE_CREATE_TABLE_STMT: From 19ab15b19ffe735a3a1163fb427724c1403e2382 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 15 Jun 2022 13:49:29 +0800 Subject: [PATCH 18/41] feat: super table join --- include/common/tmsg.h | 11 ++++--- include/common/ttokendef.h | 2 +- include/libs/nodes/cmdnodes.h | 6 ++++ include/libs/nodes/nodes.h | 1 + include/libs/nodes/plannodes.h | 9 +++++- include/util/tdef.h | 5 +++ source/common/src/tmsg.c | 6 ++-- source/libs/nodes/src/nodesCodeFuncs.c | 16 +++------- source/libs/nodes/src/nodesUtilFuncs.c | 5 ++- source/libs/parser/inc/parAst.h | 2 ++ source/libs/parser/inc/sql.y | 25 ++++++++------- source/libs/parser/src/parAstCreater.c | 25 ++++++++++----- source/libs/parser/src/parTokenizer.c | 3 +- source/libs/parser/src/parTranslater.c | 11 +++++-- source/libs/parser/src/sql.c | 28 ++++++++--------- source/libs/parser/test/parInitialCTest.cpp | 8 ++--- source/libs/planner/src/planPhysiCreater.c | 34 +++++++++++++++------ source/libs/planner/src/planSpliter.c | 7 +++-- tests/script/tsim/db/alter_option.sim | 2 +- tests/script/tsim/db/create_all_options.sim | 18 +++++------ 20 files changed, 137 insertions(+), 87 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7501c8fce0..4b624e333f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1134,11 +1134,11 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); void tFreeSTableMetaRsp(STableMetaRsp* pRsp); -void tFreeSTableIndexRsp(void *info); +void tFreeSTableIndexRsp(void* info); typedef struct { - SArray* pMetaRsp; // Array of STableMetaRsp - SArray* pIndexRsp; // Array of STableIndexRsp; + SArray* pMetaRsp; // Array of STableMetaRsp + SArray* pIndexRsp; // Array of STableIndexRsp; } SSTbHbRsp; int32_t tSerializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); @@ -1305,8 +1305,9 @@ int32_t tSerializeSSetStandbyReq(void* buf, int32_t bufLen, SSetStandbyReq* pReq int32_t tDeserializeSSetStandbyReq(void* buf, int32_t bufLen, SSetStandbyReq* pReq); typedef struct { - int32_t connId; - int32_t queryId; + int32_t connId; // todo remove + int32_t queryId; // todo remove + char queryStrId[TSDB_QUERY_ID_LEN]; } SKillQueryReq; int32_t tSerializeSKillQueryReq(void* buf, int32_t bufLen, SKillQueryReq* pReq); diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index f5495db5ae..061c734484 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -78,7 +78,7 @@ #define TK_BUFFER 60 #define TK_CACHELAST 61 #define TK_COMP 62 -#define TK_DAYS 63 +#define TK_DURATION 63 #define TK_NK_VARIABLE 64 #define TK_FSYNC 65 #define TK_MAXROWS 66 diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 8d4e94663f..1282836425 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -89,6 +89,7 @@ typedef struct STableOptions { ENodeType type; char comment[TSDB_TB_COMMENT_LEN]; double filesFactor; + int32_t delay; SNodeList* pRollupFuncs; int32_t ttl; SNodeList* pSma; @@ -286,6 +287,11 @@ typedef struct SKillStmt { int32_t targetId; } SKillStmt; +typedef struct SKillQueryStmt { + ENodeType type; + char queryId[TSDB_QUERY_ID_LEN]; +} SKillQueryStmt; + typedef struct SStreamOptions { ENodeType type; int8_t triggerType; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 3937fc13f4..3c25d8add4 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -204,6 +204,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, + QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, QUERY_NODE_PHYSICAL_PLAN_PROJECT, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 1a3bfdbb04..5d3e0cd142 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -34,7 +34,13 @@ typedef struct SLogicNode { uint8_t precision; } SLogicNode; -typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType; +typedef enum EScanType { + SCAN_TYPE_TAG = 1, + SCAN_TYPE_TABLE, + SCAN_TYPE_SYSTEM_TABLE, + SCAN_TYPE_STREAM, + SCAN_TYPE_TABLE_MERGE +} EScanType; typedef struct SScanLogicNode { SLogicNode node; @@ -262,6 +268,7 @@ typedef struct STableScanPhysiNode { } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; +typedef STableScanPhysiNode STableMergeScanPhysiNode; typedef STableScanPhysiNode SStreamScanPhysiNode; typedef struct SProjectPhysiNode { diff --git a/include/util/tdef.h b/include/util/tdef.h index 1fa8ad1912..1a8ba6a620 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -222,6 +222,8 @@ typedef enum ELogicConditionType { #define TSDB_APP_NAME_LEN TSDB_UNI_LEN #define TSDB_TB_COMMENT_LEN 1025 +#define TSDB_QUERY_ID_LEN 26 + /** * In some scenarios uint16_t (0~65535) is used to store the row len. * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. @@ -341,6 +343,9 @@ typedef enum ELogicConditionType { #define TSDB_DB_SCHEMALESS_OFF 0 #define TSDB_DEFAULT_DB_SCHEMALESS TSDB_DB_SCHEMALESS_OFF +// #define TSDB_MIN_ROLLUP_DELAY 1 +// #define TSDB_MAX_ROLLUP_DELAY 10 +// #define TSDB_DEFAULT_ROLLUP_DELAY 1 #define TSDB_MIN_ROLLUP_FILE_FACTOR 0 #define TSDB_MAX_ROLLUP_FILE_FACTOR 10 #define TSDB_DEFAULT_ROLLUP_FILE_FACTOR 0.1 diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index aafeca0b39..420ed8dcb2 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3369,8 +3369,7 @@ int32_t tSerializeSKillQueryReq(void *buf, int32_t bufLen, SKillQueryReq *pReq) tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pReq->connId) < 0) return -1; - if (tEncodeI32(&encoder, pReq->queryId) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->queryStrId) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3383,8 +3382,7 @@ int32_t tDeserializeSKillQueryReq(void *buf, int32_t bufLen, SKillQueryReq *pReq tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->connId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->queryId) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->queryStrId) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index be1929d554..d73434f3f0 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1480,14 +1480,6 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { return code; } -static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) { - return physiTableScanNodeToJson(pObj, pJson); -} - -static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { - return jsonToPhysiTableScanNode(pJson, pObj); -} - static const char* jkSysTableScanPhysiPlanMnodeEpSet = "MnodeEpSet"; static const char* jkSysTableScanPhysiPlanShowRewrite = "ShowRewrite"; static const char* jkSysTableScanPhysiPlanAccountId = "AccountId"; @@ -3964,9 +3956,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: return physiTagScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - return physiTableScanNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - return physiStreamScanNodeToJson(pObj, pJson); + return physiTableScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: return physiSysTableScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: @@ -4097,9 +4089,9 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: return jsonToPhysiTagScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - return jsonToPhysiTableScanNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - return jsonToPhysiStreamScanNode(pJson, pObj); + return jsonToPhysiTableScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: return jsonToPhysiSysTableScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 72666f833d..3bff010d43 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -209,9 +209,10 @@ SNode* nodesMakeNode(ENodeType type) { case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT: return makeNode(type, sizeof(SShowStmt)); - case QUERY_NODE_KILL_CONNECTION_STMT: case QUERY_NODE_KILL_QUERY_STMT: + return makeNode(type, sizeof(SKillQueryStmt)); case QUERY_NODE_KILL_TRANSACTION_STMT: + case QUERY_NODE_KILL_CONNECTION_STMT: return makeNode(type, sizeof(SKillStmt)); case QUERY_NODE_DELETE_STMT: return makeNode(type, sizeof(SDeleteStmt)); @@ -251,6 +252,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(STableScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: return makeNode(type, sizeof(STableSeqScanPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + return makeNode(type, sizeof(STableMergeScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: return makeNode(type, sizeof(SStreamScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 0c6663d29a..b8c6ffc843 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -60,6 +60,7 @@ typedef enum EDatabaseOptionType { typedef enum ETableOptionType { TABLE_OPTION_COMMENT = 1, TABLE_OPTION_FILE_FACTOR, + TABLE_OPTION_DELAY, TABLE_OPTION_ROLLUP, TABLE_OPTION_TTL, TABLE_OPTION_SMA @@ -187,6 +188,7 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SNode* pOptions, SNode* pQuery); SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pStreamName); SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId); +SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId); SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt); SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2); SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 83ad41aac0..0d9e6e4a73 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -168,8 +168,8 @@ db_options(A) ::= . db_options(A) ::= db_options(B) BUFFER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_BUFFER, &C); } db_options(A) ::= db_options(B) CACHELAST NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_CACHELAST, &C); } db_options(A) ::= db_options(B) COMP NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_COMP, &C); } -db_options(A) ::= db_options(B) DAYS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } -db_options(A) ::= db_options(B) DAYS NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } +db_options(A) ::= db_options(B) DURATION NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } +db_options(A) ::= db_options(B) DURATION NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_DAYS, &C); } db_options(A) ::= db_options(B) FSYNC NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_FSYNC, &C); } db_options(A) ::= db_options(B) MAXROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MAXROWS, &C); } db_options(A) ::= db_options(B) MINROWS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_MINROWS, &C); } @@ -317,8 +317,9 @@ tags_def(A) ::= TAGS NK_LP column_def_list(B) NK_RP. table_options(A) ::= . { A = createDefaultTableOptions(pCxt); } table_options(A) ::= table_options(B) COMMENT NK_STRING(C). { A = setTableOption(pCxt, B, TABLE_OPTION_COMMENT, &C); } +//table_options(A) ::= table_options(B) DELAY NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_DELAY, &C); } table_options(A) ::= table_options(B) FILE_FACTOR NK_FLOAT(C). { A = setTableOption(pCxt, B, TABLE_OPTION_FILE_FACTOR, &C); } -table_options(A) ::= table_options(B) ROLLUP NK_LP func_name_list(C) NK_RP. { A = setTableOption(pCxt, B, TABLE_OPTION_ROLLUP, C); } +table_options(A) ::= table_options(B) ROLLUP NK_LP rollup_func_list(C) NK_RP. { A = setTableOption(pCxt, B, TABLE_OPTION_ROLLUP, C); } table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableOption(pCxt, B, TABLE_OPTION_SMA, C); } @@ -330,6 +331,15 @@ alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). alter_table_option(A) ::= COMMENT NK_STRING(B). { A.type = TABLE_OPTION_COMMENT; A.val = B; } alter_table_option(A) ::= TTL NK_INTEGER(B). { A.type = TABLE_OPTION_TTL; A.val = B; } +%type rollup_func_list { SNodeList* } +%destructor rollup_func_list { nodesDestroyList($$); } +rollup_func_list(A) ::= rollup_func_name(B). { A = createNodeList(pCxt, B); } +rollup_func_list(A) ::= rollup_func_list(B) NK_COMMA rollup_func_name(C). { A = addNodeToList(pCxt, B, C); } + +rollup_func_name(A) ::= function_name(B). { A = createFunctionNode(pCxt, &B, NULL); } +rollup_func_name(A) ::= FIRST(B). { A = createFunctionNode(pCxt, &B, NULL); } +rollup_func_name(A) ::= LAST(B). { A = createFunctionNode(pCxt, &B, NULL); } + %type col_name_list { SNodeList* } %destructor col_name_list { nodesDestroyList($$); } col_name_list(A) ::= col_name(B). { A = createNodeList(pCxt, B); } @@ -378,13 +388,6 @@ table_name_cond(A) ::= table_name(B). from_db_opt(A) ::= . { A = createDefaultDatabaseCondValue(pCxt); } from_db_opt(A) ::= FROM db_name(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } -%type func_name_list { SNodeList* } -%destructor func_name_list { nodesDestroyList($$); } -func_name_list(A) ::= func_name(B). { A = createNodeList(pCxt, B); } -func_name_list(A) ::= func_name_list(B) NK_COMMA func_name(C). { A = addNodeToList(pCxt, B, C); } - -func_name(A) ::= function_name(B). { A = createFunctionNode(pCxt, &B, NULL); } - /************************************************ create index ********************************************************/ cmd ::= CREATE SMA INDEX not_exists_opt(D) index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, &A, &B, NULL, C); } @@ -466,7 +469,7 @@ stream_options(A) ::= stream_options(B) WATERMARK duration_literal(C). /************************************************ kill connection/query ***********************************************/ cmd ::= KILL CONNECTION NK_INTEGER(A). { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &A); } -cmd ::= KILL QUERY NK_INTEGER(A). { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_QUERY_STMT, &A); } +cmd ::= KILL QUERY NK_STRING(A). { pCxt->pRootNode = createKillQueryStmt(pCxt, &A); } cmd ::= KILL TRANSACTION NK_INTEGER(A). { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &A); } /************************************************ merge/redistribute/ vgroup ******************************************/ diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index c75696cede..8a7987e4ec 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -804,10 +804,10 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti case DB_OPTION_RETENTIONS: ((SDatabaseOptions*)pOptions)->pRetentions = pVal; break; -// case DB_OPTION_SCHEMALESS: -// ((SDatabaseOptions*)pOptions)->schemaless = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); -// ((SDatabaseOptions*)pOptions)->schemaless = 0; -// break; + // case DB_OPTION_SCHEMALESS: + // ((SDatabaseOptions*)pOptions)->schemaless = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + // ((SDatabaseOptions*)pOptions)->schemaless = 0; + // break; default: break; } @@ -867,6 +867,7 @@ SNode* createDefaultTableOptions(SAstCreateContext* pCxt) { CHECK_PARSER_STATUS(pCxt); STableOptions* pOptions = (STableOptions*)nodesMakeNode(QUERY_NODE_TABLE_OPTIONS); CHECK_OUT_OF_MEM(pOptions); + // pOptions->delay = TSDB_DEFAULT_ROLLUP_DELAY; pOptions->filesFactor = TSDB_DEFAULT_ROLLUP_FILE_FACTOR; pOptions->ttl = TSDB_DEFAULT_TABLE_TTL; return (SNode*)pOptions; @@ -876,7 +877,7 @@ SNode* createAlterTableOptions(SAstCreateContext* pCxt) { CHECK_PARSER_STATUS(pCxt); STableOptions* pOptions = (STableOptions*)nodesMakeNode(QUERY_NODE_TABLE_OPTIONS); CHECK_OUT_OF_MEM(pOptions); - pOptions->filesFactor = -1; + pOptions->delay = -1; pOptions->ttl = -1; return (SNode*)pOptions; } @@ -890,8 +891,8 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType sizeof(((STableOptions*)pOptions)->comment)); } break; - case TABLE_OPTION_FILE_FACTOR: - ((STableOptions*)pOptions)->filesFactor = taosStr2Double(((SToken*)pVal)->z, NULL); + case TABLE_OPTION_DELAY: + ((STableOptions*)pOptions)->delay = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case TABLE_OPTION_ROLLUP: ((STableOptions*)pOptions)->pRollupFuncs = pVal; @@ -1431,7 +1432,7 @@ SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const CHECK_PARSER_STATUS(pCxt); SDropStreamStmt* pStmt = (SDropStreamStmt*)nodesMakeNode(QUERY_NODE_DROP_STREAM_STMT); CHECK_OUT_OF_MEM(pStmt); - strncpy(pStmt->streamName, pStreamName->z, pStreamName->n); + strncpy(pStmt->streamName, pStreamName->z, TMIN(pStreamName->n, sizeof(pStmt->streamName) - 1)); pStmt->ignoreNotExists = ignoreNotExists; return (SNode*)pStmt; } @@ -1444,6 +1445,14 @@ SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId return (SNode*)pStmt; } +SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId) { + CHECK_PARSER_STATUS(pCxt); + SKillQueryStmt* pStmt = (SKillQueryStmt*)nodesMakeNode(QUERY_NODE_KILL_QUERY_STMT); + CHECK_OUT_OF_MEM(pStmt); + strncpy(pStmt->queryId, pQueryId->z, TMIN(pQueryId->n, sizeof(pStmt->queryId) - 1)); + return (SNode*)pStmt; +} + SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt) { CHECK_PARSER_STATUS(pCxt); SBalanceVgroupStmt* pStmt = (SBalanceVgroupStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_STMT); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index a9d730e1ad..55ea7171fa 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -68,7 +68,7 @@ static SKeyword keywordTable[] = { {"CONTAINS", TK_CONTAINS}, {"DATABASE", TK_DATABASE}, {"DATABASES", TK_DATABASES}, - {"DAYS", TK_DAYS}, + // {"DAYS", TK_DAYS}, {"DBS", TK_DBS}, {"DELETE", TK_DELETE}, {"DESC", TK_DESC}, @@ -78,6 +78,7 @@ static SKeyword keywordTable[] = { {"DNODES", TK_DNODES}, {"DOUBLE", TK_DOUBLE}, {"DROP", TK_DROP}, + {"DURATION", TK_DURATION}, {"EXISTS", TK_EXISTS}, {"EXPLAIN", TK_EXPLAIN}, {"FILE_FACTOR", TK_FILE_FACTOR}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 52abeadac9..bfb58d2cd1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2839,6 +2839,9 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt if (TSDB_CODE_SUCCESS == code) { code = checTableFactorOption(pCxt, pStmt->pOptions->filesFactor); } + // if (TSDB_CODE_SUCCESS == code) { + // code = checkRangeOption(pCxt, "delay", pStmt->pOptions->delay, TSDB_MIN_ROLLUP_DELAY, TSDB_MAX_ROLLUP_DELAY); + // } if (TSDB_CODE_SUCCESS == code) { code = checkTableRollupOption(pCxt, pStmt->pOptions->pRollupFuncs); } @@ -3081,6 +3084,7 @@ static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) { pReq->igExists = pStmt->ignoreExists; + // pReq->delay = pStmt->pOptions->delay; pReq->xFilesFactor = pStmt->pOptions->filesFactor; pReq->ttl = pStmt->pOptions->ttl; columnDefNodeToField(pStmt->pCols, &pReq->pColumns); @@ -3626,9 +3630,9 @@ static int32_t translateKillConnection(STranslateContext* pCxt, SKillStmt* pStmt return buildCmdMsg(pCxt, TDMT_MND_KILL_CONN, (FSerializeFunc)tSerializeSKillQueryReq, &killReq); } -static int32_t translateKillQuery(STranslateContext* pCxt, SKillStmt* pStmt) { +static int32_t translateKillQuery(STranslateContext* pCxt, SKillQueryStmt* pStmt) { SKillQueryReq killReq = {0}; - killReq.queryId = pStmt->targetId; + strcpy(killReq.queryStrId, pStmt->queryId); return buildCmdMsg(pCxt, TDMT_MND_KILL_QUERY, (FSerializeFunc)tSerializeSKillQueryReq, &killReq); } @@ -3970,7 +3974,7 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { code = translateKillConnection(pCxt, (SKillStmt*)pNode); break; case QUERY_NODE_KILL_QUERY_STMT: - code = translateKillQuery(pCxt, (SKillStmt*)pNode); + code = translateKillQuery(pCxt, (SKillQueryStmt*)pNode); break; case QUERY_NODE_KILL_TRANSACTION_STMT: code = translateKillTransaction(pCxt, (SKillStmt*)pNode); @@ -4793,6 +4797,7 @@ static int32_t buildDropTableVgroupHashmap(STranslateContext* pCxt, SDropTableCl if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code && pClause->ignoreNotExists) { code = TSDB_CODE_SUCCESS; + goto over; } *pIsSuperTable = false; diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index b2de892f30..e81610c1b4 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -564,7 +564,7 @@ static const YYCODETYPE yy_lookahead[] = { /* 1260 */ 0, 0, 40, 0, 271, 72, 0, 47, 175, 175, /* 1270 */ 47, 47, 279, 310, 47, 0, 313, 314, 315, 316, /* 1280 */ 317, 318, 289, 320, 47, 47, 293, 243, 175, 0, - /* 1290 */ 175, 0, 47, 0, 47, 0, 243, 47, 0, 81, + /* 1290 */ 175, 0, 47, 0, 22, 0, 243, 47, 0, 81, /* 1300 */ 113, 160, 156, 310, 159, 0, 313, 314, 315, 316, /* 1310 */ 317, 318, 0, 320, 152, 271, 323, 151, 0, 356, /* 1320 */ 357, 328, 0, 279, 271, 44, 0, 0, 0, 0, @@ -693,7 +693,7 @@ static const unsigned short int yy_shift_ofst[] = { /* 310 */ 929, 931, 826, 875, 934, 952, 962, 965, 974, 976, /* 320 */ 859, 935, 1260, 1261, 1222, 1263, 1193, 1266, 1220, 1093, /* 330 */ 1223, 1224, 1227, 1094, 1275, 1237, 1238, 1113, 1289, 1115, - /* 340 */ 1291, 1245, 1293, 1247, 1295, 1250, 1298, 1218, 1141, 1145, + /* 340 */ 1291, 1245, 1293, 1272, 1295, 1250, 1298, 1218, 1141, 1145, /* 350 */ 1187, 1146, 1305, 1312, 1162, 1166, 1318, 1322, 1281, 1326, /* 360 */ 1327, 1328, 1329, 1330, 1331, 1334, 1335, 1336, 1338, 1339, /* 370 */ 1340, 1341, 1343, 1344, 1345, 1347, 1348, 1309, 1351, 1352, @@ -898,7 +898,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* BUFFER => nothing */ 0, /* CACHELAST => nothing */ 0, /* COMP => nothing */ - 0, /* DAYS => nothing */ + 0, /* DURATION => nothing */ 0, /* NK_VARIABLE => nothing */ 0, /* FSYNC => nothing */ 0, /* MAXROWS => nothing */ @@ -1225,7 +1225,7 @@ static const char *const yyTokenName[] = { /* 60 */ "BUFFER", /* 61 */ "CACHELAST", /* 62 */ "COMP", - /* 63 */ "DAYS", + /* 63 */ "DURATION", /* 64 */ "NK_VARIABLE", /* 65 */ "FSYNC", /* 66 */ "MAXROWS", @@ -1600,8 +1600,8 @@ static const char *const yyRuleName[] = { /* 68 */ "db_options ::= db_options BUFFER NK_INTEGER", /* 69 */ "db_options ::= db_options CACHELAST NK_INTEGER", /* 70 */ "db_options ::= db_options COMP NK_INTEGER", - /* 71 */ "db_options ::= db_options DAYS NK_INTEGER", - /* 72 */ "db_options ::= db_options DAYS NK_VARIABLE", + /* 71 */ "db_options ::= db_options DURATION NK_INTEGER", + /* 72 */ "db_options ::= db_options DURATION NK_VARIABLE", /* 73 */ "db_options ::= db_options FSYNC NK_INTEGER", /* 74 */ "db_options ::= db_options MAXROWS NK_INTEGER", /* 75 */ "db_options ::= db_options MINROWS NK_INTEGER", @@ -1783,7 +1783,7 @@ static const char *const yyRuleName[] = { /* 251 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", /* 252 */ "stream_options ::= stream_options WATERMARK duration_literal", /* 253 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 254 */ "cmd ::= KILL QUERY NK_INTEGER", + /* 254 */ "cmd ::= KILL QUERY NK_STRING", /* 255 */ "cmd ::= KILL TRANSACTION NK_INTEGER", /* 256 */ "cmd ::= BALANCE VGROUP", /* 257 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", @@ -2646,8 +2646,8 @@ static const struct { { 254, -3 }, /* (68) db_options ::= db_options BUFFER NK_INTEGER */ { 254, -3 }, /* (69) db_options ::= db_options CACHELAST NK_INTEGER */ { 254, -3 }, /* (70) db_options ::= db_options COMP NK_INTEGER */ - { 254, -3 }, /* (71) db_options ::= db_options DAYS NK_INTEGER */ - { 254, -3 }, /* (72) db_options ::= db_options DAYS NK_VARIABLE */ + { 254, -3 }, /* (71) db_options ::= db_options DURATION NK_INTEGER */ + { 254, -3 }, /* (72) db_options ::= db_options DURATION NK_VARIABLE */ { 254, -3 }, /* (73) db_options ::= db_options FSYNC NK_INTEGER */ { 254, -3 }, /* (74) db_options ::= db_options MAXROWS NK_INTEGER */ { 254, -3 }, /* (75) db_options ::= db_options MINROWS NK_INTEGER */ @@ -2829,7 +2829,7 @@ static const struct { { 305, -4 }, /* (251) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ { 305, -3 }, /* (252) stream_options ::= stream_options WATERMARK duration_literal */ { 240, -3 }, /* (253) cmd ::= KILL CONNECTION NK_INTEGER */ - { 240, -3 }, /* (254) cmd ::= KILL QUERY NK_INTEGER */ + { 240, -3 }, /* (254) cmd ::= KILL QUERY NK_STRING */ { 240, -3 }, /* (255) cmd ::= KILL TRANSACTION NK_INTEGER */ { 240, -2 }, /* (256) cmd ::= BALANCE VGROUP */ { 240, -4 }, /* (257) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ @@ -3326,8 +3326,8 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy632 = setDatabaseOption(pCxt, yymsp[-2].minor.yy632, DB_OPTION_COMP, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy632 = yylhsminor.yy632; break; - case 71: /* db_options ::= db_options DAYS NK_INTEGER */ - case 72: /* db_options ::= db_options DAYS NK_VARIABLE */ yytestcase(yyruleno==72); + case 71: /* db_options ::= db_options DURATION NK_INTEGER */ + case 72: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==72); { yylhsminor.yy632 = setDatabaseOption(pCxt, yymsp[-2].minor.yy632, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy632 = yylhsminor.yy632; break; @@ -3907,8 +3907,8 @@ static YYACTIONTYPE yy_reduce( case 253: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 254: /* cmd ::= KILL QUERY NK_INTEGER */ -{ pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_QUERY_STMT, &yymsp[0].minor.yy0); } + case 254: /* cmd ::= KILL QUERY NK_STRING */ +{ pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } break; case 255: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 3a6c78d808..8b2c2aade7 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -46,7 +46,7 @@ TEST_F(ParserInitialCTest, createBnode) { * BUFFER value * | CACHELAST value * | COMP {0 | 1 | 2} - * | DAYS value + * | DURATION value * | FSYNC value * | MAXROWS value * | MINROWS value @@ -155,7 +155,7 @@ TEST_F(ParserInitialCTest, createDatabase) { ASSERT_EQ(req.replications, expect.replications); ASSERT_EQ(req.strict, expect.strict); ASSERT_EQ(req.cacheLastRow, expect.cacheLastRow); - //ASSERT_EQ(req.schemaless, expect.schemaless); + // ASSERT_EQ(req.schemaless, expect.schemaless); ASSERT_EQ(req.ignoreExist, expect.ignoreExist); ASSERT_EQ(req.numOfRetensions, expect.numOfRetensions); if (expect.numOfRetensions > 0) { @@ -202,7 +202,7 @@ TEST_F(ParserInitialCTest, createDatabase) { "BUFFER 64 " "CACHELAST 2 " "COMP 1 " - "DAYS 100 " + "DURATION 100 " "FSYNC 100 " "MAXROWS 1000 " "MINROWS 100 " @@ -223,7 +223,7 @@ TEST_F(ParserInitialCTest, createDatabase) { setDbDaysFunc(100); setDbKeepFunc(1440, 300 * 60, 400 * 1440); run("CREATE DATABASE IF NOT EXISTS wxy_db " - "DAYS 100m " + "DURATION 100m " "KEEP 1440m,300h,400d "); clearCreateDbReq(); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 8502ffc04a..618ff9acc6 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -181,7 +181,7 @@ static int16_t getUnsetSlotId(const SArray* pSlotIdsInfo) { } static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, - const char* pStmtName, bool output, bool reserve) { + const char* pStmtName, bool output, bool reserve) { if (NULL == pList) { return TSDB_CODE_SUCCESS; } @@ -463,10 +463,24 @@ static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubpla return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTagScan, pPhyNode); } +static ENodeType getScanOperatorType(EScanType scanType) { + switch (scanType) { + case SCAN_TYPE_TABLE: + return QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; + case SCAN_TYPE_STREAM: + return QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; + case SCAN_TYPE_TABLE_MERGE: + return QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; + default: + break; + } + return QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; +} + static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - STableScanPhysiNode* pTableScan = - (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, + getScanOperatorType(pScanLogicNode->scanType)); if (NULL == pTableScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -528,12 +542,12 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* static int32_t createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - int32_t res = createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); - if (res == TSDB_CODE_SUCCESS) { - ENodeType type = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; - setNodeType(*pPhyNode, type); - } - return res; + return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); +} + +static int32_t createTableMergeScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, + SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { + return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); } static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, @@ -547,6 +561,8 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, return createSystemTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_STREAM: return createStreamScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); + case SCAN_TYPE_TABLE_MERGE: + return createTableMergeScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); default: break; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 4b372bb7d4..4c202d457f 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -170,8 +170,8 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: return stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pNode); - // case QUERY_NODE_LOGIC_PLAN_JOIN: - // return !(((SJoinLogicNode*)pNode)->isSingleTableJoin); + case QUERY_NODE_LOGIC_PLAN_JOIN: + return !(((SJoinLogicNode*)pNode)->isSingleTableJoin); case QUERY_NODE_LOGIC_PLAN_AGG: return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: { @@ -642,6 +642,8 @@ static int32_t stbSplSplitScanNodeForJoin(SSplitContext* pCxt, SLogicSubplan* pS code = nodesListMakeStrictAppend(&pSubplan->pChildren, (SNode*)splCreateScanSubplan(pCxt, (SLogicNode*)pScan, SPLIT_FLAG_STABLE_SPLIT)); } + pScan->scanType = SCAN_TYPE_TABLE_MERGE; + ++(pCxt->groupId); return code; } @@ -703,7 +705,6 @@ static int32_t stableSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { break; } - ++(pCxt->groupId); pCxt->split = true; return code; } diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 12babea097..4692b9d91b 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -66,7 +66,7 @@ print ============= create database # | REPLICA value [1 | 3] # | WAL value [1 | 2] -sql create database db CACHELAST 3 COMP 0 DAYS 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 +sql create database db CACHELAST 3 COMP 0 DURATION 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 sql show databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index fac385a9a6..03068698e2 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -63,7 +63,7 @@ print ============= create database with all options # | PAGESIZE value [1~16384, default: 4] # | CACHELAST value [0, 1, 2, 3, default: 0] # | COMP [0 | 1 | 2, default: 2] -# | DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] +# | DURATION value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] # | FSYNC value [0 ~ 180000 ms, default: 3000] # | MAXROWS value [200~10000, default: 4096] # | MINROWS value [10~1000, default: 100] @@ -234,9 +234,9 @@ sql drop database db sql_error create database db COMP 3 sql_error create database db COMP -1 -#print ====> DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] +#print ====> DURATION value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] #print ====> KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day] -#sql create database db DAYS 60m KEEP 60m +#sql create database db DURATION 60m KEEP 60m #sql show databases #print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db #if $data6_db != 60 then @@ -246,7 +246,7 @@ sql_error create database db COMP -1 # return -1 #endi #sql drop database db -#sql create database db DAYS 60m KEEP 1d +#sql create database db DURATION 60m KEEP 1d #sql show databases #print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db #if $data6_db != 60 then @@ -255,7 +255,7 @@ sql_error create database db COMP -1 #if $data7_db != 1440,1440,1440 then # return -1 #endi -#sql create database db DAYS 3650d KEEP 365000d +#sql create database db DURATION 3650d KEEP 365000d #sql show databases #print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db #if $data6_db != 5256000 then @@ -265,10 +265,10 @@ sql_error create database db COMP -1 # return -1 #endi #sql drop database db -#sql_error create database db DAYS -59m -#sql_error create database db DAYS 59m -#sql_error create database db DAYS 5256001m -#sql_error create database db DAYS 3651d +#sql_error create database db DURATION -59m +#sql_error create database db DURATION 59m +#sql_error create database db DURATION 5256001m +#sql_error create database db DURATION 3651d #sql_error create database db KEEP -59m #sql_error create database db KEEP 14399m #sql_error create database db KEEP 525600001m From 893019d1062a96988fdc4d4b5be44ccdcab898d5 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 15 Jun 2022 13:50:13 +0800 Subject: [PATCH 19/41] enh: record git commit id in log file; disable source code backup --- Jenkinsfile2 | 11 +++++++++++ tests/parallel_test/collect_cases.sh | 18 +++++++++--------- tests/parallel_test/run.sh | 6 +++--- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 79a7bab0c3..94af11868a 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -38,6 +38,7 @@ def pre_test(){ sh ''' hostname date + env ''' sh ''' cd ${WK} @@ -83,24 +84,32 @@ def pre_test(){ cd ${WKC} git pull >/dev/null git log -5 + echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log + echo "community log: `git log -5`" >>${WKDIR}/jenkins.log git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD git log -5 + echo "community log merged: `git log -5`" >>${WKDIR}/jenkins.log cd ${WK} git pull >/dev/null git log -5 + echo "tdinternal log: `git log -5`" >>${WKDIR}/jenkins.log ''' } else if (env.CHANGE_URL =~ /\/TDinternal\//) { sh ''' cd ${WK} git pull >/dev/null git log -5 + echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log + echo "tdinternal log: `git log -5`" >>${WKDIR}/jenkins.log git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD git log -5 + echo "tdinternal log merged: `git log -5`" >>${WKDIR}/jenkins.log cd ${WKC} git pull >/dev/null git log -5 + echo "community log: `git log -5`" >>${WKDIR}/jenkins.log ''' } else { sh ''' @@ -116,6 +125,8 @@ def pre_test(){ git reset --hard git pull git log -5 + echo "python connector log: `git log -5`" >>${WKDIR}/jenkins.log + echo >>${WKDIR}/jenkins.log ''' return 1 } diff --git a/tests/parallel_test/collect_cases.sh b/tests/parallel_test/collect_cases.sh index c12413b981..bc942263d0 100755 --- a/tests/parallel_test/collect_cases.sh +++ b/tests/parallel_test/collect_cases.sh @@ -42,15 +42,15 @@ cat ../script/jenkins/basic.txt |grep -v "^#"|grep -v "^$"|sed "s/^/,,script,/" grep "^python" ../system-test/fulltest.sh |sed "s/^/,,system-test,/" >>$case_file # tar source code for run.sh to use -if [ $ent -eq 0 ]; then - cd ../../../ - rm -rf TDengine.tar.gz - tar --exclude=TDengine/debug --exclude=TDengine/sim --exclude=TDengine/release -czf TDengine.tar.gz TDengine taos-connector-python -else - cd ../../../../ - rm -rf TDinternal.tar.gz - tar --exclude=TDinternal/debug --exclude=TDinternal/sim --exclude=TDinternal/community/debug --exclude=TDinternal/community/release --exclude=TDinternal/community/sim -czf TDinternal.tar.gz TDinternal taos-connector-python -fi +# if [ $ent -eq 0 ]; then +# cd ../../../ +# rm -rf TDengine.tar.gz +# tar --exclude=TDengine/debug --exclude=TDengine/sim --exclude=TDengine/release -czf TDengine.tar.gz TDengine taos-connector-python +# else +# cd ../../../../ +# rm -rf TDinternal.tar.gz +# tar --exclude=TDinternal/debug --exclude=TDinternal/sim --exclude=TDinternal/community/debug --exclude=TDinternal/community/release --exclude=TDinternal/community/sim -czf TDinternal.tar.gz TDinternal taos-connector-python +# fi exit 0 diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index 3b0c0da8cd..1b54dc338e 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -291,7 +291,7 @@ function run_thread() { fi cmd="$scpcmd:${remote_sim_tar} $log_dir/${case_file}.sim.tar.gz" $cmd - # backup source code + # backup source code (disabled) source_tar_dir=$log_dir/TDengine_${hosts[index]} source_tar_file=TDengine.tar.gz if [ $ent -ne 0 ]; then @@ -301,8 +301,8 @@ function run_thread() { mkdir $source_tar_dir 2>/dev/null if [ $? -eq 0 ]; then cmd="$scpcmd:${workdirs[index]}/$source_tar_file $source_tar_dir" - echo "$cmd" - $cmd + # echo "$cmd" + # $cmd fi fi done From 5fed3d3a0f52f836488140f0a051b021cb0b2fe8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Jun 2022 13:58:19 +0800 Subject: [PATCH 20/41] refactor: not send create/drop tsma msg to vgroup --- source/dnode/mnode/impl/src/mndSma.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 23117a1323..b1738464b6 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -407,6 +407,7 @@ static int32_t mndSetUpdateSmaStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb return 0; } +#if 0 static int32_t mndSetCreateSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; @@ -445,6 +446,7 @@ static int32_t mndSetCreateSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } +#endif static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SSmaObj *pSma) { @@ -579,7 +581,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupCommitLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; - if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; + // if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER; if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, STREAM_TRIGGER_AT_ONCE, 0, pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -734,6 +736,7 @@ static int32_t mndSetDropSmaVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVg return 0; } +#if 0 static int32_t mndSetDropSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; @@ -774,6 +777,7 @@ static int32_t mndSetDropSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * return 0; } +#endif static int32_t mndSetDropSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { SVnodeGid *pVgid = pVgroup->vnodeGid + 0; @@ -824,7 +828,7 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; - if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER; + // if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER; if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -854,7 +858,7 @@ int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; - if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER; + // if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER; mndReleaseVgroup(pMnode, pVgroup); pVgroup = NULL; } From 406d57955ef5cadf2659bfda2c10165e742ff646 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Jun 2022 14:06:16 +0800 Subject: [PATCH 21/41] test: disable unsupport case --- tests/system-test/fulltest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 676f2da788..24a57b13e6 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -2,7 +2,7 @@ set -e set -x -python3 ./test.py -f 0-others/taosShell.py +#python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py python3 ./test.py -f 0-others/taosShellNetChk.py python3 ./test.py -f 0-others/telemetry.py @@ -105,7 +105,7 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -python3 ./test.py -f 7-tmq/basic5.py +#python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py python3 ./test.py -f 7-tmq/subscribeDb0.py python3 ./test.py -f 7-tmq/subscribeDb1.py From 045f0eb95121dfad6e9d1b830c31fe5ec3f84fdd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 15 Jun 2022 14:11:06 +0800 Subject: [PATCH 22/41] fix jsons filter error --- tests/system-test/2-query/json_tag.py | 59 ++++++++++++++------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 1107e9693e..54b41e5a94 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -244,8 +244,8 @@ class TDTestCase: tdSql.query("select * from jsons1 where jtag->'tag2'!='beijing'") tdSql.checkRows(5) #open - #tdSql.query("select * from jsons1 where jtag->'tag2'=''") - #tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag2'=''") + tdSql.checkRows(2) # # # where json value is int tdSql.query("select * from jsons1 where jtag->'tag1'=5") @@ -254,10 +254,10 @@ class TDTestCase: tdSql.query("select * from jsons1 where jtag->'tag1'=10") tdSql.checkRows(0) # open - #tdSql.query("select * from jsons1 where jtag->'tag1'<54") - #tdSql.checkRows(3) - #tdSql.query("select * from jsons1 where jtag->'tag1'<=11") - #tdSql.checkRows(3) + tdSql.query("select * from jsons1 where jtag->'tag1'<54") + tdSql.checkRows(4) + tdSql.query("select * from jsons1 where jtag->'tag1'<=11") + tdSql.checkRows(4) tdSql.query("select * from jsons1 where jtag->'tag1'>4") tdSql.checkRows(2) tdSql.query("select * from jsons1 where jtag->'tag1'>=5") @@ -271,30 +271,30 @@ class TDTestCase: tdSql.query("select * from jsons1 where jtag->'tag1'=1.232") tdSql.checkRows(1) # open - #tdSql.query("select * from jsons1 where jtag->'tag1'<1.232") - #tdSql.checkRows(0) - #tdSql.query("select * from jsons1 where jtag->'tag1'<=1.232") - #tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag1'<1.232") + tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag1'<=1.232") + tdSql.checkRows(2) tdSql.query("select * from jsons1 where jtag->'tag1'>1.23") tdSql.checkRows(3) tdSql.query("select * from jsons1 where jtag->'tag1'>=1.232") tdSql.checkRows(3) # open - #tdSql.query("select * from jsons1 where jtag->'tag1'!=1.232") - #tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag1'!=1.232") + tdSql.checkRows(6) tdSql.query("select * from jsons1 where jtag->'tag1'!=3.232") tdSql.checkRows(7) #tdSql.error("select * from jsons1 where jtag->'tag1'/0=3") #tdSql.error("select * from jsons1 where jtag->'tag1'/5=1") # # # where json value is bool - #tdSql.query("select * from jsons1 where jtag->'tag1'=true") + tdSql.query("select * from jsons1 where jtag->'tag1'=true") # open - #tdSql.checkRows(0) + tdSql.checkRows(0) #tdSql.query("select * from jsons1 where jtag->'tag1'=false") #tdSql.checkRows(1) - #tdSql.query("select * from jsons1 where jtag->'tag1'!=false") - #tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'!=false") + tdSql.checkRows(3) #tdSql.error("select * from jsons1 where jtag->'tag1'>false") # # # where json value is null @@ -304,17 +304,17 @@ class TDTestCase: # # # where json key is null # open - #tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3") - #tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3") + tdSql.checkRows(0) # # # where json value is not exist - #tdSql.query("select * from jsons1 where jtag->'tag1' is null") - #tdSql.checkData(0, 0, 'jsons1_9') - #tdSql.checkRows(1) - #tdSql.query("select * from jsons1 where jtag->'tag4' is null") - #tdSql.checkRows(9) - #tdSql.query("select * from jsons1 where jtag->'tag3' is not null") - #tdSql.checkRows(4) + tdSql.query("select * from jsons1 where jtag->'tag1' is null") + tdSql.checkData(0, 0, 'jsons1_9') + tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag4' is null") + tdSql.checkRows(9) + tdSql.query("select * from jsons1 where jtag->'tag3' is not null") + tdSql.checkRows(3) # # # test contains tdSql.query("select * from jsons1 where jtag contains 'tag1'") @@ -344,10 +344,10 @@ class TDTestCase: # # # # test with between and - #tdSql.query("select * from jsons1 where jtag->'tag1' between 1 and 30") - #tdSql.checkRows(3) - #tdSql.query("select * from jsons1 where jtag->'tag1' between 'femail' and 'beijing'") - #tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag1' between 1 and 30") + tdSql.checkRows(3) + tdSql.query("select * from jsons1 where jtag->'tag1' between 'femail' and 'beijing'") + tdSql.checkRows(2) # # # test with tbname/normal column tdSql.query("select * from jsons1 where tbname = 'jsons1_1'") @@ -362,6 +362,7 @@ class TDTestCase: # # # test where condition like # open + # syntax error #tdSql.query("select *,tbname from jsons1 where jtag->'tag2' like 'bei%'") #tdSql.checkRows(2) #tdSql.query("select *,tbname from jsons1 where jtag->'tag1' like 'fe%' and jtag->'tag2' is not null") From dc3dfad5d0af923883c0a467724bfcbd95aa3591 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 15 Jun 2022 14:13:51 +0800 Subject: [PATCH 23/41] [test: modify timeout len] --- tests/system-test/0-others/taosShellError.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/0-others/taosShellError.py b/tests/system-test/0-others/taosShellError.py index 2369e4d580..825ac015fb 100644 --- a/tests/system-test/0-others/taosShellError.py +++ b/tests/system-test/0-others/taosShellError.py @@ -48,21 +48,21 @@ def taos_command (buildPath, key, value, expectString, cfgDir, sqlString='', key #output = child.readline() #print (output.decode()) if len(expectString) != 0: - i = child.expect([expectString, taosExpect.TIMEOUT, taosExpect.EOF], timeout=6) + i = child.expect([expectString, taosExpect.TIMEOUT, taosExpect.EOF], timeout=20) else: - i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=6) + i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=20) if platform.system().lower() == 'windows': retResult = child.before else: retResult = child.before.decode() print("cmd return result:\n%s\n"%retResult) - #print(child.after.decode()) + # print(child.after.decode()) if i == 0: print ('taos login success! Here can run sql, taos> ') if len(sqlString) != 0: child.sendline (sqlString) - w = child.expect(["Query OK", taosExpect.TIMEOUT, taosExpect.EOF], timeout=1) + w = child.expect(["Query OK", taosExpect.TIMEOUT, taosExpect.EOF], timeout=10) if platform.system().lower() == 'windows': retResult = child.before else: From f740fd61c673d44e663e336e883486ce55a4a1c3 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 15 Jun 2022 14:23:05 +0800 Subject: [PATCH 24/41] fix: use 'duration' instead of 'days' for create database --- tests/pytest/alter/alter_replica.py | 2 +- tests/pytest/functions/function_max.py | 2 +- tests/pytest/functions/function_min.py | 2 +- tests/pytest/functions/queryTestCases.py | 2 +- tests/pytest/insert/before_1970.py | 2 +- tests/pytest/insert/retentionpolicy.py | 2 +- .../multilevel/fileDistributionSameLevel.py | 4 +-- tests/pytest/multilevel/retentionTest.py | 2 +- tests/pytest/perfbenchmark/bug3433.py | 2 +- tests/pytest/perfbenchmark/joinPerformance.py | 2 +- tests/pytest/perfbenchmark/taosdemoInsert.py | 2 +- tests/pytest/query/query1970YearsAf.py | 2 +- .../taosdemoTestSupportNanoInsert.py | 2 +- .../taosdemoTestSupportNanoInsert.py | 2 +- tests/pytest/tools/taosdumpTest.py | 6 ++-- tests/pytest/tools/taosdumpTestNanoSupport.py | 2 +- tests/pytest/update/merge_commit_data-0.py | 2 +- tests/pytest/update/merge_commit_data.py | 2 +- tests/pytest/update/merge_commit_data2.py | 2 +- .../update/merge_commit_data2_update0.py | 2 +- tests/pytest/update/merge_commit_last.py | 2 +- tests/pytest/util/sql.py | 2 +- tests/pytest/util/taosdemoCfg.py | 2 +- tests/script/general/db/alter_option.sim | 16 +++++----- tests/script/general/db/backup/keep.sim | 4 +-- tests/script/general/db/topic1.sim | 30 +++++++++---------- tests/script/general/import/commit.sim | 4 +-- tests/script/general/import/replica1.sim | 2 +- tests/script/general/parser/alter.sim | 2 +- .../parser/alter__for_community_version.sim | 2 +- tests/script/general/parser/create_db.sim | 6 ++-- .../create_db__for_community_version.sim | 6 ++-- tests/script/tsim/db/alter_option.sim | 14 ++++----- tests/script/tsim/db/basic6.sim | 4 +-- tests/script/tsim/db/create_all_options.sim | 4 +-- tests/script/tsim/insert/commit-merge0.sim | 2 +- tests/script/tsim/query/interval-offset.sim | 2 +- tests/script/tsim/stable/alter_count.sim | 2 +- tests/script/tsim/stable/alter_import.sim | 2 +- tests/script/unique/db/commit.sim | 2 +- tests/script/unique/import/replica2.sim | 2 +- tests/script/unique/import/replica3.sim | 2 +- tests/script/windows/db/basic.sim | 4 +-- tests/system-test/0-others/udfTest.py | 2 +- tests/system-test/0-others/udf_cluster.py | 2 +- tests/system-test/0-others/udf_create.py | 2 +- .../system-test/0-others/udf_restart_taosd.py | 2 +- tests/system-test/2-query/abs.py | 2 +- tests/system-test/2-query/sample.py | 2 +- tests/system-test/6-cluster/5dnode1mnode.py | 2 +- tests/system-test/6-cluster/5dnode2mnode.py | 4 +-- .../system-test/6-cluster/5dnode3mnodeDrop.py | 2 +- .../system-test/6-cluster/5dnode3mnodeStop.py | 2 +- .../6-cluster/5dnode3mnodeStopInsert.py | 2 +- 54 files changed, 94 insertions(+), 94 deletions(-) diff --git a/tests/pytest/alter/alter_replica.py b/tests/pytest/alter/alter_replica.py index 6cf0f65825..10047362db 100644 --- a/tests/pytest/alter/alter_replica.py +++ b/tests/pytest/alter/alter_replica.py @@ -42,7 +42,7 @@ class TDTestCase: tdDnodes.start(3) def run(self): - tdSql.execute('create database db replica 3 days 7') + tdSql.execute('create database db replica 3 duration 7') tdSql.execute('use db') for tid in range(1, 11): tdSql.execute('create table tb%d(ts timestamp, i int)' % tid) diff --git a/tests/pytest/functions/function_max.py b/tests/pytest/functions/function_max.py index c322b6af26..a53daa11d6 100644 --- a/tests/pytest/functions/function_max.py +++ b/tests/pytest/functions/function_max.py @@ -83,7 +83,7 @@ class TDTestCase: tdSql.checkData(0, 0, np.max(floatData)) # test case: https://jira.taosdata.com:18080/browse/TD-2583 - tdSql.execute("create database test days 2") + tdSql.execute("create database test duration 2") tdSql.execute("create table car(ts timestamp, speed int)") tdSql.execute("insert into car values(now, -1)") tdSql.execute("insert into car values(now-10d, null)") diff --git a/tests/pytest/functions/function_min.py b/tests/pytest/functions/function_min.py index b4d6d58f7c..db865e5ac7 100644 --- a/tests/pytest/functions/function_min.py +++ b/tests/pytest/functions/function_min.py @@ -83,7 +83,7 @@ class TDTestCase: tdSql.checkData(0, 0, np.min(floatData)) # test case: https://jira.taosdata.com:18080/browse/TD-2583 - tdSql.execute("create database test days 2") + tdSql.execute("create database test duration 2") tdSql.execute("create table car(ts timestamp, speed int)") tdSql.execute("insert into car values(now, 1)") tdSql.execute("insert into car values(now-10d, null)") diff --git a/tests/pytest/functions/queryTestCases.py b/tests/pytest/functions/queryTestCases.py index 1311ad6b3c..96de8f064c 100644 --- a/tests/pytest/functions/queryTestCases.py +++ b/tests/pytest/functions/queryTestCases.py @@ -264,7 +264,7 @@ class TDTestCase: def td4288(self): tdLog.printNoPrefix("==========TD-4288==========") - # keep ~ [days,365000] + # keep ~ [duration,365000] tdSql.execute("drop database if exists db") tdSql.execute("create database if not exists db") tdSql.query("show variables") diff --git a/tests/pytest/insert/before_1970.py b/tests/pytest/insert/before_1970.py index b2c4dc57c7..ca2b4f165f 100644 --- a/tests/pytest/insert/before_1970.py +++ b/tests/pytest/insert/before_1970.py @@ -32,7 +32,7 @@ class TDTestCase: print("==============step1") tdSql.execute("create database if not exists demo keep 36500;"); - print("==============create db demo keep 365000 days") + print("==============create db demo keep 365000 duration") tdSql.execute("use demo;") tdSql.execute("CREATE table if not exists test (ts timestamp, f1 int);") print("==============create table test") diff --git a/tests/pytest/insert/retentionpolicy.py b/tests/pytest/insert/retentionpolicy.py index 607ee26a59..78e5c08c8b 100644 --- a/tests/pytest/insert/retentionpolicy.py +++ b/tests/pytest/insert/retentionpolicy.py @@ -51,7 +51,7 @@ class TDTestRetetion: def run(self): tdLog.info("=============== step1") - tdSql.execute('create database test keep 3 days 1;') + tdSql.execute('create database test keep 3 duration 1;') tdSql.execute('use test;') tdSql.execute('create table test(ts timestamp,i int);') diff --git a/tests/pytest/multilevel/fileDistributionSameLevel.py b/tests/pytest/multilevel/fileDistributionSameLevel.py index 83ce856717..64b29914b7 100644 --- a/tests/pytest/multilevel/fileDistributionSameLevel.py +++ b/tests/pytest/multilevel/fileDistributionSameLevel.py @@ -66,7 +66,7 @@ class TDTestCase: tdDnodes.deploy(1,cfg) tdDnodes.startWithoutSleep(1) - tdSql.execute("create database test days 1") + tdSql.execute("create database test duration 1") tdSql.execute("use test") tdSql.execute("create table stb(ts timestamp, c int) tags(t int)") @@ -85,7 +85,7 @@ class TDTestCase: tdLog.info("================= step3") tdSql.execute('drop database test') for i in range(50): - tdSql.execute("create database test%d days 1" %(i)) + tdSql.execute("create database test%d duration 1" %(i)) tdSql.execute("use test%d" %(i)) tdSql.execute("create table tb (ts timestamp,i int)") for j in range(10): diff --git a/tests/pytest/multilevel/retentionTest.py b/tests/pytest/multilevel/retentionTest.py index 9ea59d9d59..94d268f257 100644 --- a/tests/pytest/multilevel/retentionTest.py +++ b/tests/pytest/multilevel/retentionTest.py @@ -56,7 +56,7 @@ class TDTestCase: tdDnodes.deploy(1,cfg) tdDnodes.startWithoutSleep(1) - tdSql.execute("create database test days 1 keep 15,5,10") + tdSql.execute("create database test duration 1 keep 15,5,10") tdSql.execute("use test") tdSql.execute("create table tb(ts timestamp, c int)") diff --git a/tests/pytest/perfbenchmark/bug3433.py b/tests/pytest/perfbenchmark/bug3433.py index e4480df6b6..d688dd0310 100644 --- a/tests/pytest/perfbenchmark/bug3433.py +++ b/tests/pytest/perfbenchmark/bug3433.py @@ -66,7 +66,7 @@ class TDTestCase: "name": "db", "drop": "yes", "replica": 1, - "days": 10, + "duration": 10, "cache": 16, "blocks": 8, "precision": "ms", diff --git a/tests/pytest/perfbenchmark/joinPerformance.py b/tests/pytest/perfbenchmark/joinPerformance.py index acd4f88c9b..2de5818c59 100644 --- a/tests/pytest/perfbenchmark/joinPerformance.py +++ b/tests/pytest/perfbenchmark/joinPerformance.py @@ -81,7 +81,7 @@ class JoinPerf: "name": self.dbname, "drop": self.drop, "replica": 1, - "days": 10, + "duration": 10, "cache": 16, "blocks": 8, "precision": "ms", diff --git a/tests/pytest/perfbenchmark/taosdemoInsert.py b/tests/pytest/perfbenchmark/taosdemoInsert.py index 59a8143d5a..37191b2624 100644 --- a/tests/pytest/perfbenchmark/taosdemoInsert.py +++ b/tests/pytest/perfbenchmark/taosdemoInsert.py @@ -75,7 +75,7 @@ class Taosdemo: "name": self.dbname, "drop": self.drop, "replica": 1, - "days": 10, + "duration": 10, "cache": 16, "blocks": 8, "precision": "ms", diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index a365369b21..c78324ff5c 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -57,7 +57,7 @@ class TDTestCase: "name": "db", "drop": "yes", "replica": 1, - "days": 10, + "duration": 10, "cache": 16, "blocks": 8, "precision": "ms", diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py index f069bb8f70..518f46b5e6 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestSupportNanoInsert.py @@ -138,7 +138,7 @@ class TDTestCase: sqls_ls = [ 'drop database if exists nsdbsql;', - 'create database nsdbsql precision "ns" keep 3600 days 6 update 1;', + 'create database nsdbsql precision "ns" keep 3600 duration 6 update 1;', 'use nsdbsql;', 'CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int);', 'CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2);', diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py index c3fdff00ec..7781a3d3d4 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py @@ -125,7 +125,7 @@ class TDTestCase: tdSql.checkData(0, 0, 600) # check taosdemo -s - sqls_ls = ['drop database if exists nsdbsql;','create database nsdbsql precision "ns" keep 36 days 6 update 1;', + sqls_ls = ['drop database if exists nsdbsql;','create database nsdbsql precision "ns" keep 36 duration 6 update 1;', 'use nsdbsql;','CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int);', 'CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2);', 'INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32);', diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py index 0dfc42f331..bc31b9fbcc 100644 --- a/tests/pytest/tools/taosdumpTest.py +++ b/tests/pytest/tools/taosdumpTest.py @@ -60,8 +60,8 @@ class TDTestCase: if not os.path.exists("./taosdumptest/tmp2"): os.makedirs("./taosdumptest/tmp2") tdSql.execute("drop database if exists db") - tdSql.execute("create database db days 11 keep 3649 blocks 8 ") - tdSql.execute("create database db1 days 12 keep 3640 blocks 7 ") + tdSql.execute("create database db duration 11 keep 3649 blocks 8 ") + tdSql.execute("create database db1 duration 12 keep 3640 blocks 7 ") tdSql.execute("use db") tdSql.execute( "create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") @@ -102,7 +102,7 @@ class TDTestCase: tdSql.query("show databases") tdSql.checkRows(2) dbresult = tdSql.queryResult - # 6--days,7--keep0,keep1,keep, 12--block, + # 6--duration,7--keep0,keep1,keep, 12--block, isCommunity = self.checkCommunity() print("iscommunity: %d" % isCommunity) diff --git a/tests/pytest/tools/taosdumpTestNanoSupport.py b/tests/pytest/tools/taosdumpTestNanoSupport.py index 55f1671daa..0101a65407 100644 --- a/tests/pytest/tools/taosdumpTestNanoSupport.py +++ b/tests/pytest/tools/taosdumpTestNanoSupport.py @@ -60,7 +60,7 @@ class TDTestCase: def build_db(precision, start_time): tdSql.execute("drop database if exists timedb1") tdSql.execute( - "create database timedb1 days 10 keep 365 blocks 8 precision "+"\""+precision+"\"") + "create database timedb1 duration 10 keep 365 blocks 8 precision "+"\""+precision+"\"") tdSql.execute("use timedb1") tdSql.execute( diff --git a/tests/pytest/update/merge_commit_data-0.py b/tests/pytest/update/merge_commit_data-0.py index 14d435f7f2..6f7951b16a 100644 --- a/tests/pytest/update/merge_commit_data-0.py +++ b/tests/pytest/update/merge_commit_data-0.py @@ -30,7 +30,7 @@ class TDTestCase: tdSql.execute(s) s = 'drop database if exists db' tdSql.execute(s) - s = 'create database db days 30' + s = 'create database db duration 30' tdSql.execute(s) s = 'use db' tdSql.execute(s) diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py index 4fb6765361..0b18071179 100644 --- a/tests/pytest/update/merge_commit_data.py +++ b/tests/pytest/update/merge_commit_data.py @@ -30,7 +30,7 @@ class TDTestCase: tdSql.execute(s) s = 'drop database if exists db' tdSql.execute(s) - s = 'create database db update 1 days 30' + s = 'create database db update 1 duration 30' tdSql.execute(s) s = 'use db' tdSql.execute(s) diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py index a334f39e86..034d9b9913 100644 --- a/tests/pytest/update/merge_commit_data2.py +++ b/tests/pytest/update/merge_commit_data2.py @@ -50,7 +50,7 @@ class TDTestCase: tdSql.execute(sql) sql = 'drop database if exists db' tdSql.execute(sql) - sql = 'create database db update 1 days 30;' + sql = 'create database db update 1 duration 30;' tdSql.execute(sql) sql = 'use db;' tdSql.execute(sql) diff --git a/tests/pytest/update/merge_commit_data2_update0.py b/tests/pytest/update/merge_commit_data2_update0.py index def50e0466..824f085630 100644 --- a/tests/pytest/update/merge_commit_data2_update0.py +++ b/tests/pytest/update/merge_commit_data2_update0.py @@ -49,7 +49,7 @@ class TDTestCase: tdSql.execute(sql) sql = 'drop database if exists db' tdSql.execute(sql) - sql = 'create database db update 0 days 30;' + sql = 'create database db update 0 duration 30;' tdSql.execute(sql) sql = 'use db;' tdSql.execute(sql) diff --git a/tests/pytest/update/merge_commit_last.py b/tests/pytest/update/merge_commit_last.py index 183cca0a1e..2e268e4e9e 100644 --- a/tests/pytest/update/merge_commit_last.py +++ b/tests/pytest/update/merge_commit_last.py @@ -34,7 +34,7 @@ class TDTestCase: def run(self): tdSql.prepare() - tdSql.execute("create database udb update 1 days 30") + tdSql.execute("create database udb update 1 duration 30") tdSql.execute("use udb") print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY") diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index bdda7c453b..79983f97d0 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -57,7 +57,7 @@ class TDSql: tdLog.notice("'reset query cache' is not supported") s = 'drop database if exists db' self.cursor.execute(s) - s = 'create database db days 300' + s = 'create database db duration 300' self.cursor.execute(s) s = 'use db' self.cursor.execute(s) diff --git a/tests/pytest/util/taosdemoCfg.py b/tests/pytest/util/taosdemoCfg.py index d211f86b81..7523a80898 100644 --- a/tests/pytest/util/taosdemoCfg.py +++ b/tests/pytest/util/taosdemoCfg.py @@ -63,7 +63,7 @@ class TDTaosdemoCfg: "name": 'db', "drop": 'yes', "replica": 1, - "days": 10, + "duration": 10, "cache": 16, "blocks": 6, "precision": "ms", diff --git a/tests/script/general/db/alter_option.sim b/tests/script/general/db/alter_option.sim index 36f4c0e7dc..89a32b5a5c 100644 --- a/tests/script/general/db/alter_option.sim +++ b/tests/script/general/db/alter_option.sim @@ -9,7 +9,7 @@ sleep 2000 sql connect print ============= create database -sql create database db cache 2 blocks 4 days 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1 +sql create database db cache 2 blocks 4 duration 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1 sql show databases if $data00 != db then return -1 @@ -87,13 +87,13 @@ sql_error alter database db quorum 4 sql_error alter database db quorum 5 sql_error alter database db quorum -1 -print ============== step days -sql_error alter database db days 0 -sql_error alter database db days 1 -sql_error alter database db days 2 -sql_error alter database db days 10 -sql_error alter database db days 50 -sql_error alter database db days 100 +print ============== step duration +sql_error alter database db duration 0 +sql_error alter database db duration 1 +sql_error alter database db duration 2 +sql_error alter database db duration 10 +sql_error alter database db duration 50 +sql_error alter database db duration 100 print ============== step keep sql show databases diff --git a/tests/script/general/db/backup/keep.sim b/tests/script/general/db/backup/keep.sim index 4d157b3985..626040e347 100644 --- a/tests/script/general/db/backup/keep.sim +++ b/tests/script/general/db/backup/keep.sim @@ -26,7 +26,7 @@ system sh/exec.sh -n dnode2 -s start sleep 2000 print ======== step1 create db -sql create database keepdb replica 1 keep 30 days 7 +sql create database keepdb replica 1 keep 30 duration 7 sql use keepdb sql create table tb (ts timestamp, i int) @@ -201,7 +201,7 @@ sql alter database keepdb keep 0 -x error2 return -1 error2: -sql alter database keepdb days 1 -x error3 +sql alter database keepdb duration 1 -x error3 return -1 error3: diff --git a/tests/script/general/db/topic1.sim b/tests/script/general/db/topic1.sim index 1639973120..b5058e8f9b 100644 --- a/tests/script/general/db/topic1.sim +++ b/tests/script/general/db/topic1.sim @@ -171,7 +171,7 @@ sql_error create topic t1 partitions -1; sql_error create topic t1 partitions 10001; print =============step3 create with db para -sql create topic db cache 2 blocks 4 days 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1 +sql create topic db cache 2 blocks 4 duration 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1 sql show databases if $data00 != db then return -1 @@ -199,7 +199,7 @@ if $data09 != 4 then endi sql drop topic db; -sql create topic db cache 2 blocks 4 days 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1 partitions 7 +sql create topic db cache 2 blocks 4 duration 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1 partitions 7 sql show databases if $data00 != db then return -1 @@ -334,19 +334,19 @@ sql_error alter topic db quorum 4 sql_error alter topic db quorum 5 sql_error alter topic db quorum -1 -print ============== step days -sql_error alter database db days 0 -sql_error alter database db days 1 -sql_error alter database db days 2 -sql_error alter database db days 10 -sql_error alter database db days 50 -sql_error alter database db days 100 -sql_error alter topic db days 0 -sql_error alter topic db days 1 -sql_error alter topic db days 2 -sql_error alter topic db days 10 -sql_error alter topic db days 50 -sql_error alter topic db days 100 +print ============== step duration +sql_error alter database db duration 0 +sql_error alter database db duration 1 +sql_error alter database db duration 2 +sql_error alter database db duration 10 +sql_error alter database db duration 50 +sql_error alter database db duration 100 +sql_error alter topic db duration 0 +sql_error alter topic db duration 1 +sql_error alter topic db duration 2 +sql_error alter topic db duration 10 +sql_error alter topic db duration 50 +sql_error alter topic db duration 100 print ============== step keep sql show databases diff --git a/tests/script/general/import/commit.sim b/tests/script/general/import/commit.sim index 3b4055d712..aefc724fdb 100644 --- a/tests/script/general/import/commit.sim +++ b/tests/script/general/import/commit.sim @@ -30,7 +30,7 @@ sleep 2000 sql connect print ========= step1 -sql create database ic1db days 7; +sql create database ic1db duration 7; sql create table ic1db.tb(ts timestamp, s int); sql insert into ic1db.tb values(now-30d, -30); sql insert into ic1db.tb values(now-20d, -20); @@ -50,7 +50,7 @@ if $rows != 12 then endi print ========= step2 -sql create database ic2db days 7; +sql create database ic2db duration 7; sql create table ic2db.tb(ts timestamp, s int); sql insert into ic2db.tb values(now, 0); sql import into ic2db.tb values(now-30d, -30); diff --git a/tests/script/general/import/replica1.sim b/tests/script/general/import/replica1.sim index 48d5455b79..1e8eabb798 100644 --- a/tests/script/general/import/replica1.sim +++ b/tests/script/general/import/replica1.sim @@ -30,7 +30,7 @@ system sh/exec.sh -n dnode1 -s start sleep 2000 sql connect -sql create database ir1db days 7 +sql create database ir1db duration 7 sql use ir1db sql create table tb(ts timestamp, i bigint) diff --git a/tests/script/general/parser/alter.sim b/tests/script/general/parser/alter.sim index d1a4702a69..78c1a3029a 100644 --- a/tests/script/general/parser/alter.sim +++ b/tests/script/general/parser/alter.sim @@ -20,7 +20,7 @@ $db = $dbPrefix . $i $mt = $mtPrefix . $i sql drop database if exists $db -sql create database $db days 10 keep 20,20,20 +sql create database $db duration 10 keep 20,20,20 sql use $db sql_error alter database $db keep "20" diff --git a/tests/script/general/parser/alter__for_community_version.sim b/tests/script/general/parser/alter__for_community_version.sim index f55fb812a7..7a9a970822 100644 --- a/tests/script/general/parser/alter__for_community_version.sim +++ b/tests/script/general/parser/alter__for_community_version.sim @@ -20,7 +20,7 @@ $db = $dbPrefix . $i $mt = $mtPrefix . $i sql drop database if exists $db -sql create database $db days 10 keep 20 +sql create database $db duration 10 keep 20 sql use $db sql show databases if $rows != 1 then diff --git a/tests/script/general/parser/create_db.sim b/tests/script/general/parser/create_db.sim index a62a45e023..040331ec4f 100644 --- a/tests/script/general/parser/create_db.sim +++ b/tests/script/general/parser/create_db.sim @@ -101,7 +101,7 @@ print db_already_exists test passed print create_db.sim case5: db_meta_data test # cfg params $replica = 1 # max=3 -$days = 10 +$duration = 10 $keep = 365,365,365 $rows_db = 1000 $cache = 16 # 16MB @@ -111,7 +111,7 @@ $ctime = 36000 # 10 hours $wal = 1 # valid value is 1, 2 $comp = 1 # max=32, automatically trimmed when exceeding -sql create database $db replica $replica days $days keep $keep maxrows $rows_db cache $cache blocks 4 ctime $ctime wal $wal comp $comp +sql create database $db replica $replica duration $duration keep $keep maxrows $rows_db cache $cache blocks 4 ctime $ctime wal $wal comp $comp sql show databases if $rows != 1 then return -1 @@ -122,7 +122,7 @@ endi if $data04 != $replica then return -1 endi -if $data06 != $days then +if $data06 != $duration then return -1 endi if $data07 != 365,365,365 then diff --git a/tests/script/general/parser/create_db__for_community_version.sim b/tests/script/general/parser/create_db__for_community_version.sim index 406e69b0e6..5dc4263d5d 100644 --- a/tests/script/general/parser/create_db__for_community_version.sim +++ b/tests/script/general/parser/create_db__for_community_version.sim @@ -101,7 +101,7 @@ print db_already_exists test passed print create_db.sim case5: db_meta_data test # cfg params $replica = 1 # max=3 -$days = 10 +$duration = 10 $keep = 365 $rows_db = 1000 $cache = 16 # 16MB @@ -111,7 +111,7 @@ $ctime = 36000 # 10 hours $wal = 1 # valid value is 1, 2 $comp = 1 # max=32, automatically trimmed when exceeding -sql create database $db replica $replica days $days keep $keep maxrows $rows_db cache $cache blocks 4 ctime $ctime wal $wal comp $comp +sql create database $db replica $replica duration $duration keep $keep maxrows $rows_db cache $cache blocks 4 ctime $ctime wal $wal comp $comp sql show databases if $rows != 1 then return -1 @@ -122,7 +122,7 @@ endi if $data04 != $replica then return -1 endi -if $data06 != $days then +if $data06 != $duration then return -1 endi if $data07 != 365 then diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 4692b9d91b..4351ee5cb1 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -62,7 +62,7 @@ print ============= create database # | PAGES value [64~16384, default: 256] # | CACHELAST value [0, 1, 2, 3] # | FSYNC value [0 ~ 180000 ms] -# | KEEP value [days, 365000] +# | KEEP value [duration, 365000] # | REPLICA value [1 | 3] # | WAL value [1 | 2] @@ -92,7 +92,7 @@ endi if $data5_db != no_strict then # strict return -1 endi -if $data6_db != 345600 then # days +if $data6_db != 345600 then # duration return -1 endi if $data7_db != 1440000m,1440000m,1440000m then # keep @@ -222,11 +222,11 @@ sql_error alter database db replica 0 #sql_error alter database db quorum 4 #sql_error alter database db quorum 5 -#print ============== modify days -sql_error alter database db days 480 -sql_error alter database db days 360 -sql_error alter database db days 0 -sql_error alter database db days 14400 # set over than keep +#print ============== modify duration +sql_error alter database db duration 480 +sql_error alter database db duration 360 +sql_error alter database db duration 0 +sql_error alter database db duration 14400 # set over than keep print ============== modify keep sql alter database db keep 2400 diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 142460f214..64103b5dac 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -15,7 +15,7 @@ $tb = $tbPrefix . $i print =============== step1 # quorum presicion -sql create database $db vgroups 8 replica 1 days 2 keep 10 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us' +sql create database $db vgroups 8 replica 1 duration 2 keep 10 minrows 80 maxrows 10000 wal 2 fsync 1000 comp 0 cachelast 2 precision 'us' sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -66,7 +66,7 @@ print =============== step4 sql_error drop database $db print =============== step5 -sql create database $db replica 1 days 15 keep 1500 +sql create database $db replica 1 duration 15 keep 1500 sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data20 != $db then diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 03068698e2..284875ee08 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -79,7 +79,7 @@ print ============= create database with all options #$data2_db : vgroups #$data3_db : ntables #$data4_db : replica -#$data6_db : days +#$data6_db : duration #$data7_db : keep #$data10_db : minrows #$data11_db : maxrows @@ -113,7 +113,7 @@ endi if $data5_db != no_strict then # strict return -1 endi -if $data6_db != 14400 then # days +if $data6_db != 14400 then # duration return -1 endi if $data7_db != 5256000m,5256000m,5256000m then # keep diff --git a/tests/script/tsim/insert/commit-merge0.sim b/tests/script/tsim/insert/commit-merge0.sim index adbd1904b2..56e818654f 100644 --- a/tests/script/tsim/insert/commit-merge0.sim +++ b/tests/script/tsim/insert/commit-merge0.sim @@ -5,7 +5,7 @@ sleep 50 sql connect print =============== create database -sql create database db days 300 keep 365000d,365000d,365000d +sql create database db duration 300 keep 365000d,365000d,365000d sql show databases if $rows != 3 then return -1 diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index dcd88e5a0c..ab6ee79d6e 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -5,7 +5,7 @@ sleep 500 sql connect print =============== create database -sql create database d0 days 300 +sql create database d0 duration 300 sql use d0 print =============== create super table and child table diff --git a/tests/script/tsim/stable/alter_count.sim b/tests/script/tsim/stable/alter_count.sim index e5af9a5735..eca8ca1559 100644 --- a/tests/script/tsim/stable/alter_count.sim +++ b/tests/script/tsim/stable/alter_count.sim @@ -6,7 +6,7 @@ system sh/exec.sh -n dnode1 -s start sql connect print ======== step1 -sql create database d1 replica 1 days 7 keep 50 +sql create database d1 replica 1 duration 7 keep 50 sql use d1 sql create table tb (ts timestamp, a int) sql insert into tb values(now-28d, -28) diff --git a/tests/script/tsim/stable/alter_import.sim b/tests/script/tsim/stable/alter_import.sim index cdd7b60e14..b968eb6a12 100644 --- a/tests/script/tsim/stable/alter_import.sim +++ b/tests/script/tsim/stable/alter_import.sim @@ -6,7 +6,7 @@ system sh/exec.sh -n dnode1 -s start sql connect print ======== step1 -sql create database d1 replica 1 days 7 keep 50 +sql create database d1 replica 1 duration 7 keep 50 sql use d1 sql create table tb (ts timestamp, a int) sql insert into tb values(now-30d, -28) diff --git a/tests/script/unique/db/commit.sim b/tests/script/unique/db/commit.sim index 661dd4505f..74c1366afb 100644 --- a/tests/script/unique/db/commit.sim +++ b/tests/script/unique/db/commit.sim @@ -24,7 +24,7 @@ system sh/exec.sh -n dnode2 -s start sleep 2000 print ======== step1 create db -sql create database commitdb replica 1 days 7 keep 30 +sql create database commitdb replica 1 duration 7 keep 30 sql use commitdb sql create table tb (ts timestamp, i int) diff --git a/tests/script/unique/import/replica2.sim b/tests/script/unique/import/replica2.sim index 54ce28543e..387567fc82 100644 --- a/tests/script/unique/import/replica2.sim +++ b/tests/script/unique/import/replica2.sim @@ -52,7 +52,7 @@ if $data4_2 != ready then goto step1 endi -sql create database ir2db replica 2 days 7 +sql create database ir2db replica 2 duration 7 sql use ir2db sql create table tb(ts timestamp, i bigint) diff --git a/tests/script/unique/import/replica3.sim b/tests/script/unique/import/replica3.sim index 5da9e141a5..5c1b8b8932 100644 --- a/tests/script/unique/import/replica3.sim +++ b/tests/script/unique/import/replica3.sim @@ -58,7 +58,7 @@ if $data4_3 != ready then goto step1 endi -sql create database ir3db replica 3 days 7 +sql create database ir3db replica 3 duration 7 sql use ir3db sql create table tb(ts timestamp, i bigint) diff --git a/tests/script/windows/db/basic.sim b/tests/script/windows/db/basic.sim index 914e456fe1..1f0fc31a6b 100644 --- a/tests/script/windows/db/basic.sim +++ b/tests/script/windows/db/basic.sim @@ -17,7 +17,7 @@ $db = $dbPrefix . $i $tb = $tbPrefix . $i print =============== step1 -sql create database $db replica 1 days 20 keep 2000 cache 16 +sql create database $db replica 1 duration 20 keep 2000 cache 16 sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data00 != $db then @@ -57,7 +57,7 @@ print =============== step4 sql_error drop database $db print =============== step5 -sql create database $db replica 1 days 15 keep 1500 +sql create database $db replica 1 duration 15 keep 1500 sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data00 != $db then diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index 57ae1c1490..9b145e9093 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -60,7 +60,7 @@ class TDTestCase: def prepare_data(self): tdSql.execute("drop database if exists db ") - tdSql.execute("create database if not exists db days 300") + tdSql.execute("create database if not exists db duration 300") tdSql.execute("use db") tdSql.execute( '''create table stb1 diff --git a/tests/system-test/0-others/udf_cluster.py b/tests/system-test/0-others/udf_cluster.py index c5c0c7b8f2..9ef3137a7e 100644 --- a/tests/system-test/0-others/udf_cluster.py +++ b/tests/system-test/0-others/udf_cluster.py @@ -63,7 +63,7 @@ class TDTestCase: def prepare_data(self): tdSql.execute("drop database if exists db") - tdSql.execute("create database if not exists db replica 1 days 300") + tdSql.execute("create database if not exists db replica 1 duration 300") tdSql.execute("use db") tdSql.execute( '''create table stb1 diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 170d9b1421..11ad8e1584 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -62,7 +62,7 @@ class TDTestCase: def prepare_data(self): tdSql.execute("drop database if exists db ") - tdSql.execute("create database if not exists db days 300") + tdSql.execute("create database if not exists db duration 300") tdSql.execute("use db") tdSql.execute( '''create table stb1 diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index 94b14f01ba..c9eb22cf15 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -59,7 +59,7 @@ class TDTestCase: def prepare_data(self): tdSql.execute("drop database if exists db ") - tdSql.execute("create database if not exists db days 300") + tdSql.execute("create database if not exists db duration 300") tdSql.execute("use db") tdSql.execute( '''create table stb1 diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index d9c37be996..d779cc26cd 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -68,7 +68,7 @@ class TDTestCase: def prepare_tag_datas(self): # prepare datas - tdSql.execute("create database if not exists testdb keep 3650 days 1000") + tdSql.execute("create database if not exists testdb keep 3650 duration 1000") tdSql.execute(" use testdb ") tdSql.execute( '''create table stb1 diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index 8c0cd83d4f..a84d93404a 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -628,7 +628,7 @@ class TDTestCase: def basic_sample_query(self): tdSql.execute(" drop database if exists db ") - tdSql.execute(" create database if not exists db days 300 ") + tdSql.execute(" create database if not exists db duration 300 ") tdSql.execute(" use db ") tdSql.execute( '''create table stb1 diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 7c3715cd0b..75134224db 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -104,7 +104,7 @@ class TDTestCase: tdSql.error("drop mnode on dnode 1;") tdSql.execute("drop database if exists db") - tdSql.execute("create database if not exists db replica 1 days 300") + tdSql.execute("create database if not exists db replica 1 duration 300") tdSql.execute("use db") tdSql.execute( '''create table stb1 diff --git a/tests/system-test/6-cluster/5dnode2mnode.py b/tests/system-test/6-cluster/5dnode2mnode.py index 9d9cc9c0d6..04d9c9b3b4 100644 --- a/tests/system-test/6-cluster/5dnode2mnode.py +++ b/tests/system-test/6-cluster/5dnode2mnode.py @@ -104,7 +104,7 @@ class TDTestCase: tdSql.error("drop mnode on dnode 1;") tdSql.execute("drop database if exists db") - tdSql.execute("create database if not exists db replica 1 days 300") + tdSql.execute("create database if not exists db replica 1 duration 300") tdSql.execute("use db") tdSql.execute( '''create table stb1 @@ -163,7 +163,7 @@ class TDTestCase: # fisrt add data : db\stable\childtable\general table tdSql.execute("drop database if exists db2") - tdSql.execute("create database if not exists db2 replica 1 days 300") + tdSql.execute("create database if not exists db2 replica 1 duration 300") tdSql.execute("use db2") tdSql.execute( '''create table stb1 diff --git a/tests/system-test/6-cluster/5dnode3mnodeDrop.py b/tests/system-test/6-cluster/5dnode3mnodeDrop.py index 1512fc9897..9c18c63b85 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeDrop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeDrop.py @@ -53,7 +53,7 @@ class TDTestCase: # fisrt add data : db\stable\childtable\general table for couti in count: tdSql.execute("drop database if exists db%d" %couti) - tdSql.execute("create database if not exists db%d replica 1 days 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) tdSql.execute("use db%d" %couti) tdSql.execute( '''create table stb1 diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index a9784f2d0f..08da7f9101 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -53,7 +53,7 @@ class TDTestCase: # fisrt add data : db\stable\childtable\general table for couti in count: tdSql.execute("drop database if exists db%d" %couti) - tdSql.execute("create database if not exists db%d replica 1 days 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) tdSql.execute("use db%d" %couti) tdSql.execute( '''create table stb1 diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py b/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py index 95cd26dedc..bdb6cf28e1 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py @@ -54,7 +54,7 @@ class TDTestCase: # fisrt add data : db\stable\childtable\general table for couti in count: tdSql.execute("drop database if exists db%d" %couti) - tdSql.execute("create database if not exists db%d replica 1 days 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) tdSql.execute("use db%d" %couti) tdSql.execute( '''create table stb1 From ac9917a39bcd3dbd560479d09e17cf52754e1795 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 15 Jun 2022 14:26:57 +0800 Subject: [PATCH 25/41] fix: adapt to different coredump configurations --- tests/parallel_test/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index 1b54dc338e..f6f09d2823 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -255,7 +255,7 @@ function run_thread() { $cmd # 2>/dev/null local case_info=`echo "$line"|cut -d, -f 3,4` local corefile=`ls $log_dir/${case_file}.coredump/` - corefile=`find $log_dir/${case_file}.coredump/ -name "core.*"` + corefile=`find $log_dir/${case_file}.coredump/ -name "*"` echo -e "$case_info \e[31m failed\e[0m" echo "=========================log============================" cat $log_dir/$case_file.log From 6d1ff4c18b719a58c88d30f0ccfc89a6bf7b544d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 15 Jun 2022 14:33:45 +0800 Subject: [PATCH 26/41] fix dead lock issue --- source/libs/catalog/src/ctgAsync.c | 15 +++++++++++---- source/libs/catalog/src/ctgDbg.c | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 6e24d632e2..8dabd56934 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -784,8 +784,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * _return: if (dbCache) { - ctgRUnlockVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } ctgHandleTaskEnd(pTask, code); @@ -943,7 +942,6 @@ _return: int32_t ctgHandleGetUserRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - SCtgDBCache *dbCache = NULL; SCtgUserCtx* ctx = (SCtgUserCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; bool pass = false; @@ -1012,7 +1010,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); @@ -1061,6 +1059,9 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgCache.vgInfo->vgHash, (SArray**)&pTask->res)); + + ctgReleaseVgInfoToCache(pCtg, dbCache); + dbCache = NULL; CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1095,6 +1096,9 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); + + ctgReleaseVgInfoToCache(pCtg, dbCache); + dbCache = NULL; CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1170,6 +1174,9 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask *pTask) { pInfo->vgVer = dbCache->vgCache.vgInfo->vgVersion; pInfo->dbId = dbCache->dbId; pInfo->tbNum = dbCache->vgCache.vgInfo->numOfTable; + + ctgReleaseVgInfoToCache(pCtg, dbCache); + dbCache = NULL; } else { pInfo->vgVer = CTG_DEFAULT_INVALID_VERSION; } diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 9de1ea22be..ff93bedb21 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {0}; +SCtgDebug gCTGDebug = {.lockEnable = true, .apiEnable = true}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); From cc513ff36b7ce9f092613144e7952b2d8c0a3d70 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Jun 2022 14:33:53 +0800 Subject: [PATCH 27/41] test: enable test case --- tests/system-test/fulltest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 24a57b13e6..676f2da788 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -2,7 +2,7 @@ set -e set -x -#python3 ./test.py -f 0-others/taosShell.py +python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py python3 ./test.py -f 0-others/taosShellNetChk.py python3 ./test.py -f 0-others/telemetry.py @@ -105,7 +105,7 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -#python3 ./test.py -f 7-tmq/basic5.py +python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py python3 ./test.py -f 7-tmq/subscribeDb0.py python3 ./test.py -f 7-tmq/subscribeDb1.py From 5ea1f3512fcfc907a0a0e5f23b313ab980d8de9a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Jun 2022 14:43:30 +0800 Subject: [PATCH 28/41] feat: increase enable and sysinfo limits for user privilege --- include/common/tmsg.h | 9 +++++++ source/common/src/systable.c | 4 ++- source/common/src/tmsg.c | 14 ++++++++++ source/dnode/mnode/impl/inc/mndDef.h | 3 +++ source/dnode/mnode/impl/src/mndUser.c | 36 +++++++++++++++++++++----- source/libs/parser/src/parTranslater.c | 2 ++ 6 files changed, 60 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index fb32cb382a..6474147eca 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -135,6 +135,8 @@ typedef enum _mgmt_table { #define TSDB_ALTER_USER_REMOVE_WRITE_DB 0x6 #define TSDB_ALTER_USER_ADD_ALL_DB 0x7 #define TSDB_ALTER_USER_REMOVE_ALL_DB 0x8 +#define TSDB_ALTER_USER_ENABLE 0x9 +#define TSDB_ALTER_USER_SYSINFO 0xA #define TSDB_ALTER_USER_PRIVILEGES 0x2 @@ -534,6 +536,8 @@ int32_t tDeserializeSDropUserReq(void* buf, int32_t bufLen, SDropUserReq* pReq); typedef struct { int8_t createType; int8_t superUser; // denote if it is a super user or not + int8_t sysInfo; + int8_t enable; char user[TSDB_USER_LEN]; char pass[TSDB_USET_PASSWORD_LEN]; } SCreateUserReq; @@ -544,6 +548,8 @@ int32_t tDeserializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pR typedef struct { int8_t alterType; int8_t superUser; + int8_t sysInfo; + int8_t enable; char user[TSDB_USER_LEN]; char pass[TSDB_USET_PASSWORD_LEN]; char dbname[TSDB_DB_FNAME_LEN]; @@ -563,6 +569,9 @@ typedef struct { char user[TSDB_USER_LEN]; int32_t version; int8_t superAuth; + int8_t sysInfo; + int8_t enable; + int8_t reserve; SHashObj* createdDbs; SHashObj* readDbs; SHashObj* writeDbs; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 08977abd61..12afb4999b 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -170,7 +170,9 @@ static const SSysDbTableSchema userTblDistSchema[] = { static const SSysDbTableSchema userUsersSchema[] = { {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 2f7ca249ef..0ba4c0056b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1160,6 +1160,8 @@ int32_t tSerializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pReq if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI8(&encoder, pReq->createType) < 0) return -1; if (tEncodeI8(&encoder, pReq->superUser) < 0) return -1; + if (tEncodeI8(&encoder, pReq->sysInfo) < 0) return -1; + if (tEncodeI8(&encoder, pReq->enable) < 0) return -1; if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; tEndEncode(&encoder); @@ -1176,6 +1178,8 @@ int32_t tDeserializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI8(&decoder, &pReq->createType) < 0) return -1; if (tDecodeI8(&decoder, &pReq->superUser) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->sysInfo) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->enable) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; tEndDecode(&decoder); @@ -1191,6 +1195,8 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq) if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1; if (tEncodeI8(&encoder, pReq->superUser) < 0) return -1; + if (tEncodeI8(&encoder, pReq->sysInfo) < 0) return -1; + if (tEncodeI8(&encoder, pReq->enable) < 0) return -1; if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; if (tEncodeCStr(&encoder, pReq->dbname) < 0) return -1; @@ -1208,6 +1214,8 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI8(&decoder, &pReq->alterType) < 0) return -1; if (tDecodeI8(&decoder, &pReq->superUser) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->sysInfo) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->enable) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->dbname) < 0) return -1; @@ -1245,6 +1253,9 @@ int32_t tDeserializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq * int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) { if (tEncodeCStr(pEncoder, pRsp->user) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->superAuth) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->sysInfo) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->enable) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->reserve) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->version) < 0) return -1; int32_t numOfCreatedDbs = taosHashGetSize(pRsp->createdDbs); @@ -1300,6 +1311,9 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs if (tDecodeCStrTo(pDecoder, pRsp->user) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->superAuth) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->sysInfo) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->enable) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->reserve) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->version) < 0) return -1; int32_t numOfCreatedDbs = 0; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 6dd5d1622c..0e0846ef84 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -226,6 +226,9 @@ typedef struct { int64_t createdTime; int64_t updateTime; int8_t superUser; + int8_t sysInfo; + int8_t enable; + int8_t reserve; int32_t acctId; int32_t authVersion; SHashObj* readDbs; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 345d756f43..a86b9c904e 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -68,6 +68,8 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char tstrncpy(userObj.acct, acct, TSDB_USER_LEN); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; + userObj.sysInfo = 1; + userObj.enable = 1; if (strcmp(user, TSDB_DEFAULT_USER) == 0) { userObj.superUser = 1; @@ -128,6 +130,9 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER) + SDB_SET_INT8(pRaw, dataPos, pUser->sysInfo, _OVER) + SDB_SET_INT8(pRaw, dataPos, pUser->enable, _OVER) + SDB_SET_INT8(pRaw, dataPos, pUser->reserve, _OVER) SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER) @@ -184,6 +189,9 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pUser->sysInfo, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pUser->enable, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pUser->reserve, _OVER) SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER) int32_t numOfReadDbs = 0; @@ -256,6 +264,8 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { taosWLockLatch(&pOld->lock); pOld->updateTime = pNew->updateTime; pOld->authVersion = pNew->authVersion; + pOld->sysInfo = pNew->sysInfo; + pOld->enable = pNew->enable; memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); TSWAP(pOld->readDbs, pNew->readDbs); TSWAP(pOld->writeDbs, pNew->writeDbs); @@ -286,6 +296,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; userObj.superUser = pCreate->superUser; + userObj.sysInfo = pCreate->sysInfo; + userObj.enable = pCreate->enable; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { @@ -481,6 +493,14 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { newUser.superUser = alterReq.superUser; } + if (alterReq.alterType == TSDB_ALTER_USER_ENABLE) { + newUser.enable = alterReq.enable; + } + + if (alterReq.alterType == TSDB_ALTER_USER_SYSINFO) { + newUser.sysInfo = alterReq.sysInfo; + } + if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) { if (strcmp(alterReq.dbname, "1.*") != 0) { int32_t len = strlen(alterReq.dbname) + 1; @@ -740,19 +760,21 @@ static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - - char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->pMeta->pSchemas[cols].bytes); - colDataAppend(pColInfo, numOfRows, (const char *)name, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + colDataAppend(pColInfo, numOfRows, (const char *)&pUser->superUser, false); - const char *src = pUser->superUser ? "super" : "normal"; - char b[10 + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src)); - colDataAppend(pColInfo, numOfRows, (const char *)b, false); + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + colDataAppend(pColInfo, numOfRows, (const char *)&pUser->enable, false); + + cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + colDataAppend(pColInfo, numOfRows, (const char *)&pUser->sysInfo, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 52abeadac9..389a96dfb7 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3253,6 +3253,8 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt strcpy(createReq.user, pStmt->useName); createReq.createType = 0; createReq.superUser = 0; + createReq.sysInfo = 1; + createReq.enable = 1; strcpy(createReq.pass, pStmt->password); return buildCmdMsg(pCxt, TDMT_MND_CREATE_USER, (FSerializeFunc)tSerializeSCreateUserReq, &createReq); From 84e6f48bdc71f6f2ff51b49b647e785690645504 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Wed, 15 Jun 2022 14:55:02 +0800 Subject: [PATCH 29/41] feat(taosAdapter): taosAdapter for 3.0 (#13856) --- .gitmodules | 3 ++ cmake/cmake.define | 27 +++++++++++++ tools/CMakeLists.txt | 92 ++++++++++++++++++++++++++++++++++++++++++++ tools/taosadapter | 1 + 4 files changed, 123 insertions(+) create mode 160000 tools/taosadapter diff --git a/.gitmodules b/.gitmodules index bc38453f19..34a2da2894 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "tools/taos-tools"] path = tools/taos-tools url = https://github.com/taosdata/taos-tools +[submodule "tools/taosadapter"] + path = tools/taosadapter + url = https://github.com/taosdata/taosadapter.git diff --git a/cmake/cmake.define b/cmake/cmake.define index 8d71870e7d..f58c1ad354 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -18,6 +18,33 @@ if (NOT DEFINED TD_GRANT) SET(TD_GRANT FALSE) endif() +IF ("${BUILD_HTTP}" STREQUAL "") + IF (TD_LINUX) + IF (TD_ARM_32) + SET(TD_BUILD_HTTP TRUE) + ELSE () + SET(TD_BUILD_HTTP TRUE) + ENDIF () + ELSEIF (TD_DARWIN) + SET(TD_BUILD_HTTP TRUE) + ELSE () + SET(TD_BUILD_HTTP TRUE) + ENDIF () +ELSEIF (${BUILD_HTTP} MATCHES "false") + SET(TD_BUILD_HTTP FALSE) +ELSEIF (${BUILD_HTTP} MATCHES "true") + SET(TD_BUILD_HTTP TRUE) +ELSEIF (${BUILD_HTTP} MATCHES "internal") + SET(TD_BUILD_HTTP FALSE) + SET(TD_BUILD_TAOSA_INTERNAL TRUE) +ELSE () + SET(TD_BUILD_HTTP TRUE) +ENDIF () + +IF (TD_BUILD_HTTP) + ADD_DEFINITIONS(-DHTTP_EMBEDDED) +ENDIF () + IF ("${BUILD_TOOLS}" STREQUAL "") IF (TD_LINUX) IF (TD_ARM_32) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index afd3b36f2d..da1afe84a7 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -9,3 +9,95 @@ IF (TD_TAOS_TOOLS) ENDIF () add_subdirectory(shell) +IF (TD_BUILD_HTTP) + MESSAGE("") + MESSAGE("${Yellow} use original embedded httpd ${ColourReset}") + MESSAGE("") + # ADD_SUBDIRECTORY(http) +ELSEIF(TD_BUILD_TAOSA_INTERNAL) + MESSAGE("${Yellow} use taosa internal as httpd ${ColourReset}") +ELSE () + MESSAGE("") + MESSAGE("${Green} use taosadapter as httpd, platform is ${PLATFORM_ARCH_STR} ${ColourReset}") + + EXECUTE_PROCESS( + COMMAND git rev-parse --abbrev-ref HEAD + RESULT_VARIABLE result_taos_version + OUTPUT_VARIABLE taos_version + ) + + STRING(FIND ${taos_version} release is_release_branch) + + IF ("${is_release_branch}" STREQUAL "0") + STRING(SUBSTRING "${taos_version}" 12 -1 taos_version) + STRING(STRIP "${taos_version}" taos_version) + ELSE () + STRING(CONCAT taos_version "_branch_" "${taos_version}") + STRING(STRIP "${taos_version}" taos_version) + ENDIF () + EXECUTE_PROCESS( + COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR}/taosadapter + ) + EXECUTE_PROCESS( + COMMAND git rev-parse --short HEAD + RESULT_VARIABLE commit_sha1 + OUTPUT_VARIABLE taosadapter_commit_sha1 + ) + IF ("${taosadapter_commit_sha1}" STREQUAL "") + SET(taosadapter_commit_sha1 "unknown") + ELSE () + STRING(SUBSTRING "${taosadapter_commit_sha1}" 0 7 taosadapter_commit_sha1) + STRING(STRIP "${taosadapter_commit_sha1}" taosadapter_commit_sha1) + ENDIF () + MESSAGE("${Green} taosAdapter will use ${taos_version} and commit ${taosadapter_commit_sha1} as version ${ColourReset}") + EXECUTE_PROCESS( + COMMAND cd .. + ) + MESSAGE("CURRENT SOURCE DIR ${CMAKE_CURRENT_SOURCE_DIR}") + IF (TD_LINUX) + include(ExternalProject) + ExternalProject_Add(taosadapter + PREFIX "taosadapter" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosadapter + BUILD_ALWAYS off + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosadapter no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" + INSTALL_COMMAND + COMMAND curl -sL https://github.com/upx/upx/releases/download/v3.96/upx-3.96-${PLATFORM_ARCH_STR}_linux.tar.xz -o upx.tar.xz && tar -xvJf upx.tar.xz -C ${CMAKE_BINARY_DIR} --strip-components 1 > /dev/null && ${CMAKE_BINARY_DIR}/upx taosadapter || : + COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin + ) + ELSEIF (TD_DARWIN) + include(ExternalProject) + ExternalProject_Add(taosadapter + PREFIX "taosadapter" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosadapter + BUILD_ALWAYS off + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosadapter no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" + INSTALL_COMMAND + COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ + COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin + ) + ELSE () + MESSAGE("${Yellow} Windows system still use original embedded httpd ${ColourReset}") + ENDIF () +ENDIF () diff --git a/tools/taosadapter b/tools/taosadapter new file mode 160000 index 0000000000..9ce3f5c98e --- /dev/null +++ b/tools/taosadapter @@ -0,0 +1 @@ +Subproject commit 9ce3f5c98ef95d9c7c596c4ed7302b0ed69a92b2 From 055d2dd15551fe86c4fbff008d38cc7823a67ece Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Wed, 15 Jun 2022 15:06:12 +0800 Subject: [PATCH 30/41] fix: incorrect coredump file check --- tests/parallel_test/run.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index f6f09d2823..e9871637bd 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -255,7 +255,6 @@ function run_thread() { $cmd # 2>/dev/null local case_info=`echo "$line"|cut -d, -f 3,4` local corefile=`ls $log_dir/${case_file}.coredump/` - corefile=`find $log_dir/${case_file}.coredump/ -name "*"` echo -e "$case_info \e[31m failed\e[0m" echo "=========================log============================" cat $log_dir/$case_file.log From 11564a3dc79edfea55be0c1a2c7a7b790f0b716c Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 15 Jun 2022 15:24:22 +0800 Subject: [PATCH 31/41] fix: use 'duration' instead of 'days' for create database --- source/libs/parser/src/parTranslater.c | 1 - source/libs/planner/src/planPhysiCreater.c | 3 ++- source/libs/planner/src/planSpliter.c | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bfb58d2cd1..9e0de48420 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4797,7 +4797,6 @@ static int32_t buildDropTableVgroupHashmap(STranslateContext* pCxt, SDropTableCl if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code && pClause->ignoreNotExists) { code = TSDB_CODE_SUCCESS; - goto over; } *pIsSuperTable = false; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 618ff9acc6..4a0348151b 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -470,7 +470,8 @@ static ENodeType getScanOperatorType(EScanType scanType) { case SCAN_TYPE_STREAM: return QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; case SCAN_TYPE_TABLE_MERGE: - return QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; + return QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; + // return QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; default: break; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 4c202d457f..82f6b7fe14 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -392,6 +392,7 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo (SNode*)splCreateScanSubplan(pCxt, pPartWindow, SPLIT_FLAG_STABLE_SPLIT)); } pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + ++(pCxt->groupId); return code; } @@ -408,6 +409,7 @@ static int32_t stbSplSplitIntervalForStream(SSplitContext* pCxt, SStableSplitInf (SNode*)splCreateScanSubplan(pCxt, pPartWindow, SPLIT_FLAG_STABLE_SPLIT)); } pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + ++(pCxt->groupId); return code; } @@ -496,6 +498,7 @@ static int32_t stbSplSplitAggNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) (SNode*)splCreateScanSubplan(pCxt, pPartAgg, SPLIT_FLAG_STABLE_SPLIT)); } pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + ++(pCxt->groupId); return code; } @@ -610,6 +613,7 @@ static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) (SNode*)splCreateScanSubplan(pCxt, pPartSort, SPLIT_FLAG_STABLE_SPLIT)); } pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + ++(pCxt->groupId); return code; } @@ -619,6 +623,7 @@ static int32_t stbSplSplitScanNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); } + ++(pCxt->groupId); return code; } From 4fc7e0aae98b75c21fe915847e2a22269f4c38e1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 15 Jun 2022 15:31:58 +0800 Subject: [PATCH 32/41] fix jsons filter error --- tests/system-test/2-query/json_tag.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 54b41e5a94..b6890a8503 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -243,7 +243,6 @@ class TDTestCase: tdSql.checkRows(2) tdSql.query("select * from jsons1 where jtag->'tag2'!='beijing'") tdSql.checkRows(5) - #open tdSql.query("select * from jsons1 where jtag->'tag2'=''") tdSql.checkRows(2) # @@ -253,7 +252,6 @@ class TDTestCase: tdSql.checkData(0, 1, 2) tdSql.query("select * from jsons1 where jtag->'tag1'=10") tdSql.checkRows(0) - # open tdSql.query("select * from jsons1 where jtag->'tag1'<54") tdSql.checkRows(4) tdSql.query("select * from jsons1 where jtag->'tag1'<=11") @@ -270,7 +268,6 @@ class TDTestCase: # # where json value is double tdSql.query("select * from jsons1 where jtag->'tag1'=1.232") tdSql.checkRows(1) - # open tdSql.query("select * from jsons1 where jtag->'tag1'<1.232") tdSql.checkRows(1) tdSql.query("select * from jsons1 where jtag->'tag1'<=1.232") @@ -279,7 +276,6 @@ class TDTestCase: tdSql.checkRows(3) tdSql.query("select * from jsons1 where jtag->'tag1'>=1.232") tdSql.checkRows(3) - # open tdSql.query("select * from jsons1 where jtag->'tag1'!=1.232") tdSql.checkRows(6) tdSql.query("select * from jsons1 where jtag->'tag1'!=3.232") @@ -289,7 +285,6 @@ class TDTestCase: # # # where json value is bool tdSql.query("select * from jsons1 where jtag->'tag1'=true") - # open tdSql.checkRows(0) #tdSql.query("select * from jsons1 where jtag->'tag1'=false") #tdSql.checkRows(1) @@ -303,7 +298,6 @@ class TDTestCase: #tdSql.checkRows(1) # # # where json key is null - # open tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3") tdSql.checkRows(0) # From d8c2a68f27df8ec9f3c108c10682ca7fb6d0e13b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 15 Jun 2022 10:44:36 +0800 Subject: [PATCH 33/41] feat(stream): update data fo partition by --- source/libs/executor/inc/executorimpl.h | 4 + source/libs/executor/src/scanoperator.c | 154 +++++++++++++----- source/libs/executor/src/timewindowoperator.c | 4 +- source/libs/function/src/builtinsimpl.c | 9 +- source/libs/planner/src/planOptimizer.c | 34 ++-- 5 files changed, 145 insertions(+), 60 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ab60acab53..c4d16c89e1 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -391,7 +391,9 @@ typedef struct SStreamBlockScanInfo { void* streamBlockReader;// stream block reader handle SArray* pColMatchInfo; // SNode* pCondition; + int32_t tsArrayIndex; SArray* tsArray; + uint64_t groupId; SUpdateInfo* pUpdateInfo; SExprInfo* pPseudoExpr; @@ -582,6 +584,7 @@ typedef struct SPartitionOperatorInfo { int32_t* columnOffset; // start position for each column data void* pGroupIter; // group iterator int32_t pageIndex; // page index of current group + SSDataBlock* pUpdateRes; } SPartitionOperatorInfo; typedef struct SWindowRowsSup { @@ -907,6 +910,7 @@ int32_t compareTimeWindow(const void* p1, const void* p2, const void* param); int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); +void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); #ifdef __cplusplus } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d30e4ef6db..e44e05224f 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -750,16 +750,103 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { return true; } +static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) { + for (int32_t j = 0; j < source->info.numOfCols; j++) { + SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j); + SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j); + if (colDataIsNull_s(pSourceCol, sourceRowId)) { + colDataAppendNULL(pDestCol, dest->info.rows); + } else { + colDataAppend(pDestCol, dest->info.rows, colDataGetData(pSourceCol, sourceRowId), false); + } + } + dest->info.rows++; +} + +static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t rowId) { + uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); + if (groupId) { + return *groupId; + } + return 0; + /* Todo(liuyao) for partition by column + recordNewGroupKeys(pTableScanInfo->pGroupCols, pTableScanInfo->pGroupColVals, pBlock, rowId); + int32_t len = buildGroupKeys(pTableScanInfo->keyBuf, pTableScanInfo->pGroupColVals); + uint64_t resId = 0; + uint64_t* groupId = taosHashGet(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len); + if (groupId) { + return *groupId; + } else if (len != 0) { + resId = calcGroupId(pTableScanInfo->keyBuf, len); + taosHashPut(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len, &resId, sizeof(uint64_t)); + } + return resId; + */ +} + static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) { - SSDataBlock* pResult = NULL; - pResult = doTableScan(pInfo->pOperatorDumy); - if (pResult == NULL) { - if (prepareDataScan(pInfo)) { - // scan next window data - pResult = doTableScan(pInfo->pOperatorDumy); + while (1) { + SSDataBlock* pResult = NULL; + pResult = doTableScan(pInfo->pOperatorDumy); + if (pResult == NULL) { + if (prepareDataScan(pInfo)) { + // scan next window data + pResult = doTableScan(pInfo->pOperatorDumy); + } + } + if (!pResult) { + return NULL; + } + + if (pResult->info.groupId == pInfo->groupId) { + return pResult; + } + } + +/* Todo(liuyao) for partition by column + SSDataBlock* pBlock = createOneDataBlock(pResult, true); + blockDataCleanup(pResult); + for (int32_t i = 0; i < pBlock->info.rows; i++) { + uint64_t id = getGroupId(pInfo->pOperatorDumy, pBlock, i); + if (id == pInfo->groupId) { + copyOneRow(pResult, pBlock, i); } } return pResult; +*/ +} + +static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) { + blockDataCleanup(pUpdateBlock); + int32_t size = taosArrayGetSize(pInfo->tsArray); + if (pInfo->tsArrayIndex < size) { + SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex); + ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); + blockDataEnsureCapacity(pUpdateBlock, size); + ASSERT(pBlock->info.numOfCols == pUpdateBlock->info.numOfCols); + + int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex); + pInfo->groupId = getGroupId(pInfo->pOperatorDumy, pBlock, rowId); + int32_t i = 0; + for ( ; i < size; i++) { + rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, i + pInfo->tsArrayIndex); + uint64_t id = getGroupId(pInfo->pOperatorDumy, pBlock, rowId); + if (pInfo->groupId != id) { + break; + } + copyOneRow(pUpdateBlock, pBlock, rowId); + } + pUpdateBlock->info.rows = i; + pInfo->tsArrayIndex += i; + pUpdateBlock->info.groupId = pInfo->groupId; + pUpdateBlock->info.type = STREAM_REPROCESS; + blockDataUpdateTsWindow(pUpdateBlock, 0); + } + // all rows have same group id + ASSERT(pInfo->tsArrayIndex >= size); + if (size > 0 && pInfo->tsArrayIndex == size) { + taosArrayClear(pInfo->tsArray); + } } static void getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, @@ -767,41 +854,21 @@ static void getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool invertible, SSD SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* ts = (TSKEY*)pColDataInfo->pData; - for (int32_t i = 0; i < pBlock->info.rows; i++) { - if (updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, ts[i])) { - taosArrayPush(pInfo->tsArray, ts + i); + for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { + if (updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, ts[rowId])) { + taosArrayPush(pInfo->tsArray, &rowId); } } if (!pUpdateBlock) { taosArrayClear(pInfo->tsArray); return; } - int32_t size = taosArrayGetSize(pInfo->tsArray); - if (size > 0 && invertible) { - // Todo(liuyao) get from tsdb - // SSDataBlock* p = createOneDataBlock(pBlock, true); - // p->info.type = STREAM_INVERT; - // taosArrayClear(pInfo->tsArray); - // return p; - SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex); - ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); - blockDataEnsureCapacity(pUpdateBlock, size); - for (int32_t i = 0; i < size; i++) { - TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->tsArray, i); - colDataAppend(pCol, i, (char*)pTs, false); - } - for (int32_t i = 0; i < pUpdateBlock->info.numOfCols; i++) { - if (i == pInfo->primaryTsIndex) { - continue; - } - SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, i); - colDataAppendNNULL(pCol, 0, size); - } - pUpdateBlock->info.rows = size; - pUpdateBlock->info.type = STREAM_REPROCESS; - blockDataUpdateTsWindow(pUpdateBlock, 0); - taosArrayClear(pInfo->tsArray); - } + setUpdateData(pInfo, pBlock, pUpdateBlock); + // Todo(liuyao) get from tsdb + // SSDataBlock* p = createOneDataBlock(pBlock, true); + // p->info.type = STREAM_INVERT; + // taosArrayClear(pInfo->tsArray); + // return p; } static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { @@ -833,7 +900,6 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; return pInfo->pRes; } else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) { - blockDataCleanup(pInfo->pRes); pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; if (!isStateWindow(pInfo)) { prepareDataScan(pInfo); @@ -848,7 +914,15 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) { SSDataBlock* pSDB = doDataScan(pInfo); if (pSDB == NULL) { - pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes); + if (pInfo->pUpdateRes->info.rows > 0) { + if (!isStateWindow(pInfo)) { + prepareDataScan(pInfo); + } + return pInfo->pUpdateRes; + } else { + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + } } else { getUpdateDataBlock(pInfo, true, pSDB, NULL); return pSDB; @@ -941,7 +1015,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { if (rows == 0) { pOperator->status = OP_EXEC_DONE; } else if (pInfo->pUpdateInfo) { - blockDataCleanup(pInfo->pUpdateRes); + pInfo->tsArrayIndex = 0; getUpdateDataBlock(pInfo, true, pInfo->pRes, pInfo->pUpdateRes); if (pInfo->pUpdateRes->info.rows > 0) { if (pInfo->pUpdateRes->info.type == STREAM_REPROCESS) { @@ -1020,7 +1094,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan goto _error; } - pInfo->tsArray = taosArrayInit(4, sizeof(TSKEY)); + pInfo->tsArray = taosArrayInit(4, sizeof(int32_t)); if (pInfo->tsArray == NULL) { goto _error; } @@ -1047,6 +1121,8 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan pInfo->pOperatorDumy = pTableScanDummy; pInfo->interval = pSTInfo->interval; pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; + pInfo->groupId = 0; + pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; pOperator->blocking = false; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0151474008..5a18649cab 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1985,7 +1985,7 @@ static void clearUpdateDataBlock(SSDataBlock* pBlock) { blockDataCleanup(pBlock); } -static void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) { +void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) { ASSERT(pDest->info.capacity >= pSource->info.rows); clearUpdateDataBlock(pDest); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); @@ -1997,6 +1997,8 @@ static void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_ colDataAppendNNULL(pCol, 0, pSource->info.rows); } pDest->info.rows = pSource->info.rows; + pDest->info.groupId = pSource->info.groupId; + pDest->info.type = pSource->info.type; blockDataUpdateTsWindow(pDest, 0); } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c6ab9f859d..94208043e7 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2345,14 +2345,7 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SAPercentileInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); ASSERT(pDBuf->algo == pSBuf->algo); - if (pDBuf->algo == APERCT_ALGO_TDIGEST) { - tdigestMerge(pDBuf->pTDigest, pSBuf->pTDigest); - } else { - SHistogramInfo* pTmp = tHistogramMerge(pDBuf->pHisto, pSBuf->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pDBuf->pHisto, pTmp, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); - pDBuf->pHisto->elems = (SHistBin*)((char*)pDBuf->pHisto + sizeof(SHistogramInfo)); - tHistogramDestroy(&pTmp); - } + apercentileTransferInfo(pSBuf, pDBuf); pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 65d73b4cdb..41b80eaaa8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -104,10 +104,14 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { return false; } if (NULL == pNode->pParent || (QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) && - QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent))) { + QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) && + QUERY_NODE_LOGIC_PLAN_PARTITION != nodeType(pNode->pParent))) { return false; } - if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) { + if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) || + (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode->pParent) && + pNode->pParent->pParent && + QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent)) ) { return true; } return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); @@ -217,16 +221,22 @@ static int32_t osdGetDataRequired(SNodeList* pFuncs) { } static void setScanWindowInfo(SScanLogicNode* pScan) { - if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pScan->node.pParent)) { - pScan->interval = ((SWindowLogicNode*)pScan->node.pParent)->interval; - pScan->offset = ((SWindowLogicNode*)pScan->node.pParent)->offset; - pScan->sliding = ((SWindowLogicNode*)pScan->node.pParent)->sliding; - pScan->intervalUnit = ((SWindowLogicNode*)pScan->node.pParent)->intervalUnit; - pScan->slidingUnit = ((SWindowLogicNode*)pScan->node.pParent)->slidingUnit; - pScan->triggerType = ((SWindowLogicNode*)pScan->node.pParent)->triggerType; - pScan->watermark = ((SWindowLogicNode*)pScan->node.pParent)->watermark; - pScan->tsColId = ((SColumnNode*)((SWindowLogicNode*)pScan->node.pParent)->pTspk)->colId; - pScan->filesFactor = ((SWindowLogicNode*)pScan->node.pParent)->filesFactor; + SLogicNode* pParent = pScan->node.pParent; + if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pParent) && + pParent->pParent && + QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pParent->pParent)) { + pParent = pParent->pParent; + } + if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pParent)) { + pScan->interval = ((SWindowLogicNode*)pParent)->interval; + pScan->offset = ((SWindowLogicNode*)pParent)->offset; + pScan->sliding = ((SWindowLogicNode*)pParent)->sliding; + pScan->intervalUnit = ((SWindowLogicNode*)pParent)->intervalUnit; + pScan->slidingUnit = ((SWindowLogicNode*)pParent)->slidingUnit; + pScan->triggerType = ((SWindowLogicNode*)pParent)->triggerType; + pScan->watermark = ((SWindowLogicNode*)pParent)->watermark; + pScan->tsColId = ((SColumnNode*)((SWindowLogicNode*)pParent)->pTspk)->colId; + pScan->filesFactor = ((SWindowLogicNode*)pParent)->filesFactor; } } From 5f73b165f59bb0e40bdd31a84dd258a5dac59c35 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 15 Jun 2022 15:59:56 +0800 Subject: [PATCH 34/41] os: add taosd assert kill --- source/libs/catalog/src/ctgCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 62890b8326..277516686b 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1899,7 +1899,7 @@ _return: void ctgUpdateThreadUnexpectedStopped(void) { - if (CTG_IS_LOCKED(&gCtgMgmt.lock) > 0) CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); + if (!atomic_load_8((int8_t*)&gCtgMgmt.exit) && CTG_IS_LOCKED(&gCtgMgmt.lock) > 0) CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); } void ctgCleanupCacheQueue(void) { From 8eb08fa0d4c021f1a8b74e2096282641793475a8 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 15 Jun 2022 16:11:20 +0800 Subject: [PATCH 35/41] feat(stream): trigger mode add max delay --- examples/c/stream_demo.c | 3 +- include/libs/executor/executor.h | 9 ++- include/libs/stream/tstream.h | 32 ++++++++- source/client/src/tmq.c | 84 ++++++++++++++++++++++ source/dnode/mnode/impl/inc/mndDef.h | 4 +- source/dnode/mnode/impl/inc/mndStream.h | 2 +- source/dnode/mnode/impl/src/mndDef.c | 8 +-- source/dnode/mnode/impl/src/mndScheduler.c | 3 + source/dnode/mnode/impl/src/mndSma.c | 5 +- source/dnode/mnode/impl/src/mndStream.c | 17 +++-- source/dnode/vnode/src/tq/tq.c | 21 +++--- source/libs/executor/src/executor.c | 20 ++++-- source/libs/function/src/builtins.c | 4 +- source/libs/stream/inc/streamInc.h | 7 ++ source/libs/stream/src/stream.c | 68 +++++++++++++++++- source/libs/stream/src/streamExec.c | 14 ++-- source/libs/stream/src/streamTask.c | 2 + source/util/src/tarray.c | 6 +- 18 files changed, 261 insertions(+), 48 deletions(-) diff --git a/examples/c/stream_demo.c b/examples/c/stream_demo.c index 943fcbdb53..ab59fa5e47 100644 --- a/examples/c/stream_demo.c +++ b/examples/c/stream_demo.c @@ -82,7 +82,8 @@ int32_t create_stream() { /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ pRes = taos_query( - pConn, "create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from st1 interval(10m)"); + pConn, + "create stream stream1 trigger max_delay 10s into outstb as select _wstartts, sum(k) from st1 interval(10m)"); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 288248422b..3f05c27453 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -38,8 +38,10 @@ typedef struct SReadHandle { SMsgCb* pMsgCb; } SReadHandle; -#define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1 -#define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2 +enum { + STREAM_DATA_TYPE_SUBMIT_BLOCK = 1, + STREAM_DATA_TYPE_SSDATA_BLOCK = 2, +}; typedef enum { OPTR_EXEC_MODEL_BATCH = 0x1, @@ -102,7 +104,8 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, * @param tversion * @return */ -int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, int32_t* tversion); +int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, + int32_t* tversion); /** * The main task execution function, including query on both table and multiple tables, diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 960794792b..2c9d66a828 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -58,6 +58,7 @@ enum { enum { STREAM_INPUT__DATA_SUBMIT = 1, STREAM_INPUT__DATA_BLOCK, + STREAM_INPUT__TRIGGER, STREAM_INPUT__CHECKPOINT, }; @@ -85,6 +86,11 @@ typedef struct { int8_t type; } SStreamCheckpoint; +typedef struct { + int8_t type; + SSDataBlock* pBlock; +} SStreamTrigger; + enum { STREAM_QUEUE__SUCESS = 1, STREAM_QUEUE__FAILED, @@ -98,6 +104,9 @@ typedef struct { int8_t status; } SStreamQueue; +int32_t streamInit(); +void streamCleanUp(); + SStreamQueue* streamQueueOpen(); void streamQueueClose(SStreamQueue* queue); @@ -220,6 +229,11 @@ enum { TASK_INPUT_TYPE__DATA_BLOCK, }; +enum { + TASK_TRIGGER_STATUS__IN_ACTIVE = 1, + TASK_TRIGGER_STATUS__ACTIVE, +}; + struct SStreamTask { int64_t streamId; int32_t taskId; @@ -262,8 +276,16 @@ struct SStreamTask { SStreamQueue* inputQueue; SStreamQueue* outputQueue; + // trigger + int8_t triggerStatus; + int64_t triggerParam; + void* timer; + // application storage // void* ahandle; + + // msg handle + SMsgCb* pMsgCb; }; SStreamTask* tNewSStreamTask(int64_t streamId); @@ -292,6 +314,13 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem taosWriteQitem(pTask->inputQueue->queue, pItem); } else if (pItem->type == STREAM_INPUT__CHECKPOINT) { taosWriteQitem(pTask->inputQueue->queue, pItem); + } else if (pItem->type == STREAM_INPUT__TRIGGER) { + taosWriteQitem(pTask->inputQueue->queue, pItem); + } + + if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0 && + pTask->triggerStatus == TASK_TRIGGER_STATUS__IN_ACTIVE) { + atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE); } // TODO: back pressure @@ -370,7 +399,8 @@ typedef struct { int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); -int32_t streamTriggerByWrite(SStreamTask* pTask, int32_t vgId, SMsgCb* pMsgCb); +int32_t streamLaunchByWrite(SStreamTask* pTask, int32_t vgId, SMsgCb* pMsgCb); +int32_t streamSetupTrigger(SStreamTask* pTask); int32_t streamTaskRun(SStreamTask* pTask); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 10c6c3623a..dd900e6045 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -992,6 +992,90 @@ CREATE_MSG_FAIL: return -1; } +bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { + bool set = false; + + int32_t topicNumGet = taosArrayGetSize(pRsp->topics); + char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; + tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, + topicNumGet); + + SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); + if (newTopics == NULL) { + return false; + } + + SHashObj* pHash = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); + if (pHash == NULL) { + taosArrayDestroy(newTopics); + return false; + } + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < topicNumCur; i++) { + // find old topic + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if (pTopicCur->vgs) { + int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); + tscDebug("consumer %ld new vg num: %d", tmq->consumerId, vgNumCur); + if (vgNumCur == 0) break; + for (int32_t j = 0; j < vgNumCur; j++) { + SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); + sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); + tscDebug("consumer %ld epoch %d vg %d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey); + taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); + } + break; + } + } + + for (int32_t i = 0; i < topicNumGet; i++) { + SMqClientTopic topic = {0}; + SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); + topic.schema = pTopicEp->schema; + taosHashClear(pHash); + topic.topicName = strdup(pTopicEp->topic); + tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN); + + tscDebug("consumer %ld update topic: %s", tmq->consumerId, topic.topicName); + + int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); + topic.vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); + for (int32_t j = 0; j < vgNumGet; j++) { + SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); + sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); + int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); + int64_t offset = tmq->resetOffsetCfg; + if (pOffset != NULL) { + offset = *pOffset; + } + + tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); + SMqClientVg clientVg = { + .pollCnt = 0, + .currentOffset = offset, + .vgId = pVgEp->vgId, + .epSet = pVgEp->epSet, + .vgStatus = TMQ_VG_STATUS__IDLE, + .vgSkipCnt = 0, + }; + taosArrayPush(topic.vgs, &clientVg); + set = true; + } + taosArrayPush(newTopics, &topic); + } + if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); + taosHashCleanup(pHash); + tmq->clientTopics = newTopics; + + if (taosArrayGetSize(tmq->clientTopics) == 0) + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC); + else + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); + + atomic_store_32(&tmq->epoch, epoch); + return set; +} + bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ bool set = false; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 6dd5d1622c..2e543149c0 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -554,8 +554,8 @@ typedef struct { SVgObj fixedSinkVg; int64_t smaId; // 0 for unused int8_t trigger; - int32_t triggerParam; - int64_t waterMark; + int64_t triggerParam; + int64_t watermark; char* sql; char* physicalPlan; SArray* tasks; // SArray> diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 15cd9fa043..b5d22cb7a5 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -31,7 +31,7 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); -int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark, STrans *pTrans); +int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 1e2c61ff89..6eac91bb43 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -33,8 +33,8 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1; if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; - if (tEncodeI32(pEncoder, pObj->triggerParam) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->waterMark) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->triggerParam) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->watermark) < 0) return -1; if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; @@ -85,8 +85,8 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; - if (tDecodeI32(pDecoder, &pObj->triggerParam) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->waterMark) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->triggerParam) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->watermark) < 0) return -1; if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index a3b7754a4e..5b745f3cea 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -387,6 +387,9 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // input pFinalTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; + // trigger + pFinalTask->triggerParam = pStream->triggerParam; + // dispatch if (mndAddDispatcherToInnerTask(pMnode, pTrans, pStream, pFinalTask) < 0) { qDestroyQueryPlan(pPlan); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index a14eb78ffe..003afb4fa0 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -561,6 +561,8 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.sql = pCreate->sql; streamObj.createdBy = STREAM_CREATED_BY__SMA; streamObj.smaId = smaObj.uid; + streamObj.watermark = 0; + streamObj.trigger = STREAM_TRIGGER_AT_ONCE; if (mndAllocSmaVgroup(pMnode, pDb, &streamObj.fixedSinkVg) != 0) { mError("sma:%s, failed to create since %s", smaObj.name, terrstr()); @@ -583,7 +585,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER; - if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, STREAM_TRIGGER_AT_ONCE, 0, pTrans) != 0) goto _OVER; + if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -1012,7 +1014,6 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool rsp->suid = pStb->uid; rsp->version = pStb->smaVer; mndReleaseStb(pMnode, pStb); - while (1) { pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 96d199fcb6..4a7da8f2d1 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -235,16 +235,15 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64 } if (TSDB_CODE_SUCCESS == code) { - code = nodesNodeToString((SNode*)pPlan, false, pStr, NULL); + code = nodesNodeToString((SNode *)pPlan, false, pStr, NULL); } nodesDestroyNode(pAst); - nodesDestroyNode((SNode*)pPlan); + nodesDestroyNode((SNode *)pPlan); terrno = code; return code; } -int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark, - STrans *pTrans) { +int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) { SNode *pAst = NULL; if (nodesStringToNode(ast, &pAst) < 0) { @@ -258,7 +257,6 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast // free nodesDestroyNode(pAst); - #if 0 printf("|"); for (int i = 0; i < pStream->outputSchema.nCols; i++) { @@ -268,7 +266,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast #endif - if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(ast, triggerType, watermark, &pStream->physicalPlan)) { + if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(ast, pStream->trigger, pStream->watermark, &pStream->physicalPlan)) { mError("topic:%s, failed to get plan since %s", pStream->name, terrstr()); return -1; } @@ -391,7 +389,8 @@ static int32_t mndCreateStream(SMnode *pMnode, SRpcMsg *pReq, SCMCreateStreamReq streamObj.smaId = 0; /*streamObj.physicalPlan = "";*/ streamObj.trigger = pCreate->triggerType; - streamObj.waterMark = pCreate->watermark; + streamObj.watermark = pCreate->watermark; + streamObj.triggerParam = pCreate->maxDelay; if (streamObj.targetSTbName[0]) { pDb = mndAcquireDbByStb(pMnode, streamObj.targetSTbName); @@ -409,7 +408,7 @@ static int32_t mndCreateStream(SMnode *pMnode, SRpcMsg *pReq, SCMCreateStreamReq } mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name); - if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pCreate->triggerType, pCreate->watermark, pTrans) != 0) { + if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) { mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -566,7 +565,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetSTbName, true); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->waterMark, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->trigger, false); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9f34ae39c0..7d2dfeeba5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -375,6 +375,8 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) goto FAIL; + pTask->pMsgCb = &pTq->pVnode->msgCb; + // exec if (pTask->execType != TASK_EXEC__NONE) { // expand runners @@ -406,9 +408,12 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { tdGetSTSChemaFromSSChema(&pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols); ASSERT(pTask->tbSink.pTSchema); } + + streamSetupTrigger(pTask); + tqInfo("deploy stream task id %d child id %d on vg %d", pTask->taskId, pTask->childId, pTq->pVnode->config.vgId); - taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), pTask, sizeof(SStreamTask)); + taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)); return 0; FAIL: @@ -431,7 +436,7 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { while (1) { pIter = taosHashIterate(pTq->pStreamTasks, pIter); if (pIter == NULL) break; - SStreamTask* pTask = (SStreamTask*)pIter; + SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue; if (!failed) { @@ -439,7 +444,7 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { continue; } - if (streamTriggerByWrite(pTask, pTq->pVnode->config.vgId, &pTq->pVnode->msgCb) < 0) { + if (streamLaunchByWrite(pTask, pTq->pVnode->config.vgId, &pTq->pVnode->msgCb) < 0) { continue; } } else { @@ -459,7 +464,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { // SStreamTaskRunReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; - SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); streamTaskProcessRunReq(pTask, &pTq->pVnode->msgCb); return 0; } @@ -473,7 +478,7 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) { tDecoderInit(&decoder, msgBody, msgLen); tDecodeStreamDispatchReq(&decoder, &req); int32_t taskId = req.taskId; - SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); SRpcMsg rsp = { .info = pMsg->info, .code = 0, @@ -485,7 +490,7 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; - SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); streamProcessRecoverReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg); return 0; } @@ -493,7 +498,7 @@ int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t taskId = pRsp->taskId; - SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); streamProcessDispatchRsp(pTask, &pTq->pVnode->msgCb, pRsp); return 0; } @@ -501,7 +506,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverRsp* pRsp = pMsg->pCont; int32_t taskId = pRsp->taskId; - SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); streamProcessRecoverRsp(pTask, pRsp); return 0; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index fd62849e56..bebfc75f17 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -19,7 +19,8 @@ #include "tdatablock.h" #include "vnode.h" -static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, bool assignUid, char* id) { +static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, bool assignUid, + char* id) { ASSERT(pOperator != NULL); if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { @@ -43,6 +44,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu if (pInfo->blockType == 0) { pInfo->blockType = type; } else if (pInfo->blockType != type) { + ASSERT(0); return TSDB_CODE_QRY_APP_ERROR; } @@ -51,7 +53,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } - } else { + } else if (type == STREAM_DATA_TYPE_SSDATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; @@ -62,6 +64,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock); taosArrayPush(pInfo->pBlockLists, &p); } + } else { + ASSERT(0); } return TSDB_CODE_SUCCESS; @@ -83,7 +87,8 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void**)pBlocks, numOfBlocks, type, assignUid, GET_TASKID(pTaskInfo)); + int32_t code = + doSetStreamBlock(pTaskInfo->pRoot, (void**)pBlocks, numOfBlocks, type, assignUid, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); } else { @@ -162,7 +167,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo pInfo = pInfo->pDownstream[0]; } - int32_t code = 0; + int32_t code = 0; SStreamBlockScanInfo* pScanInfo = pInfo->info; if (isAdd) { // add new table id SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList); @@ -178,9 +183,10 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo return code; } -int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, int32_t* tversion) { +int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, + int32_t* tversion) { ASSERT(tinfo != NULL && dbName != NULL && tableName != NULL); - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo; + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; *sversion = pTaskInfo->schemaVer.sversion; *tversion = pTaskInfo->schemaVer.tversion; @@ -196,4 +202,4 @@ int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tab } return 0; -} \ No newline at end of file +} diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 833f6db0c3..62a0c663b6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1682,7 +1682,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "top", .type = FUNCTION_TYPE_TOP, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateTopBot, .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, @@ -1717,7 +1717,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "bottom", .type = FUNCTION_TYPE_BOTTOM, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateTopBot, .getEnvFunc = getTopBotFuncEnv, .initFunc = topBotFunctionSetup, diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 27746fff78..8aaf4953de 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -23,6 +23,13 @@ extern "C" { #endif +typedef struct { + int8_t inited; + void* timer; +} SStreamGlobalEnv; + +static SStreamGlobalEnv streamEnv; + int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index d3828d9ee4..6dcbfad957 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -14,8 +14,74 @@ */ #include "streamInc.h" +#include "ttimer.h" -int32_t streamTriggerByWrite(SStreamTask* pTask, int32_t vgId, SMsgCb* pMsgCb) { +int32_t streamInit() { + int8_t old; + while (1) { + old = atomic_val_compare_exchange_8(&streamEnv.inited, 0, 2); + if (old != 2) break; + } + + if (old == 0) { + streamEnv.timer = taosTmrInit(10000, 100, 10000, "STREAM"); + if (streamEnv.timer == NULL) { + atomic_store_8(&streamEnv.inited, 0); + return -1; + } + atomic_store_8(&streamEnv.inited, 1); + } + return 0; +} + +void streamCleanUp() { + int8_t old; + while (1) { + old = atomic_val_compare_exchange_8(&streamEnv.inited, 1, 2); + if (old != 2) break; + } + + if (old == 1) { + taosTmrCleanUp(streamEnv.timer); + atomic_store_8(&streamEnv.inited, 0); + } +} + +void streamTriggerByTimer(void* param, void* tmrId) { + SStreamTask* pTask = (void*)param; + + if (atomic_load_8(&pTask->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { + SStreamTrigger* trigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM); + if (trigger == NULL) return; + trigger->type = STREAM_INPUT__TRIGGER; + trigger->pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + if (trigger->pBlock == NULL) { + taosFreeQitem(trigger); + return; + } + trigger->pBlock->info.type = STREAM_GET_ALL; + + atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE); + + streamTaskInput(pTask, (SStreamQueueItem*)trigger); + streamLaunchByWrite(pTask, pTask->nodeId, pTask->pMsgCb); + } + + taosTmrReset(streamTriggerByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); +} + +int32_t streamSetupTrigger(SStreamTask* pTask) { + if (pTask->triggerParam != 0) { + if (streamInit() < 0) { + return -1; + } + pTask->timer = taosTmrStart(streamTriggerByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer); + pTask->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; + } + return 0; +} + +int32_t streamLaunchByWrite(SStreamTask* pTask, int32_t vgId, SMsgCb* pMsgCb) { int8_t execStatus = atomic_load_8(&pTask->status); if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index fe1a857743..04428136ae 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -20,15 +20,17 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) void* exec = pTask->exec.executor; // set input - if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { + SStreamQueueItem* pItem = (SStreamQueueItem*)data; + if (pItem->type == STREAM_INPUT__TRIGGER) { + SStreamTrigger* pTrigger = (SStreamTrigger*)data; + qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_DATA_TYPE_SSDATA_BLOCK, false); + } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; - ASSERT(pSubmit->type == STREAM_INPUT__DATA_SUBMIT); - + ASSERT(pTask->inputType == STREAM_INPUT__DATA_SUBMIT); qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false); - } else if (pTask->inputType == STREAM_INPUT__DATA_BLOCK) { + } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)data; - ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - + ASSERT(pTask->inputType == STREAM_INPUT__DATA_BLOCK); SArray* blocks = pBlock->blocks; qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 5d8546bcb9..7a7d9d15ad 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -72,6 +72,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tSerializeSUseDbRspImp(pEncoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; /*if (tEncodeI8(pEncoder, pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/ } + if (tEncodeI64(pEncoder, pTask->triggerParam) < 0) return -1; /*tEndEncode(pEncoder);*/ return pEncoder->pos; @@ -121,6 +122,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { /*if (tDecodeI8(pDecoder, &pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/ if (tDeserializeSUseDbRspImp(pDecoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; } + if (tDecodeI64(pDecoder, &pTask->triggerParam) < 0) return -1; /*tEndDecode(pDecoder);*/ return 0; diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 2357f760d1..ec90e5a9b9 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -174,7 +174,11 @@ void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp) } void* taosArrayAddAll(SArray* pArray, const SArray* pInput) { - return taosArrayAddBatch(pArray, pInput->pData, (int32_t)taosArrayGetSize(pInput)); + if (pInput) { + return taosArrayAddBatch(pArray, pInput->pData, (int32_t)taosArrayGetSize(pInput)); + } else { + return NULL; + } } void* taosArrayPop(SArray* pArray) { From 4b18bd718eaa4767579aec660c902d2751632542 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 15 Jun 2022 16:14:17 +0800 Subject: [PATCH 36/41] refactor(sync): adjust errno --- include/libs/sync/sync.h | 10 +---- include/util/taoserror.h | 2 + source/dnode/mnode/impl/src/mndMain.c | 15 ++++--- source/dnode/mnode/impl/src/mndSync.c | 18 ++++---- source/dnode/vnode/src/vnd/vnodeSvr.c | 9 ++-- source/dnode/vnode/src/vnd/vnodeSync.c | 27 ++++-------- source/libs/sync/src/syncMain.c | 44 ++++++++++++------- .../test/syncConfigChangeSnapshotTest.cpp | 2 +- .../libs/sync/test/syncConfigChangeTest.cpp | 2 +- source/libs/sync/test/syncReplicateTest.cpp | 2 +- source/libs/sync/test/syncTestTool.cpp | 2 +- source/util/src/terror.c | 2 + 12 files changed, 71 insertions(+), 64 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index a369b81d26..2d21b3531a 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -24,7 +24,7 @@ extern "C" { #include "tdef.h" #include "tmsgcb.h" -#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 typedef uint64_t SyncNodeId; @@ -44,14 +44,6 @@ typedef enum { TAOS_SYNC_STATE_ERROR = 103, } ESyncState; -typedef enum { - TAOS_SYNC_PROPOSE_SUCCESS = 0, - TAOS_SYNC_PROPOSE_NOT_LEADER = 1, - TAOS_SYNC_ONLY_ONE_REPLICA = 2, - TAOS_SYNC_NOT_IN_NEW_CONFIG = 3, - TAOS_SYNC_OTHER_ERROR = 100, -} ESyncProposeCode; - typedef enum { TAOS_SYNC_FSM_CB_SUCCESS = 0, TAOS_SYNC_FSM_CB_OTHER_ERROR = 1, diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 30cbbbeb11..6fc84e023d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -411,6 +411,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) #define TSDB_CODE_SYN_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x0910) +#define TSDB_CODE_SYN_ONE_REPLICA TAOS_DEF_ERROR_CODE(0, 0x0911) +#define TSDB_CODE_SYN_NOT_IN_NEW_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0912) #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) // tq diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 59599ee134..00118eac1e 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -380,17 +380,19 @@ void mndStop(SMnode *pMnode) { int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SSyncMgmt *pMgmt = &pMnode->syncMgmt; - int32_t code = TAOS_SYNC_OTHER_ERROR; + int32_t code = 0; if (!syncEnvIsStart()) { mError("failed to process sync msg:%p type:%s since syncEnv stop", pMsg, TMSG_INFO(pMsg->msgType)); - return TAOS_SYNC_OTHER_ERROR; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } SSyncNode *pSyncNode = syncNodeAcquire(pMgmt->sync); if (pSyncNode == NULL) { mError("failed to process sync msg:%p type:%s since syncNode is null", pMsg, TMSG_INFO(pMsg->msgType)); - return TAOS_SYNC_OTHER_ERROR; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } char logBuf[512] = {0}; @@ -451,7 +453,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { tmsgSendRsp(&rsp); } else { mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - code = TAOS_SYNC_OTHER_ERROR; + code = -1; } } else { if (pMsg->msgType == TDMT_SYNC_TIMEOUT) { @@ -492,10 +494,13 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { tmsgSendRsp(&rsp); } else { mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - code = TAOS_SYNC_OTHER_ERROR; + code = -1; } } + if (code != 0) { + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + } return code; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 9940037356..63708aef8a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -234,9 +234,9 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { int32_t code = syncPropose(pMgmt->sync, &rsp, isWeak); if (code == 0) { tsem_wait(&pMgmt->syncSem); - } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) { + } else if (code == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { terrno = TSDB_CODE_APP_NOT_READY; - } else if (code == TAOS_SYNC_OTHER_ERROR) { + } else if (code == -1 && terrno == TSDB_CODE_SYN_INTERNAL_ERROR) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; } else { terrno = TSDB_CODE_APP_ERROR; @@ -257,13 +257,13 @@ void mndSyncStart(SMnode *pMnode) { syncStart(pMgmt->sync); mDebug("mnode sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); -/* - if (pMgmt->standby) { - syncStartStandBy(pMgmt->sync); - } else { - syncStart(pMgmt->sync); - } -*/ + /* + if (pMgmt->standby) { + syncStartStandBy(pMgmt->sync); + } else { + syncStart(pMgmt->sync); + } + */ } void mndSyncStop(SMnode *pMnode) {} diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index cd64bc8a9c..34ff9ffa0f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -296,7 +296,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { } int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - int32_t ret = TAOS_SYNC_OTHER_ERROR; + int32_t ret = 0; if (syncEnvIsStart()) { SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); @@ -381,15 +381,18 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { tmsgSendRsp(&rsp); } else { vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType); - ret = TAOS_SYNC_OTHER_ERROR; + ret = -1; } syncNodeRelease(pSyncNode); } else { vError("==vnodeProcessSyncReq== error syncEnv stop"); - ret = TAOS_SYNC_OTHER_ERROR; + ret = -1; } + if (ret != 0) { + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + } return ret; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 17c2c186be..975dff9fb6 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -98,7 +98,8 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { if (code == 0) { vnodeAccumBlockMsg(pVnode, pMsg->msgType); - } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) { + + } else if (code == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { SEpSet newEpSet = {0}; syncGetEpSet(pVnode->sync, &newEpSet); SEp *pEp = &newEpSet.eps[newEpSet.inUse]; @@ -247,29 +248,17 @@ static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -int32_t vnodeSnapshotStartRead(struct SSyncFSM *pFsm, void **ppReader) { - return 0; -} +int32_t vnodeSnapshotStartRead(struct SSyncFSM *pFsm, void **ppReader) { return 0; } -int32_t vnodeSnapshotStopRead(struct SSyncFSM *pFsm, void *pReader) { - return 0; -} +int32_t vnodeSnapshotStopRead(struct SSyncFSM *pFsm, void *pReader) { return 0; } -int32_t vnodeSnapshotDoRead(struct SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) { - return 0; -} +int32_t vnodeSnapshotDoRead(struct SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) { return 0; } -int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void **ppWriter) { - return 0; -} +int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void **ppWriter) { return 0; } -int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { - return 0; -} +int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { return 0; } -int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) { - return 0; -} +int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) { return 0; } static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7b1fd010ab..953294e489 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -149,12 +149,14 @@ void syncStop(int64_t rid) { int32_t syncSetStandby(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return TAOS_SYNC_OTHER_ERROR; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); - return TAOS_SYNC_OTHER_ERROR; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } // state change @@ -177,7 +179,8 @@ int32_t syncSetStandby(int64_t rid) { int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return TAOS_SYNC_OTHER_ERROR; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } ASSERT(rid == pSyncNode->rid); @@ -201,7 +204,8 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg if (!IamInNew) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); - return TAOS_SYNC_NOT_IN_NEW_CONFIG; + terrno = TSDB_CODE_SYN_NOT_IN_NEW_CONFIG; + return -1; } char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); @@ -219,7 +223,8 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return TAOS_SYNC_OTHER_ERROR; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } ASSERT(rid == pSyncNode->rid); @@ -246,7 +251,8 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { if (!IamInNew) { sError("sync reconfig error, not in new config"); taosReleaseRef(tsNodeRefId, pSyncNode->rid); - return TAOS_SYNC_NOT_IN_NEW_CONFIG; + terrno = TSDB_CODE_SYN_NOT_IN_NEW_CONFIG; + return -1; } char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); @@ -272,13 +278,15 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { int32_t syncLeaderTransfer(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return TAOS_SYNC_OTHER_ERROR; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } ASSERT(rid == pSyncNode->rid); if (pSyncNode->peersNum == 0) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); - return TAOS_SYNC_OTHER_ERROR; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } SNodeInfo newLeader = (pSyncNode->peersNodeInfo)[0]; @@ -291,7 +299,8 @@ int32_t syncLeaderTransfer(int64_t rid) { int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return TAOS_SYNC_OTHER_ERROR; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } ASSERT(rid == pSyncNode->rid); int32_t ret = 0; @@ -299,7 +308,8 @@ int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader) { if (pSyncNode->replicaNum == 1) { sError("only one replica, cannot drop leader"); taosReleaseRef(tsNodeRefId, pSyncNode->rid); - return TAOS_SYNC_ONLY_ONE_REPLICA; + terrno = TSDB_CODE_SYN_ONE_REPLICA; + return -1; } SyncLeaderTransfer* pMsg = syncLeaderTransferBuild(pSyncNode->vgId); @@ -538,11 +548,12 @@ void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { } int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { - int32_t ret = TAOS_SYNC_PROPOSE_SUCCESS; + int32_t ret = 0; SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return TAOS_SYNC_OTHER_ERROR; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + return -1; } assert(rid == pSyncNode->rid); sDebug("vgId:%d sync event propose msgType:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); @@ -553,7 +564,7 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { } int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) { - int32_t ret = TAOS_SYNC_PROPOSE_SUCCESS; + int32_t ret = 0; sDebug("vgId:%d sync event propose msgType:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -567,14 +578,17 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg); if (pSyncNode->FpEqMsg != NULL && (*pSyncNode->FpEqMsg)(pSyncNode->msgcb, &rpcMsg) == 0) { - ret = TAOS_SYNC_PROPOSE_SUCCESS; + ret = 0; } else { + ret = -1; + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; sError("syncPropose pSyncNode->FpEqMsg is NULL"); } syncClientRequestDestroy(pSyncMsg); } else { + ret = -1; + terrno = TSDB_CODE_SYN_NOT_LEADER; sError("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state)); - ret = TAOS_SYNC_PROPOSE_NOT_LEADER; } return ret; diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index 10b54d0aa4..3ee2b458d6 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -338,7 +338,7 @@ int main(int argc, char** argv) { if (alreadySend < writeRecordNum) { SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); int32_t ret = syncPropose(rid, pRpcMsg, false); - if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { sTrace("%s value%d write not leader", s, alreadySend); } else { assert(ret == 0); diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index 1e64a8a6f7..b1dc53d804 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -251,7 +251,7 @@ int main(int argc, char** argv) { if (alreadySend < writeRecordNum) { SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); int32_t ret = syncPropose(rid, pRpcMsg, false); - if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { sTrace("%s value%d write not leader", s, alreadySend); } else { assert(ret == 0); diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index bf9e34fffb..d955e64f81 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -188,7 +188,7 @@ int main(int argc, char** argv) { if (alreadySend < writeRecordNum) { SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); int32_t ret = syncPropose(rid, pRpcMsg, false); - if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { sTrace("%s value%d write not leader", s, alreadySend); } else { assert(ret == 0); diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index ebcd7368cc..21049454f4 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -391,7 +391,7 @@ int main(int argc, char** argv) { if (alreadySend < writeRecordNum) { SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); int32_t ret = syncPropose(rid, pRpcMsg, false); - if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { + if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { sTrace("%s value%d write not leader, leaderTransferWait:%d", simpleStr, alreadySend, leaderTransferWait); } else { assert(ret == 0); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 8e6284d2cb..621f947b64 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -413,6 +413,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGLEN, "Invalid msg length") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGTYPE, "Invalid msg type") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_LEADER, "Sync not leader") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_ONE_REPLICA, "Sync one replica") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_IN_NEW_CONFIG, "Sync not in new config") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") // wal From 87bcbe003ee84d5d3dd38fe685bbc966a009f46b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 15 Jun 2022 16:40:45 +0800 Subject: [PATCH 37/41] feat(stream): stream state&session support partition by --- source/libs/executor/inc/executorimpl.h | 8 +- source/libs/executor/src/executorimpl.c | 7 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 178 ++++++++++-------- 4 files changed, 108 insertions(+), 87 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c4d16c89e1..18d35ecc56 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -365,7 +365,9 @@ typedef struct SCatchSupporter { } SCatchSupporter; typedef struct SStreamAggSupporter { - SArray* pResultRows; + SHashObj* pResultRows; + SArray* pCurWins; + int32_t valueSize; int32_t keySize; char* pKeyBuf; // window key buffer SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file @@ -899,9 +901,9 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, - SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t size); + SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t size); SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize); -SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, int32_t* pIndex); +SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int64_t gap, int32_t* pIndex); int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 99a9dcb6af..53477eb56f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4814,6 +4814,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .calTrigger = pIntervalPhyNode->window.triggerType, .maxTs = INT64_MIN, }; + ASSERT(as.calTrigger != STREAM_TRIGGER_MAX_DELAY); int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type); @@ -5498,14 +5499,16 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo } int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput, - size_t size) { + int32_t size) { pSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); pSup->keySize = sizeof(int64_t) + sizeof(TSKEY); pSup->pKeyBuf = taosMemoryCalloc(1, pSup->keySize); - pSup->pResultRows = taosArrayInit(1024, size); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pSup->pResultRows = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); if (pSup->pKeyBuf == NULL || pSup->pResultRows == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + pSup->valueSize = size; pSup->pScanWindow = taosArrayInit(4, sizeof(STimeWindow)); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e44e05224f..571e9ae120 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -716,7 +716,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { int64_t gap = pInfo->sessionSup.gap; int32_t winIndex = 0; SResultWindowInfo* pCurWin = - getSessionTimeWindow(pAggSup->pResultRows, tsCols[pInfo->updateResIndex], gap, &winIndex); + getSessionTimeWindow(pAggSup, tsCols[pInfo->updateResIndex], pSDB->info.groupId, gap, &winIndex); win = pCurWin->win; pInfo->updateResIndex += updateSessionWindowInfo(pCurWin, tsCols, pSDB->info.rows, pInfo->updateResIndex, gap, NULL); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5a18649cab..738c5b2b26 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1320,6 +1320,15 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { break; } + if (pBlock->info.type == STREAM_REPROCESS) { + doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, 0, pOperator->numOfExprs, pBlock, NULL); + qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); + continue; + } else if (pBlock->info.type == STREAM_GET_ALL) { + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); + continue; + } + // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the // caller. Note that all the time window are not close till now. // the pDataBlock are always the same one, no need to call this again @@ -1328,16 +1337,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { setInverFunction(pInfo->binfo.pCtx, pOperator->numOfExprs, pBlock->info.type); } - if (pBlock->info.type == STREAM_REPROCESS) { - doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, 0, pOperator->numOfExprs, pBlock, NULL); - qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); - continue; - } else if (pBlock->info.type == STREAM_GET_ALL && - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_MAX_DELAY) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); - continue; - } - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); } @@ -2038,7 +2037,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } - setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true); if (pBlock->info.type == STREAM_REPROCESS) { SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, pInfo->primaryTsIndex, pOperator->numOfExprs, @@ -2058,12 +2056,12 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex); taosArrayDestroy(pUpWins); break; - } else if (pBlock->info.type == STREAM_GET_ALL && isFinalInterval(pInfo) && - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_MAX_DELAY) { + } else if (pBlock->info.type == STREAM_GET_ALL && isFinalInterval(pInfo)) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); continue; } + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true); if (isFinalInterval(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -2125,6 +2123,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, .calTrigger = pIntervalPhyNode->window.triggerType, .maxTs = INT64_MIN, }; + ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY); pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, 4096); @@ -2190,8 +2189,13 @@ _error: } void destroyStreamAggSupporter(SStreamAggSupporter* pSup) { - taosArrayDestroy(pSup->pResultRows); taosMemoryFreeClear(pSup->pKeyBuf); + void **pIte = NULL; + while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) { + SArray *pWins = (SArray *) (*pIte); + taosArrayDestroy(pWins); + } + taosHashCleanup(pSup->pResultRows); destroyDiskbasedBuf(pSup->pResultBuf); } @@ -2333,7 +2337,22 @@ static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) { return taosArrayPush(pWinInfos, &win); } -SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, int32_t* pIndex) { +SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) { + void** ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t)); + SArray* pWinInfos = NULL; + if (ite == NULL) { + pWinInfos = taosArrayInit(1024, pAggSup->valueSize); + taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void *)); + } else { + pWinInfos = *ite; + } + return pWinInfos; +} + +SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int64_t gap, int32_t* pIndex) { + SArray* pWinInfos = getWinInfos(pAggSup, groupId); + pAggSup->pCurWins = pWinInfos; + int32_t size = taosArrayGetSize(pWinInfos); if (size == 0) { *pIndex = 0; @@ -2389,7 +2408,7 @@ static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pRes SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { assert(pWinInfo->win.skey <= pWinInfo->win.ekey); // too many time window in query - int32_t size = taosArrayGetSize(pAggSup->pResultRows); + int32_t size = taosArrayGetSize(pAggSup->pCurWins); if (size > MAX_INTERVAL_TIME_WINDOW) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } @@ -2449,25 +2468,6 @@ static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlo pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pTaskInfo); } -int32_t copyWinInfoToDataBlock(SSDataBlock* pBlock, SStreamAggSupporter* pAggSup, int32_t start, int32_t num, - int32_t numOfExprs, SOptrBasicInfo* pBinfo) { - for (int32_t i = start; i < num; i += 1) { - SResultWindowInfo* pWinInfo = taosArrayGet(pAggSup->pResultRows, start); - SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pWinInfo->pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pWinInfo->pos.offset); - for (int32_t j = 0; j < numOfExprs; ++j) { - SResultRowEntryInfo* pResultInfo = getResultCell(pRow, j, pBinfo->rowCellInfoOffset); - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); - char* in = GET_ROWCELL_INTERBUF(pBinfo->pCtx[j].resultInfo); - colDataAppend(pColInfoData, pBlock->info.rows, in, pResultInfo->isNullRes); - } - pBlock->info.rows += pRow->numOfRows; - releaseBufPage(pAggSup->pResultBuf, bufPage); - } - blockDataUpdateTsWindow(pBlock, -1); - return TSDB_CODE_SUCCESS; -} - int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) { SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex); int32_t size = taosArrayGetSize(pWinInfos); @@ -2484,15 +2484,15 @@ int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SHashObj* pStUpdated, SHashObj* pStDeleted) { - SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pResultRows, startIndex); + SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex); SResultRow* pCurResult = NULL; setWindowOutputBuf(pCurWin, &pCurResult, pInfo->binfo.pCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->streamAggSup, pTaskInfo); num += startIndex + 1; - ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pResultRows)); + ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins)); // Just look for the window behind StartIndex for (int32_t i = startIndex + 1; i < num; i++) { - SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pResultRows, i); + SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i); SResultRow* pWinResult = NULL; setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->streamAggSup, pTaskInfo); @@ -2503,7 +2503,7 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &pWinInfo->win.skey, sizeof(TSKEY)); pWinInfo->isOutput = false; } - taosArrayRemove(pInfo->streamAggSup.pResultRows, i); + taosArrayRemove(pInfo->streamAggSup.pCurWins, i); } } @@ -2533,7 +2533,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; for (int32_t i = 0; i < pSDataBlock->info.rows;) { int32_t winIndex = 0; - SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup->pResultRows, tsCols[i], gap, &winIndex); + SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], pSDataBlock->info.groupId, gap, &winIndex); winRows = updateSessionWindowInfo(pCurWin, tsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted); code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { @@ -2543,7 +2543,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, // forwardRows, // pInfo->order, false); - int32_t winNum = getNumCompactWindow(pAggSup->pResultRows, winIndex, gap); + int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap); if (winNum > 0) { compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pTaskInfo, pStUpdated, pStDeleted); } @@ -2566,7 +2566,7 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SOptrBasicInfo* int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { int32_t winIndex = 0; - SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup->pResultRows, tsCols[i], gap, &winIndex); + SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], pBlock->info.groupId, gap, &winIndex); step = updateSessionWindowInfo(pCurWin, tsCols, pBlock->info.rows, i, gap, NULL); ASSERT(isInWindow(pCurWin, tsCols[i], gap)); doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pBinfo, numOfOutput); @@ -2627,7 +2627,7 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin for (int32_t j = 0; j < numOfChildren; j++) { SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; - SArray* pChWins = pChInfo->streamAggSup.pResultRows; + SArray* pChWins = getWinInfos(&pChInfo->streamAggSup, groupId); int32_t chWinSize = taosArrayGetSize(pChWins); int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey); for (int32_t k = index; k > 0 && k < chWinSize; k++) { @@ -2651,36 +2651,44 @@ typedef SResultWindowInfo* (*__get_win_info_)(void*); SResultWindowInfo* getSessionWinInfo(void* pData) { return (SResultWindowInfo*)pData; } SResultWindowInfo* getStateWinInfo(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; } -int32_t closeSessionWindow(SArray* pWins, STimeWindowAggSupp* pTwSup, SArray* pClosed, +int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn) { // Todo(liuyao) save window to tdb - int32_t size = taosArrayGetSize(pWins); - for (int32_t i = 0; i < size; i++) { - void* pWin = taosArrayGet(pWins, i); - SResultWindowInfo* pSeWin = fn(pWin); - if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) { - if (!pSeWin->isClosed) { - pSeWin->isClosed = true; - if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed); - pSeWin->isOutput = true; + void **pIte = NULL; + while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { + SArray *pWins = (SArray *) (*pIte); + int32_t size = taosArrayGetSize(pWins); + for (int32_t i = 0; i < size; i++) { + void* pWin = taosArrayGet(pWins, i); + SResultWindowInfo* pSeWin = fn(pWin); + if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) { + if (!pSeWin->isClosed) { + pSeWin->isClosed = true; + if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed); + pSeWin->isOutput = true; + } } + continue; } - continue; + break; } - break; } return TSDB_CODE_SUCCESS; } -int32_t getAllSessionWindow(SArray* pWins, SArray* pClosed, __get_win_info_ fn) { - int32_t size = taosArrayGetSize(pWins); - for (int32_t i = 0; i < size; i++) { - void* pWin = taosArrayGet(pWins, i); - SResultWindowInfo* pSeWin = fn(pWin); - if (!pSeWin->isClosed) { - int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed); - pSeWin->isOutput = true; +int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) { + void **pIte = NULL; + while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { + SArray *pWins = (SArray *) (*pIte); + int32_t size = taosArrayGetSize(pWins); + for (int32_t i = 0; i < size; i++) { + void* pWin = taosArrayGet(pWins, i); + SResultWindowInfo* pSeWin = fn(pWin); + if (!pSeWin->isClosed) { + int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed); + pSeWin->isOutput = true; + } } } return TSDB_CODE_SUCCESS; @@ -2714,8 +2722,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (pBlock == NULL) { break; } - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); + if (pBlock->info.type == STREAM_REPROCESS) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doClearSessionWindows(&pInfo->streamAggSup, &pInfo->binfo, pBlock, 0, pOperator->numOfExprs, pInfo->gap, pWins); @@ -2729,12 +2736,13 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } taosArrayDestroy(pWins); continue; - } else if (pBlock->info.type == STREAM_GET_ALL && - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_MAX_DELAY) { + } else if (pBlock->info.type == STREAM_GET_ALL) { getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getSessionWinInfo); continue; } + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); if (isFinalSession(pInfo)) { int32_t childIndex = 0; // Todo(liuyao) get child id from SSDataBlock SOptrBasicInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); @@ -2873,7 +2881,9 @@ bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) { return pKeyData && compareVal(pKeyData, &pWin->stateKey); } -SStateWindowInfo* getStateWindowByTs(SArray* pWinInfos, TSKEY ts, int32_t* pIndex) { +SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) { + SArray* pWinInfos = getWinInfos(pAggSup, groupId); + pAggSup->pCurWins = pWinInfos; int32_t size = taosArrayGetSize(pWinInfos); int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey); SStateWindowInfo* pWin = NULL; @@ -2896,7 +2906,10 @@ SStateWindowInfo* getStateWindowByTs(SArray* pWinInfos, TSKEY ts, int32_t* pInde return NULL; } -SStateWindowInfo* getStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol, int32_t* pIndex) { +SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, + uint64_t groupId, char* pKeyData, SColumn* pCol, int32_t* pIndex) { + SArray* pWinInfos = getWinInfos(pAggSup, groupId); + pAggSup->pCurWins = pWinInfos; int32_t size = taosArrayGetSize(pWinInfos); if (size == 0) { *pIndex = 0; @@ -2987,16 +3000,16 @@ static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc for (int32_t i = 0; i < pBlock->info.rows; i += step) { char* pKeyData = colDataGetData(pKeyColInfo, i); int32_t winIndex = 0; - SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup->pResultRows, tsCol[i], &winIndex); + SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], pBlock->info.groupId, &winIndex); if (!pCurWin) { continue; } - step = updateStateWindowInfo(pAggSup->pResultRows, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual, + step = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual, pSeDeleted); ASSERT(isTsInWindow(pCurWin, tsCol[i]) || isEqualStateKey(pCurWin, pKeyData)); taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); - deleteWindow(pAggSup->pResultRows, winIndex); + deleteWindow(pAggSup->pCurWins, winIndex); } } @@ -3026,13 +3039,15 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl char* pKeyData = colDataGetData(pKeyColInfo, i); int32_t winIndex = 0; bool allEqual = true; - SStateWindowInfo* pCurWin = getStateWindow(pAggSup->pResultRows, tsCols[i], pKeyData, &pInfo->stateCol, &winIndex); - winRows = updateStateWindowInfo(pAggSup->pResultRows, winIndex, tsCols, pKeyColInfo, pSDataBlock->info.rows, i, - &allEqual, pInfo->pSeDeleted); + SStateWindowInfo* pCurWin = + getStateWindow(pAggSup, tsCols[i], pSDataBlock->info.groupId, pKeyData, + &pInfo->stateCol, &winIndex); + winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, pKeyColInfo, + pSDataBlock->info.rows, i, &allEqual, pInfo->pSeDeleted); if (!allEqual) { taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); - deleteWindow(pAggSup->pResultRows, winIndex); + deleteWindow(pAggSup->pCurWins, winIndex); continue; } code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pTaskInfo); @@ -3079,17 +3094,18 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pBlock == NULL) { break; } - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); + if (pBlock->info.type == STREAM_REPROCESS) { doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId, pSeUpdated, pInfo->pSeDeleted); continue; - } else if (pBlock->info.type == STREAM_GET_ALL && - pInfo->twAggSup.calTrigger == STREAM_TRIGGER_MAX_DELAY) { + } else if (pBlock->info.type == STREAM_GET_ALL) { getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getStateWinInfo); continue; } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } From 3c4663848bc05dfb769f838d0010475e98ea3a1d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 15 Jun 2022 16:47:03 +0800 Subject: [PATCH 38/41] feat: support compute only use qnode --- include/libs/nodes/plannodes.h | 4 ++- source/libs/nodes/src/nodesCodeFuncs.c | 8 ++++- source/libs/planner/src/planScaleOut.c | 50 +++++++++++++++++++++++++- source/libs/planner/src/planSpliter.c | 19 +++++++--- 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 5d3e0cd142..26d055d7d2 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -177,7 +177,8 @@ typedef enum ESubplanType { SUBPLAN_TYPE_MERGE = 1, SUBPLAN_TYPE_PARTIAL, SUBPLAN_TYPE_SCAN, - SUBPLAN_TYPE_MODIFY + SUBPLAN_TYPE_MODIFY, + SUBPLAN_TYPE_COMPUTE } ESubplanType; typedef struct SSubplanId { @@ -196,6 +197,7 @@ typedef struct SLogicSubplan { SVgroupsInfo* pVgroupList; int32_t level; int32_t splitFlag; + int32_t numOfComputeNodes; } SLogicSubplan; typedef struct SQueryLogicPlan { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index d73434f3f0..968bb75997 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1117,6 +1117,7 @@ static const char* jkLogicSubplanVgroupsSize = "VgroupsSize"; static const char* jkLogicSubplanVgroups = "Vgroups"; static const char* jkLogicSubplanLevel = "Level"; static const char* jkLogicSubplanSplitFlag = "SplitFlag"; +static const char* jkLogicSubplanNumOfComputeNodes = "NumOfComputeNodes"; static int32_t logicSubplanToJson(const void* pObj, SJson* pJson) { const SLogicSubplan* pNode = (const SLogicSubplan*)pObj; @@ -1143,6 +1144,9 @@ static int32_t logicSubplanToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkLogicSubplanSplitFlag, pNode->splitFlag); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicSubplanNumOfComputeNodes, pNode->numOfComputeNodes); + } return code; } @@ -1159,7 +1163,6 @@ static int32_t jsonToLogicSubplan(const SJson* pJson, void* pObj) { } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkLogicSubplanType, pNode->subplanType, code); - ; } int32_t objSize = 0; if (TSDB_CODE_SUCCESS == code) { @@ -1174,6 +1177,9 @@ static int32_t jsonToLogicSubplan(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkLogicSubplanSplitFlag, &pNode->splitFlag); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkLogicSubplanNumOfComputeNodes, &pNode->numOfComputeNodes); + } return code; } diff --git a/source/libs/planner/src/planScaleOut.c b/source/libs/planner/src/planScaleOut.c index 9707b36f4a..a0b63ad6ff 100644 --- a/source/libs/planner/src/planScaleOut.c +++ b/source/libs/planner/src/planScaleOut.c @@ -115,7 +115,45 @@ static int32_t scaleOutForScan(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, } } -static int32_t pushHierarchicalPlan(SNodeList* pParentsGroup, SNodeList* pCurrentGroup) { +static int32_t scaleOutForCompute(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + int32_t code = TSDB_CODE_SUCCESS; + for (int32_t i = 0; i < pSubplan->numOfComputeNodes; ++i) { + SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); + if (NULL == pNewSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + code = nodesListStrictAppend(pGroup, (SNode*)pNewSubplan); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static int32_t pushHierarchicalPlanForCompute(SNodeList* pParentsGroup, SNodeList* pCurrentGroup) { + SNode* pChild = NULL; + SNode* pParent = NULL; + int32_t code = TSDB_CODE_SUCCESS; + FORBOTH(pChild, pCurrentGroup, pParent, pParentsGroup) { + code = nodesListMakeAppend(&(((SLogicSubplan*)pParent)->pChildren), pChild); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&(((SLogicSubplan*)pChild)->pParents), pParent); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static bool isComputeGroup(SNodeList* pGroup) { + if (0 == LIST_LENGTH(pGroup)) { + return false; + } + return SUBPLAN_TYPE_COMPUTE == ((SLogicSubplan*)nodesListGetNode(pGroup, 0))->subplanType; +} + +static int32_t pushHierarchicalPlanForNormal(SNodeList* pParentsGroup, SNodeList* pCurrentGroup) { int32_t code = TSDB_CODE_SUCCESS; bool topLevel = (0 == LIST_LENGTH(pParentsGroup)); SNode* pChild = NULL; @@ -138,6 +176,13 @@ static int32_t pushHierarchicalPlan(SNodeList* pParentsGroup, SNodeList* pCurren return code; } +static int32_t pushHierarchicalPlan(SNodeList* pParentsGroup, SNodeList* pCurrentGroup) { + if (isComputeGroup(pParentsGroup)) { + return pushHierarchicalPlanForCompute(pParentsGroup, pCurrentGroup); + } + return pushHierarchicalPlanForNormal(pParentsGroup, pCurrentGroup); +} + static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pParentsGroup) { SNodeList* pCurrentGroup = nodesMakeList(); if (NULL == pCurrentGroup) { @@ -155,6 +200,9 @@ static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32 case SUBPLAN_TYPE_MODIFY: code = scaleOutForModify(pCxt, pSubplan, level, pCurrentGroup); break; + case SUBPLAN_TYPE_COMPUTE: + code = scaleOutForCompute(pCxt, pSubplan, level, pCurrentGroup); + break; default: break; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 82f6b7fe14..3a9e1d7405 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -994,8 +994,20 @@ static int32_t qnodeSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { } int32_t code = splCreateExchangeNodeForSubplan(pCxt, info.pSubplan, info.pSplitNode, info.pSubplan->subplanType); if (TSDB_CODE_SUCCESS == code) { - code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, (SNode*)splCreateScanSubplan(pCxt, info.pSplitNode, 0)); + SLogicSubplan* pScanSubplan = splCreateScanSubplan(pCxt, info.pSplitNode, 0); + if (NULL != pScanSubplan) { + if (NULL != info.pSubplan->pVgroupList) { + info.pSubplan->numOfComputeNodes = info.pSubplan->pVgroupList->numOfVgroups; + TSWAP(pScanSubplan->pVgroupList, info.pSubplan->pVgroupList); + } else { + info.pSubplan->numOfComputeNodes = 1; + } + code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, (SNode*)pScanSubplan); + } else { + code = TSDB_CODE_OUT_OF_MEMORY; + } } + info.pSubplan->subplanType = SUBPLAN_TYPE_COMPUTE; ++(pCxt->groupId); pCxt->split = true; return code; @@ -1007,8 +1019,7 @@ static const SSplitRule splitRuleSet[] = { {.pName = "SingleTableJoinSplit", .splitFunc = singleTableJoinSplit}, {.pName = "UnionAllSplit", .splitFunc = unionAllSplit}, {.pName = "UnionDistinctSplit", .splitFunc = unionDistinctSplit}, - {.pName = "SmaIndexSplit", .splitFunc = smaIndexSplit}, - {.pName = "QnodeSplit", .splitFunc = qnodeSplit} + {.pName = "SmaIndexSplit", .splitFunc = smaIndexSplit} }; // clang-format on @@ -1039,7 +1050,7 @@ static int32_t applySplitRule(SPlanContext* pCxt, SLogicSubplan* pSubplan) { } } } while (split); - return TSDB_CODE_SUCCESS; + return qnodeSplit(&cxt, pSubplan); } static void setVgroupsInfo(SLogicNode* pNode, SLogicSubplan* pSubplan) { From dcd65e6deaf08b5a6c503d837e96830e7b2c5d60 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 15 Jun 2022 17:17:15 +0800 Subject: [PATCH 39/41] refactor(sync): do not heartbeat when 1 replica --- source/libs/sync/src/syncMain.c | 60 +++++++++++++++++---------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 953294e489..a0d8487b63 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1808,37 +1808,39 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = (SSyncNode*)param; - if (atomic_load_64(&pSyncNode->heartbeatTimerLogicClockUser) <= - atomic_load_64(&pSyncNode->heartbeatTimerLogicClock)) { - SyncTimeout* pSyncMsg = - syncTimeoutBuild2(SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pSyncNode->heartbeatTimerLogicClock), - pSyncNode->heartbeatTimerMS, pSyncNode->vgId, pSyncNode); - SRpcMsg rpcMsg; - syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); - syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); - if (pSyncNode->FpEqMsg != NULL) { - int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); - if (code != 0) { - sError("vgId:%d sync enqueue timer msg error, code:%d", pSyncNode->vgId, code); - rpcFreeCont(rpcMsg.pCont); - syncTimeoutDestroy(pSyncMsg); - return; + if (pSyncNode->replicaNum > 1) { + if (atomic_load_64(&pSyncNode->heartbeatTimerLogicClockUser) <= + atomic_load_64(&pSyncNode->heartbeatTimerLogicClock)) { + SyncTimeout* pSyncMsg = + syncTimeoutBuild2(SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pSyncNode->heartbeatTimerLogicClock), + pSyncNode->heartbeatTimerMS, pSyncNode->vgId, pSyncNode); + SRpcMsg rpcMsg; + syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); + syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); + if (pSyncNode->FpEqMsg != NULL) { + int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + if (code != 0) { + sError("vgId:%d sync enqueue timer msg error, code:%d", pSyncNode->vgId, code); + rpcFreeCont(rpcMsg.pCont); + syncTimeoutDestroy(pSyncMsg); + return; + } + } else { + sError("syncNodeEqHeartbeatTimer FpEqMsg is NULL"); + } + syncTimeoutDestroy(pSyncMsg); + + if (syncEnvIsStart()) { + taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, + &pSyncNode->pHeartbeatTimer); + } else { + sError("sync env is stop, syncNodeEqHeartbeatTimer"); } } else { - sError("syncNodeEqHeartbeatTimer FpEqMsg is NULL"); + sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 + "", + pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); } - syncTimeoutDestroy(pSyncMsg); - - if (syncEnvIsStart()) { - taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, - &pSyncNode->pHeartbeatTimer); - } else { - sError("sync env is stop, syncNodeEqHeartbeatTimer"); - } - } else { - sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 - "", - pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); } } @@ -2184,7 +2186,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ths->pFsm->FpRestoreFinishCb(ths->pFsm); } ths->restoreFinish = true; - sDebug("vgId:%d sync event restore finish", ths->vgId); + sDebug("vgId:%d sync event restore finish, index:%ld", ths->vgId, pEntry->index); } } From d7a6c4976c413b019744139d5639508e50a41db7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 15 Jun 2022 17:27:01 +0800 Subject: [PATCH 40/41] fix(stream): max delay should not be seen by operator (#13864) --- source/dnode/mnode/impl/src/mndStream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 4a7da8f2d1..f447a93392 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -228,7 +228,7 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64 .pAstRoot = pAst, .topicQuery = false, .streamQuery = true, - .triggerType = triggerType, + .triggerType = triggerType == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : triggerType, .watermark = watermark, }; code = qCreateQueryPlan(&cxt, &pPlan, NULL); From 6711d79434e276267e70950ba55c0c80bd35cbd3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Jun 2022 16:34:20 +0800 Subject: [PATCH 41/41] feat: increase enable and sysinfo limits for user privilege --- include/util/taoserror.h | 3 +- source/dnode/mnode/impl/inc/mndAuth.h | 45 ++++-- source/dnode/mnode/impl/src/mndAuth.c | 170 +++++++++++++-------- source/dnode/mnode/impl/src/mndBnode.c | 21 +-- source/dnode/mnode/impl/src/mndDb.c | 40 +---- source/dnode/mnode/impl/src/mndDnode.c | 20 +-- source/dnode/mnode/impl/src/mndFunc.c | 30 +--- source/dnode/mnode/impl/src/mndMnode.c | 21 +-- source/dnode/mnode/impl/src/mndQnode.c | 21 +-- source/dnode/mnode/impl/src/mndShow.c | 3 + source/dnode/mnode/impl/src/mndSma.c | 20 +-- source/dnode/mnode/impl/src/mndSnode.c | 21 +-- source/dnode/mnode/impl/src/mndStb.c | 28 +--- source/dnode/mnode/impl/src/mndStream.c | 33 ++-- source/dnode/mnode/impl/src/mndTopic.c | 27 ++-- source/dnode/mnode/impl/src/mndTrans.c | 9 +- source/dnode/mnode/impl/src/mndUser.c | 13 +- source/dnode/mnode/impl/src/mndVgroup.c | 48 ++---- source/util/src/terror.c | 1 + tests/system-test/0-others/user_control.py | 20 ++- 20 files changed, 219 insertions(+), 375 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 23897c7c72..538d6c58c0 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -132,7 +132,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) #define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0301) #define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0302) -#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0303) +#define TSDB_CODE_MND_USER_DISABLED TAOS_DEF_ERROR_CODE(0, 0x0303) +#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0304) // mnode-show #define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310) diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index de59a11cd7..45841ca367 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -22,23 +22,42 @@ extern "C" { #endif +typedef enum { + MND_OPER_CREATE_USER = 1, + MND_OPER_DROP_USER, + MND_OPER_ALTER_USER, + MND_OPER_CREATE_BNODE, + MND_OPER_DROP_BNODE, + MND_OPER_CREATE_DNODE, + MND_OPER_DROP_DNODE, + MND_OPER_CREATE_MNODE, + MND_OPER_DROP_MNODE, + MND_OPER_CREATE_QNODE, + MND_OPER_DROP_QNODE, + MND_OPER_CREATE_SNODE, + MND_OPER_DROP_SNODE, + MND_OPER_REDISTRIBUTE_VGROUP, + MND_OPER_SPLIT_VGROUP, + MND_OPER_BALANCE_VGROUP, + MND_OPER_CREATE_FUNC, + MND_OPER_DROP_FUNC, + MND_OPER_KILL_TRANS, + MND_OPER_CREATE_DB, + MND_OPER_ALTER_DB, + MND_OPER_DROP_DB, + MND_OPER_COMPACT_DB, + MND_OPER_USE_DB, + MND_OPER_WRITE_DB, + MND_OPER_READ_DB, +} EOperType; + int32_t mndInitAuth(SMnode *pMnode); void mndCleanupAuth(SMnode *pMnode); -int32_t mndCheckCreateUserAuth(SUserObj *pOperUser); +int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType); +int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); +int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType); int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); -int32_t mndCheckDropUserAuth(SUserObj *pOperUser); - -int32_t mndCheckNodeAuth(SUserObj *pOperUser); -int32_t mndCheckFuncAuth(SUserObj *pOperUser); -int32_t mndCheckTransAuth(SUserObj *pOperUser); - -int32_t mndCheckCreateDbAuth(SUserObj *pOperUser); -int32_t mndCheckAlterDropCompactDbAuth(SUserObj *pOperUser, SDbObj *pDb); -int32_t mndCheckUseDbAuth(SUserObj *pOperUser, SDbObj *pDb); - -int32_t mndCheckWriteAuth(SUserObj *pOperUser, SDbObj *pDb); -int32_t mndCheckReadAuth(SUserObj *pOperUser, SDbObj *pDb); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 1532fcc140..f036fc48f7 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -73,29 +73,44 @@ static int32_t mndProcessAuthReq(SRpcMsg *pReq) { return code; } -int32_t mndCheckCreateUserAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) return 0; +int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { + int32_t code = 0; + SUserObj *pUser = mndAcquireUser(pMnode, user); + + if (pUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + code = -1; + goto _OVER; + } + + if (pUser->superUser) { + goto _OVER; + } + + if (!pUser->enable) { + terrno = TSDB_CODE_MND_USER_DISABLED; + code = -1; + goto _OVER; + } + terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; + code = -1; + +_OVER: + mndReleaseUser(pMnode, pUser); + return code; } int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { + if (pOperUser->superUser) return 0; + if (!pOperUser->enable) { + terrno = TSDB_CODE_MND_USER_DISABLED; + return -1; + } + if (pAlter->alterType == TSDB_ALTER_USER_PASSWD) { - if (pOperUser->superUser || strcmp(pUser->user, pOperUser->user) == 0) { - return 0; - } - } else if (pAlter->alterType == TSDB_ALTER_USER_SUPERUSER) { - if (strcmp(pUser->user, TSDB_DEFAULT_USER) == 0) { - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - - if (pOperUser->superUser) { - return 0; - } - } else { - if (pOperUser->superUser) { - return 0; + if (strcmp(pUser->user, pOperUser->user) == 0) { + if (pOperUser->sysInfo) return 0; } } @@ -103,65 +118,92 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserRe return -1; } -int32_t mndCheckDropUserAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) return 0; - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; -} +int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType) { + int32_t code = 0; + SUserObj *pUser = mndAcquireUser(pMnode, user); -int32_t mndCheckNodeAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) return 0; - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; -} + if (pUser == NULL) { + code = -1; + goto _OVER; + } -int32_t mndCheckFuncAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) return 0; - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; -} + if (pUser->superUser) { + goto _OVER; + } -int32_t mndCheckTransAuth(SUserObj *pOperUser) { - if (pOperUser->superUser) return 0; - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; -} + if (!pUser->enable) { + terrno = TSDB_CODE_MND_USER_DISABLED; + code = -1; + goto _OVER; + } -int32_t mndCheckCreateDbAuth(SUserObj *pOperUser) { return 0; } - -int32_t mndCheckAlterDropCompactDbAuth(SUserObj *pOperUser, SDbObj *pDb) { - if (pOperUser->superUser || strcmp(pOperUser->user, pDb->createUser) == 0) { - return 0; + if (!pUser->sysInfo) { + terrno = TSDB_CODE_MND_NO_RIGHTS; + code = -1; + goto _OVER; } terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; + code = -1; + +_OVER: + mndReleaseUser(pMnode, pUser); + return code; } -int32_t mndCheckUseDbAuth(SUserObj *pOperUser, SDbObj *pDb) { return 0; } +int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) { + int32_t code = 0; + SUserObj *pUser = mndAcquireUser(pMnode, user); -int32_t mndCheckWriteAuth(SUserObj *pOperUser, SDbObj *pDb) { - if (pOperUser->superUser || strcmp(pOperUser->user, pDb->createUser) == 0) { - return 0; + if (pUser == NULL) { + code = -1; + goto _OVER; } - if (taosHashGet(pOperUser->writeDbs, pDb->name, strlen(pDb->name) + 1) != NULL) { - return 0; + if (pUser->superUser) goto _OVER; + + if (!pUser->enable) { + terrno = TSDB_CODE_MND_USER_DISABLED; + code = -1; + goto _OVER; + } + + if (operType == MND_OPER_CREATE_DB) { + if (pUser->sysInfo) goto _OVER; + } + + if (operType == MND_OPER_ALTER_DB) { + if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; + } + + if (operType == MND_OPER_DROP_DB) { + if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; + } + + if (operType == MND_OPER_COMPACT_DB) { + if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; + } + + if (operType == MND_OPER_USE_DB) { + if (strcmp(pUser->user, pDb->createUser) == 0) goto _OVER; + if (taosHashGet(pUser->readDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER; + if (taosHashGet(pUser->writeDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER; + } + + if (operType == MND_OPER_WRITE_DB) { + if (strcmp(pUser->user, pDb->createUser) == 0) goto _OVER; + if (taosHashGet(pUser->writeDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER; + } + + if (operType == MND_OPER_READ_DB) { + if (strcmp(pUser->user, pDb->createUser) == 0) goto _OVER; + if (taosHashGet(pUser->readDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER; } terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; -} - -int32_t mndCheckReadAuth(SUserObj *pOperUser, SDbObj *pDb) { - if (pOperUser->superUser || strcmp(pOperUser->user, pDb->createUser) == 0) { - return 0; - } - - if (taosHashGet(pOperUser->readDbs, pDb->name, strlen(pDb->name) + 1) != NULL) { - return 0; - } - - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; + code = -1; + +_OVER: + mndReleaseUser(pMnode, pUser); + return code; } diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index ed07e15c63..aa908b983d 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -269,7 +269,6 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { int32_t code = -1; SBnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; - SUserObj *pUser = NULL; SMCreateBnodeReq createReq = {0}; if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { @@ -293,13 +292,7 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckNodeAuth(pUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_BNODE) != 0) { goto _OVER; } @@ -313,7 +306,6 @@ _OVER: mndReleaseBnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); - mndReleaseUser(pMnode, pUser); return code; } @@ -382,7 +374,6 @@ _OVER: static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; - SUserObj *pUser = NULL; SBnodeObj *pObj = NULL; SMDropBnodeReq dropReq = {0}; @@ -403,13 +394,7 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckNodeAuth(pUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_BNODE) != 0) { goto _OVER; } @@ -422,8 +407,6 @@ _OVER: } mndReleaseBnode(pMnode, pObj); - mndReleaseUser(pMnode, pUser); - return code; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 43263af573..80f88b1060 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -526,7 +526,7 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckCreateDbAuth(pUser) != 0) { + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_DB, NULL) != 0) { goto _OVER; } @@ -684,7 +684,6 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SDbObj *pDb = NULL; - SUserObj *pUser = NULL; SAlterDbReq alterReq = {0}; SDbObj dbObj = {0}; @@ -701,12 +700,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto _OVER; - } - - if (mndCheckAlterDropCompactDbAuth(pUser, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_ALTER_DB, pDb) != 0) { goto _OVER; } @@ -733,7 +727,6 @@ _OVER: } mndReleaseDb(pMnode, pDb); - mndReleaseUser(pMnode, pUser); taosArrayDestroy(dbObj.cfg.pRetensions); return code; @@ -967,7 +960,6 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SDbObj *pDb = NULL; - SUserObj *pUser = NULL; SDropDbReq dropReq = {0}; if (tDeserializeSDropDbReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { @@ -988,12 +980,7 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { } } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto _OVER; - } - - if (mndCheckAlterDropCompactDbAuth(pUser, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_DROP_DB, pDb) != 0) { goto _OVER; } @@ -1006,8 +993,6 @@ _OVER: } mndReleaseDb(pMnode, pDb); - mndReleaseUser(pMnode, pUser); - return code; } @@ -1103,7 +1088,6 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SDbObj *pDb = NULL; - SUserObj *pUser = NULL; SUseDbReq usedbReq = {0}; SUseDbRsp usedbRsp = {0}; @@ -1143,12 +1127,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto _OVER; - } - - if (mndCheckUseDbAuth(pUser, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_USE_DB, pDb) != 0) { goto _OVER; } @@ -1179,7 +1158,6 @@ _OVER: } mndReleaseDb(pMnode, pDb); - mndReleaseUser(pMnode, pUser); tFreeSUsedbRsp(&usedbRsp); return code; @@ -1260,7 +1238,6 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SDbObj *pDb = NULL; - SUserObj *pUser = NULL; SCompactDbReq compactReq = {0}; if (tDeserializeSCompactDbReq(pReq->pCont, pReq->contLen, &compactReq) != 0) { @@ -1275,12 +1252,7 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto _OVER; - } - - if (mndCheckAlterDropCompactDbAuth(pUser, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { goto _OVER; } @@ -1292,8 +1264,6 @@ _OVER: } mndReleaseDb(pMnode, pDb); - mndReleaseUser(pMnode, pUser); - return code; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index c936c0f93d..53cbb9b669 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -499,7 +499,6 @@ _OVER: static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; - SUserObj *pUser = NULL; SDnodeObj *pDnode = NULL; SCreateDnodeReq createReq = {0}; @@ -522,13 +521,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckNodeAuth(pUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_DNODE) != 0) { goto _OVER; } @@ -541,7 +534,6 @@ _OVER: } mndReleaseDnode(pMnode, pDnode); - mndReleaseUser(pMnode, pUser); return code; } @@ -586,7 +578,6 @@ _OVER: static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; - SUserObj *pUser = NULL; SDnodeObj *pDnode = NULL; SMnodeObj *pMObj = NULL; SMDropMnodeReq dropReq = {0}; @@ -631,13 +622,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckNodeAuth(pUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } @@ -650,7 +635,6 @@ _OVER: } mndReleaseDnode(pMnode, pDnode); - mndReleaseUser(pMnode, pUser); mndReleaseMnode(pMnode, pMObj); return code; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 7e5dbb9566..dfdc0a3c1a 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -274,7 +274,6 @@ _OVER: static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; - SUserObj *pUser = NULL; SFuncObj *pFunc = NULL; SCreateFuncReq createReq = {0}; @@ -309,23 +308,17 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { goto _OVER; } - if (createReq.codeLen <= 1) { - terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; - goto _OVER; - } + if (createReq.codeLen <= 1) { + terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; + goto _OVER; + } if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckFuncAuth(pUser)) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_FUNC) != 0) { goto _OVER; } @@ -338,16 +331,13 @@ _OVER: } mndReleaseFunc(pMnode, pFunc); - mndReleaseUser(pMnode, pUser); tFreeSCreateFuncReq(&createReq); - return code; } static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; - SUserObj *pUser = NULL; SFuncObj *pFunc = NULL; SDropFuncReq dropReq = {0}; @@ -375,13 +365,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckFuncAuth(pUser)) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_FUNC) != 0) { goto _OVER; } @@ -394,8 +378,6 @@ _OVER: } mndReleaseFunc(pMnode, pFunc); - mndReleaseUser(pMnode, pUser); - return code; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index f6cef945e2..7ec490d52f 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -381,7 +381,6 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { int32_t code = -1; SMnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; - SUserObj *pUser = NULL; SMCreateMnodeReq createReq = {0}; if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { @@ -415,13 +414,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckNodeAuth(pUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_MNODE) != 0) { goto _OVER; } @@ -435,7 +428,6 @@ _OVER: mndReleaseMnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); - mndReleaseUser(pMnode, pUser); return code; } @@ -594,7 +586,6 @@ _OVER: static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; - SUserObj *pUser = NULL; SMnodeObj *pObj = NULL; SMDropMnodeReq dropReq = {0}; @@ -630,13 +621,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckNodeAuth(pUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } @@ -649,8 +634,6 @@ _OVER: } mndReleaseMnode(pMnode, pObj); - mndReleaseUser(pMnode, pUser); - return code; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index aac6eaba47..595287a3af 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -271,7 +271,6 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { int32_t code = -1; SQnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; - SUserObj *pUser = NULL; SMCreateQnodeReq createReq = {0}; if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { @@ -295,13 +294,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckNodeAuth(pUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_QNODE) != 0) { goto _OVER; } @@ -315,7 +308,6 @@ _OVER: mndReleaseQnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); - mndReleaseUser(pMnode, pUser); return code; } @@ -384,7 +376,6 @@ _OVER: static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; - SUserObj *pUser = NULL; SQnodeObj *pObj = NULL; SMDropQnodeReq dropReq = {0}; @@ -405,13 +396,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckNodeAuth(pUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_QNODE) != 0) { goto _OVER; } @@ -424,8 +409,6 @@ _OVER: } mndReleaseQnode(pMnode, pObj); - mndReleaseUser(pMnode, pUser); - return code; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 6e569a04cc..d312955202 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "mndShow.h" #include "systable.h" +#include "mndAuth.h" #define SHOW_STEP_SIZE 100 @@ -228,6 +229,8 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); + // if (mndCheckShowAuth(pMnode, pReq->conn.user, pShow->type) != 0) return -1; + int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index a14eb78ffe..fa36670dbc 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -628,7 +628,6 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { SSmaObj *pSma = NULL; SStreamObj *pStream = NULL; SDbObj *pDb = NULL; - SUserObj *pUser = NULL; SMCreateSmaReq createReq = {0}; if (tDeserializeSMCreateSmaReq(pReq->pCont, pReq->contLen, &createReq) != 0) { @@ -672,12 +671,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto _OVER; - } - - if (mndCheckWriteAuth(pUser, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -693,7 +687,6 @@ _OVER: mndReleaseSma(pMnode, pSma); mndReleaseStream(pMnode, pStream); mndReleaseDb(pMnode, pDb); - mndReleaseUser(pMnode, pUser); tFreeSMCreateSmaReq(&createReq); return code; @@ -908,7 +901,6 @@ _OVER: static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; - SUserObj *pUser = NULL; SDbObj *pDb = NULL; SSmaObj *pSma = NULL; SMDropSmaReq dropReq = {0}; @@ -938,12 +930,7 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto _OVER; - } - - if (mndCheckWriteAuth(pUser, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -956,9 +943,6 @@ _OVER: } mndReleaseDb(pMnode, pDb); - mndReleaseSma(pMnode, pSma); - mndReleaseUser(pMnode, pUser); - return code; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 7d21528260..c84dc2f3dd 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -277,7 +277,6 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { int32_t code = -1; SSnodeObj *pObj = NULL; SDnodeObj *pDnode = NULL; - SUserObj *pUser = NULL; SMCreateSnodeReq createReq = {0}; if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { @@ -301,13 +300,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckNodeAuth(pUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_SNODE) != 0) { goto _OVER; } @@ -322,7 +315,6 @@ _OVER: mndReleaseSnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); - mndReleaseUser(pMnode, pUser); return code; } @@ -392,7 +384,6 @@ _OVER: static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; - SUserObj *pUser = NULL; SSnodeObj *pObj = NULL; SMDropSnodeReq dropReq = {0}; @@ -413,13 +404,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckNodeAuth(pUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_SNODE) != 0) { goto _OVER; } @@ -432,8 +417,6 @@ _OVER: } mndReleaseSnode(pMnode, pObj); - mndReleaseUser(pMnode, pUser); - return code; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3e91bfa926..1abf31729b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -769,7 +769,6 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { int32_t code = -1; SStbObj *pStb = NULL; SDbObj *pDb = NULL; - SUserObj *pUser = NULL; SMCreateStbReq createReq = {0}; if (tDeserializeSMCreateStbReq(pReq->pCont, pReq->contLen, &createReq) != 0) { @@ -803,12 +802,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto _OVER; - } - - if (mndCheckWriteAuth(pUser, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -832,7 +826,6 @@ _OVER: mndReleaseStb(pMnode, pStb); mndReleaseDb(pMnode, pDb); - mndReleaseUser(pMnode, pUser); tFreeSMCreateStbReq(&createReq); return code; @@ -1427,7 +1420,6 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { int32_t code = -1; SDbObj *pDb = NULL; SStbObj *pStb = NULL; - SUserObj *pUser = NULL; SMAlterStbReq alterReq = {0}; if (tDeserializeSMAlterStbReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { @@ -1458,12 +1450,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto _OVER; - } - - if (mndCheckWriteAuth(pUser, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1477,7 +1464,6 @@ _OVER: mndReleaseStb(pMnode, pStb); mndReleaseDb(pMnode, pDb); - mndReleaseUser(pMnode, pUser); taosArrayDestroy(alterReq.pFields); return code; @@ -1565,7 +1551,6 @@ _OVER: static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; - SUserObj *pUser = NULL; SDbObj *pDb = NULL; SStbObj *pStb = NULL; SMDropStbReq dropReq = {0}; @@ -1595,12 +1580,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto _OVER; - } - - if (mndCheckWriteAuth(pUser, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1614,8 +1594,6 @@ _OVER: mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); - mndReleaseUser(pMnode, pUser); - return code; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 96d199fcb6..d1bc059d52 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -293,7 +293,6 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { SStbObj *pStb = NULL; SDbObj *pDb = NULL; - SUserObj *pUser = NULL; SMCreateStbReq createReq = {0}; tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); @@ -335,12 +334,8 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre goto _OVER; } - pUser = mndAcquireUser(pMnode, user); - if (pUser == NULL) { - goto _OVER; - } - if (mndCheckWriteAuth(pUser, pDb) != 0) { + if (mndCheckDbAuth(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -368,7 +363,6 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre _OVER: mndReleaseStb(pMnode, pStb); mndReleaseDb(pMnode, pDb); - mndReleaseUser(pMnode, pUser); return -1; } @@ -436,19 +430,18 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { int32_t code = -1; SStreamObj *pStream = NULL; SDbObj *pDb = NULL; - SUserObj *pUser = NULL; SCMCreateStreamReq createStreamReq = {0}; if (tDeserializeSCMCreateStreamReq(pReq->pCont, pReq->contLen, &createStreamReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto CREATE_STREAM_OVER; + goto _OVER; } mDebug("stream:%s, start to create, sql:%s", createStreamReq.name, createStreamReq.sql); if (mndCheckCreateStreamReq(&createStreamReq) != 0) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); - goto CREATE_STREAM_OVER; + goto _OVER; } pStream = mndAcquireStream(pMnode, createStreamReq.name); @@ -456,41 +449,35 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { if (createStreamReq.igExists) { mDebug("stream:%s, already exist, ignore exist is set", createStreamReq.name); code = 0; - goto CREATE_STREAM_OVER; + goto _OVER; } else { terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; - goto CREATE_STREAM_OVER; + goto _OVER; } } else if (terrno != TSDB_CODE_MND_STREAM_NOT_EXIST) { - goto CREATE_STREAM_OVER; + goto _OVER; } pDb = mndAcquireDb(pMnode, createStreamReq.sourceDB); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - goto CREATE_STREAM_OVER; + goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto CREATE_STREAM_OVER; - } - - if (mndCheckWriteAuth(pUser, pDb) != 0) { - goto CREATE_STREAM_OVER; + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + goto _OVER; } code = mndCreateStream(pMnode, pReq, &createStreamReq, pDb); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; -CREATE_STREAM_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); } mndReleaseStream(pMnode, pStream); mndReleaseDb(pMnode, pDb); - mndReleaseUser(pMnode, pUser); tFreeSCMCreateStreamReq(&createStreamReq); return code; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 3247008d58..4c2730ce94 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -387,7 +387,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * return -1; } - if (nodesNodeToString((SNode*)pPlan, false, &topicObj.physicalPlan, NULL) != 0) { + if (nodesNodeToString((SNode *)pPlan, false, &topicObj.physicalPlan, NULL) != 0) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); taosMemoryFree(topicObj.ast); taosMemoryFree(topicObj.sql); @@ -440,19 +440,18 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { int32_t code = -1; SMqTopicObj *pTopic = NULL; SDbObj *pDb = NULL; - SUserObj *pUser = NULL; SCMCreateTopicReq createTopicReq = {0}; if (tDeserializeSCMCreateTopicReq(pReq->pCont, pReq->contLen, &createTopicReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto CREATE_TOPIC_OVER; + goto _OVER; } mDebug("topic:%s, start to create, sql:%s", createTopicReq.name, createTopicReq.sql); if (mndCheckCreateTopicReq(&createTopicReq) != 0) { mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); - goto CREATE_TOPIC_OVER; + goto _OVER; } pTopic = mndAcquireTopic(pMnode, createTopicReq.name); @@ -460,41 +459,35 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { if (createTopicReq.igExists) { mDebug("topic:%s, already exist, ignore exist is set", createTopicReq.name); code = 0; - goto CREATE_TOPIC_OVER; + goto _OVER; } else { terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST; - goto CREATE_TOPIC_OVER; + goto _OVER; } } else if (terrno != TSDB_CODE_MND_TOPIC_NOT_EXIST) { - goto CREATE_TOPIC_OVER; + goto _OVER; } pDb = mndAcquireDb(pMnode, createTopicReq.subDbName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - goto CREATE_TOPIC_OVER; + goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto CREATE_TOPIC_OVER; - } - - if (mndCheckWriteAuth(pUser, pDb) != 0) { - goto CREATE_TOPIC_OVER; + if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + goto _OVER; } code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; -CREATE_TOPIC_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); } mndReleaseTopic(pMnode, pTopic); mndReleaseDb(pMnode, pDb); - mndReleaseUser(pMnode, pUser); tFreeSCMCreateTopicReq(&createTopicReq); return code; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1ec4799419..2e9124e514 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1364,7 +1364,6 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SKillTransReq killReq = {0}; int32_t code = -1; - SUserObj *pUser = NULL; STrans *pTrans = NULL; if (tDeserializeSKillTransReq(pReq->pCont, pReq->contLen, &killReq) != 0) { @@ -1374,12 +1373,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) { mInfo("trans:%d, start to kill", killReq.transId); - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - goto _OVER; - } - - if (mndCheckTransAuth(pUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_KILL_TRANS) != 0) { goto _OVER; } @@ -1395,7 +1389,6 @@ _OVER: mError("trans:%d, failed to kill since %s", killReq.transId, terrstr()); } - mndReleaseUser(pMnode, pUser); mndReleaseTrans(pMnode, pTrans); return code; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index a86b9c904e..9590823106 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -360,7 +360,7 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckCreateUserAuth(pOperUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_USER) != 0) { goto _OVER; } @@ -623,7 +623,6 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SUserObj *pUser = NULL; - SUserObj *pOperUser = NULL; SDropUserReq dropReq = {0}; if (tDeserializeSDropUserReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { @@ -644,13 +643,7 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { goto _OVER; } - pOperUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pOperUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckDropUserAuth(pOperUser) != 0) { + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_USER) != 0) { goto _OVER; } @@ -662,9 +655,7 @@ _OVER: mError("user:%s, failed to drop since %s", dropReq.user, terrstr()); } - mndReleaseUser(pMnode, pOperUser); mndReleaseUser(pMnode, pUser); - return code; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index cd1d930846..755f4ef0b2 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1177,7 +1177,6 @@ _OVER: static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - SUserObj *pUser = NULL; SDnodeObj *pNew1 = NULL; SDnodeObj *pNew2 = NULL; SDnodeObj *pNew3 = NULL; @@ -1200,13 +1199,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } mInfo("vgId:%d, start to redistribute to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - if (mndCheckNodeAuth(pUser) != 0) goto _OVER; + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) goto _OVER; pVgroup = mndAcquireVgroup(pMnode, req.vgId); if (pVgroup == NULL) goto _OVER; @@ -1368,7 +1362,6 @@ _OVER: mndReleaseDnode(pMnode, pOld1); mndReleaseDnode(pMnode, pOld2); mndReleaseDnode(pMnode, pOld3); - mndReleaseUser(pMnode, pUser); mndReleaseVgroup(pMnode, pVgroup); mndReleaseDb(pMnode, pDb); @@ -1493,12 +1486,11 @@ _OVER: } static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - int32_t vgId = 2; - SUserObj *pUser = NULL; - SVgObj *pVgroup = NULL; - SDbObj *pDb = NULL; + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + int32_t vgId = 2; + SVgObj *pVgroup = NULL; + SDbObj *pDb = NULL; mDebug("vgId:%d, start to split", vgId); @@ -1508,19 +1500,12 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { pDb = mndAcquireDb(pMnode, pVgroup->dbName); if (pDb == NULL) goto _OVER; - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - - if (mndCheckNodeAuth(pUser) != 0) goto _OVER; + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_SPLIT_VGROUP) != 0) goto _OVER; code = mndSplitVgroup(pMnode, pReq, pDb, pVgroup); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - mndReleaseUser(pMnode, pUser); mndReleaseVgroup(pMnode, pVgroup); mndReleaseDb(pMnode, pDb); return code; @@ -1631,21 +1616,15 @@ _OVER: } static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SUserObj *pUser = NULL; - SArray *pArray = NULL; - void *pIter = NULL; - int64_t curMs = taosGetTimestampMs(); + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SArray *pArray = NULL; + void *pIter = NULL; + int64_t curMs = taosGetTimestampMs(); mDebug("start to balance vgroup"); - pUser = mndAcquireUser(pMnode, pReq->conn.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto _OVER; - } - if (mndCheckNodeAuth(pUser) != 0) goto _OVER; + if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_BALANCE_VGROUP) != 0) goto _OVER; while (1) { SDnodeObj *pDnode = NULL; @@ -1676,7 +1655,6 @@ _OVER: mError("failed to balance vgroup since %s", terrstr()); } - mndReleaseUser(pMnode, pUser); taosArrayDestroy(pArray); return code; } \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 079d5ef590..8694f760da 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -137,6 +137,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt cl TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Mnode not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_DISABLED, "User is disabled") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, "Invalid message connection") // mnode-show diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index 4d59129b91..ce8ac6941b 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -246,20 +246,26 @@ class TDTestCase: user = self.root_user with taos_connect(user=user.name, passwd=user.passwd) as use: time.sleep(2) - use.query("use db") - use.query("show tables") if check_priv == PRIVILEGES_ALL: + use.query("use db") + use.query("show tables") use.query("select * from ct1") use.query("insert into t1 (ts) values (now())") elif check_priv == PRIVILEGES_READ: + use.query("use db") + use.query("show tables") use.query("select * from ct1") use.error("insert into t1 (ts) values (now())") elif check_priv == PRIVILEGES_WRITE: + use.query("use db") + use.query("show tables") use.error("select * from ct1") use.query("insert into t1 (ts) values (now())") elif check_priv is None: - use.error("select * from ct1") - use.error("insert into t1 (ts) values (now())") + use.error("use db") + use.error("show tables") + use.error("select * from db.ct1") + use.error("insert into db.t1 (ts) values (now())") def __change_user_priv(self, user: User, pre_priv, invoke=False): if user.priv == pre_priv and invoke : @@ -610,7 +616,7 @@ class TDTestCase: tdLog.printNoPrefix("==========step0: init, user list only has root account") tdSql.query("show users") tdSql.checkData(0, 0, "root") - tdSql.checkData(0, 1, "super") + tdSql.checkData(0, 1, "1") # root用户权限 # 创建用户测试 @@ -676,7 +682,7 @@ class TDTestCase: tdSql.query("show users") tdSql.checkRows(1) tdSql.checkData(0, 0, "root") - tdSql.checkData(0, 1, "super") + tdSql.checkData(0, 1, "1") tdDnodes.stop(1) tdDnodes.start(1) @@ -690,7 +696,7 @@ class TDTestCase: tdSql.query("show users") tdSql.checkRows(1) tdSql.checkData(0, 0, "root") - tdSql.checkData(0, 1, "super") + tdSql.checkData(0, 1, "1") def stop(self):