From eec6b668772e32983d4b54e32c3c51400a2514cb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 28 Feb 2024 17:05:11 +0800 Subject: [PATCH 01/54] fix:[TD-28869]get error in askEp because consumer is dropped when unsubscribe topic --- source/client/src/clientTmq.c | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 9b74456da2..3ce4e97e22 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1009,19 +1009,8 @@ int32_t tmq_unsubscribe(tmq_t* tmq) { } taosSsleep(2); // sleep 2s for hb to send offset and rows to server - int32_t rsp; - int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); - while (1) { - rsp = tmq_subscribe(tmq, lst); - if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) { - break; - } else { - retryCnt++; - taosMsleep(500); - } - } - + int32_t rsp = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); return rsp; } @@ -1271,10 +1260,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } int32_t retryCnt = 0; - while (syncAskEp(tmq) != 0) { - if (retryCnt++ > MAX_RETRY_COUNT) { + while ((code = syncAskEp(tmq)) != 0) { + if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes", tmq->consumerId); - code = TSDB_CODE_MND_CONSUMER_NOT_READY; goto FAIL; } @@ -2154,18 +2142,8 @@ int32_t tmq_consumer_close(tmq_t* tmq) { } taosSsleep(2); // sleep 2s for hb to send offset and rows to server - int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); - while (1) { - int32_t rsp = tmq_subscribe(tmq, lst); - if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) { - break; - } else { - retryCnt++; - taosMsleep(500); - } - } - + tmq_subscribe(tmq, lst); tmq_list_destroy(lst); } else { tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); From 690c82a49eac51f83edde9f7f4378dc38706816f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 28 Feb 2024 20:48:42 +0800 Subject: [PATCH 02/54] test: Splitting test cases to avoid case timeouts --- tests/parallel_test/cases.task | 3 +- ...m.py => tmqVnodeTransform-db-removewal.py} | 214 ++++++-------- .../7-tmq/tmqVnodeTransform-stb-removewal.py | 266 ++++++++++++++++++ 3 files changed, 350 insertions(+), 133 deletions(-) rename tests/system-test/7-tmq/{tmqVnodeTransform.py => tmqVnodeTransform-db-removewal.py} (78%) create mode 100644 tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bdccf33c32..dc9286cffe 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -238,7 +238,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True -,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3 #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3 diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py similarity index 78% rename from tests/system-test/7-tmq/tmqVnodeTransform.py rename to tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py index c2b002ead6..a853489c3f 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py @@ -122,135 +122,7 @@ class TDTestCase: tdLog.debug(f"redistributeSql:{redistributeSql}") tdSql.query(redistributeSql) tdLog.debug("redistributeSql ok") - - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 1, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ctb', - 'ctbStartIdx': 0, - 'ctbNum': 10, - 'rowsPerTbl': 1000, - 'batchNum': 10, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - topicNameList = ['topic1'] - # expectRowsList = [] - tmqCom.initConsumerTable() - - tdLog.info("create topics from stb with filter") - queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) - # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - # tdSql.query(queryString) - # expectRowsList.append(tdSql.getRows()) - - # init consume info, and start tmq_sim, then check consume result - tdLog.info("insert consume info to consume processor") - consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 - topicList = topicNameList[0] - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") - - tdLog.info("create ctb1") - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) - - tmqCom.getStartConsumeNotifyFromTmqsim() - tmqCom.getStartCommitNotifyFromTmqsim() - - #restart dnode & remove wal - self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups(); - - tdLog.info("create ctb2") - paraDict['ctbPrefix'] = "ctbn" - tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], - ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("insert data") - pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) - pInsertThread.join() - pInsertThread1.join() - - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - - if expectrowcnt / 2 > resultList[0]: - tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) - tdLog.exit("%d tmq consume rows error!"%consumerId) - - # tmqCom.checkFileContent(consumerId, queryString) - - time.sleep(10) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 1 end ...... ") - - def tmqCase2(self): - tdLog.printNoPrefix("======== test case 2: ") - paraDict = {'dbName':'dbt'} - - ntbName = "ntb" - - topicNameList = ['topic2'] - tmqCom.initConsumerTable() - - sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName) - tdLog.info("create nomal table sql: %s"%sqlString) - tdSql.execute(sqlString) - - tdLog.info("create topics from nomal table") - queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) - sqlString = "create topic %s as %s" %(topicNameList[0], queryString) - tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) - tdSql.query("flush database %s"%(paraDict['dbName'])) - #restart dnode & remove wal - self.restartAndRemoveWal() - - # redistribute vgroup - self.redistributeVgroups(); - - sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) - tdLog.info("alter table sql: %s"%sqlString) - tdSql.error(sqlString) - expectRows = 0 - resultList = tmqCom.selectConsumeResult(expectRows) - time.sleep(1) - for i in range(len(topicNameList)): - tdSql.query("drop topic %s"%topicNameList[i]) - - tdLog.printNoPrefix("======== test case 2 end ...... ") - + def tmqCase3(self): tdLog.printNoPrefix("======== test case 3: ") paraDict = {'dbName': 'dbt', @@ -330,12 +202,90 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 3 end ...... ") + def tmqCaseDbname(self): + tdLog.printNoPrefix("======== test case 4 subscrib Dbname start: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stbn', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic4'] + tmqCom.initConsumerTable() + + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create topics from database ") + queryString = "database %s "%(paraDict['dbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + time.sleep(1) + # restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups() + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + + time.sleep(6) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ") + def run(self): - self.prepareTestEnv() - self.tmqCase1() - self.tmqCase2() self.prepareTestEnv() self.tmqCase3() + self.prepareTestEnv() + self.tmqCaseDbname() def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py new file mode 100644 index 0000000000..40879d5c66 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py @@ -0,0 +1,266 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 1000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + # tdLog.info("create ctb") + # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + # ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + # tdLog.info("insert data") + # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def restartAndRemoveWal(self): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + os.system('rm -rf ' + dataPath) + tdLog.debug("dataPath:%s"%dataPath) + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def redistributeVgroups(self): + dnodesList = [] + tdSql.query("show dnodes") + for result in tdSql.queryResult: + dnodesList.append(result[0]) + print("dnodeList:",dnodesList) + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodesList.remove(result[0]) + vnodeId = result[1] + print("its all data",dnodesList) + # if self.replicaVar == 1: + # redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0]) + # else: + redistributeSql = f"redistribute vgroup {vnodeId} " + for vgdnode in dnodesList: + redistributeSql += f"dnode {vgdnode} " + print(redistributeSql) + + tdLog.debug(f"redistributeSql:{redistributeSql}") + tdSql.query(redistributeSql) + tdLog.debug("redistributeSql ok") + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups(); + + tdLog.info("create ctb2") + paraDict['ctbPrefix'] = "ctbn" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 > resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName':'dbt'} + + ntbName = "ntb" + + topicNameList = ['topic2'] + tmqCom.initConsumerTable() + + sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName) + tdLog.info("create nomal table sql: %s"%sqlString) + tdSql.execute(sqlString) + + tdLog.info("create topics from nomal table") + queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query("flush database %s"%(paraDict['dbName'])) + #restart dnode & remove wal + self.restartAndRemoveWal() + + # redistribute vgroup + self.redistributeVgroups(); + + sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName) + tdLog.info("alter table sql: %s"%sqlString) + tdSql.error(sqlString) + expectRows = 0 + resultList = tmqCom.selectConsumeResult(expectRows) + time.sleep(1) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From d37adfbd859f967df7830040f0c9660ba42386f6 Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Thu, 29 Feb 2024 10:31:28 +0800 Subject: [PATCH 03/54] docs: fix python demo code --- docs/en/08-client-libraries/07-python.mdx | 14 +++++++------- docs/zh/08-connector/30-python.mdx | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/en/08-client-libraries/07-python.mdx b/docs/en/08-client-libraries/07-python.mdx index aacfd0fe53..556bb84042 100644 --- a/docs/en/08-client-libraries/07-python.mdx +++ b/docs/en/08-client-libraries/07-python.mdx @@ -887,13 +887,13 @@ The `poll` function is used to consume data in tmq. The parameter of the `poll` ```python while True: - res = consumer.poll(1) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() + err = message.error() if err is not None: raise err - val = res.value() + val = message.value() for block in val: print(block.fetchall()) @@ -906,10 +906,10 @@ The `poll` function is used to consume data in tmq. The parameter of the `poll` ```python while True: - res = consumer.poll(timeout=1.0) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() + err = message.error() if err is not None: raise err for block in message: diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index e7160ab094..d49c96b69f 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -896,13 +896,13 @@ Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 flo ```python while True: - res = consumer.poll(1) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() + err = message.error() if err is not None: raise err - val = res.value() + val = message.value() for block in val: print(block.fetchall()) @@ -915,10 +915,10 @@ Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 flo ```python while True: - res = consumer.poll(timeout=1.0) - if not res: + message = consumer.poll(1) + if not message: continue - err = res.error() + err = message.error() if err is not None: raise err for block in message: From 7f7b61af5cc9b58c1b60cf023bbae4c5cd23743d Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Tue, 20 Feb 2024 10:03:29 +0800 Subject: [PATCH 04/54] docs: fix connector-python create consumer --- docs/en/08-client-libraries/07-python.mdx | 4 ++-- docs/zh/08-connector/30-python.mdx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/08-client-libraries/07-python.mdx b/docs/en/08-client-libraries/07-python.mdx index 556bb84042..82b889e0d3 100644 --- a/docs/en/08-client-libraries/07-python.mdx +++ b/docs/en/08-client-libraries/07-python.mdx @@ -842,12 +842,12 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) In addition to native connections, the client library also supports subscriptions via websockets. -The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer). +The syntax for creating a consumer is "consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer). ```python import taosws -consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) ``` diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index d49c96b69f..491566e4e3 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -856,7 +856,7 @@ taosws `Consumer` API 提供了基于 Websocket 订阅 TMQ 数据的 API。创 ```python import taosws -consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) +consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) ``` From d5eda8c0a9368bf111f2174883515fe1023b9b5c Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 29 Feb 2024 10:42:27 +0800 Subject: [PATCH 05/54] enh: adjust an error logging msg in process sync msg --- source/dnode/mnode/impl/src/mndMain.c | 3 ++- source/dnode/vnode/src/vnd/vnodeSync.c | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index b188d314d9..69a0bd477d 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -709,7 +709,8 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { int32_t code = syncProcessMsg(pMgmt->sync, pMsg); if (code != 0) { - mGError("vgId:1, failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); + mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType), + terrstr(), code); } return code; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 5f4b7b8442..f9f2ae6b21 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -372,8 +372,8 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int32_t code = syncProcessMsg(pVnode->sync, pMsg); if (code != 0) { - vGError("vgId:%d, failed to process sync msg:%p type:%s since %s", pVnode->config.vgId, pMsg, - TMSG_INFO(pMsg->msgType), terrstr()); + vGError("vgId:%d, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pVnode->config.vgId, pMsg, + TMSG_INFO(pMsg->msgType), terrstr(), code); } return code; From d4900d06339cd2085663d133421f2dfbe7a2fe6e Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Thu, 29 Feb 2024 10:42:12 +0800 Subject: [PATCH 06/54] docs: fix python tmq demo --- docs/en/08-client-libraries/07-python.mdx | 6 ++---- docs/zh/08-connector/30-python.mdx | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/en/08-client-libraries/07-python.mdx b/docs/en/08-client-libraries/07-python.mdx index 82b889e0d3..3110afcf10 100644 --- a/docs/en/08-client-libraries/07-python.mdx +++ b/docs/en/08-client-libraries/07-python.mdx @@ -902,16 +902,14 @@ while True: -The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data. +The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. ```python while True: message = consumer.poll(1) if not message: continue - err = message.error() - if err is not None: - raise err + for block in message: for row in block: print(row) diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index 491566e4e3..71dc82316e 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -911,16 +911,14 @@ while True: -Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。 +Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。 ```python while True: message = consumer.poll(1) if not message: continue - err = message.error() - if err is not None: - raise err + for block in message: for row in block: print(row) From e17832e27eb5cdb0860504111fba46d4ba28e6d7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 29 Feb 2024 14:38:09 +0800 Subject: [PATCH 07/54] fix:[TD-28869]get error in askEp because consumer is dropped when unsubscribe --- source/client/src/clientTmq.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 3ce4e97e22..11e88dc306 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2135,16 +2135,19 @@ int32_t tmq_consumer_close(tmq_t* tmq) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) { // if auto commit is set, commit before close consumer. Otherwise, do nothing. if (tmq->autoCommit) { - int32_t rsp = tmq_commit_sync(tmq, NULL); - if (rsp != 0) { - return rsp; + int32_t code = tmq_commit_sync(tmq, NULL); + if (code != 0) { + return code; } } taosSsleep(2); // sleep 2s for hb to send offset and rows to server tmq_list_t* lst = tmq_list_new(); - tmq_subscribe(tmq, lst); + int32_t code = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); + if (code != 0) { + return code; + } } else { tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); } From 4e67da70175cc9fc9d766cd877124507d91db5dc Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 29 Feb 2024 07:24:29 +0000 Subject: [PATCH 08/54] fix/TD-28920 --- include/util/tdef.h | 2 ++ source/dnode/mnode/impl/src/mndCompact.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 8371ee24bb..e2d1beb5a5 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -187,6 +187,8 @@ typedef enum ELogicConditionType { LOGIC_COND_TYPE_NOT, } ELogicConditionType; +#define TSDB_INT32_ID_LEN 11 + #define TSDB_NAME_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 83254b0f73..2de968f90a 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -454,7 +454,7 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){ code = TSDB_CODE_ACTION_IN_PROGRESS; - char obj[MND_COMPACT_ID_LEN] = {0}; + char obj[TSDB_INT32_ID_LEN] = {0}; sprintf(obj, "%d", pCompact->compactId); auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen); From 474ec01cdb2018cd85d7aed93f3f0e1b790cb1fb Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 29 Feb 2024 18:27:34 +0800 Subject: [PATCH 09/54] delete unused code --- include/common/tmsg.h | 14 +-- source/common/src/tmsg.c | 227 ++++++++++++++++++++------------------- 2 files changed, 121 insertions(+), 120 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 36242ddea8..958789178a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -581,8 +581,8 @@ typedef struct { }; } SSubmitRsp; -int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); -int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); +// int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); +// int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); // void tFreeSSubmitBlkRsp(void* param); void tFreeSSubmitRsp(SSubmitRsp* pRsp); @@ -885,8 +885,8 @@ typedef struct { int64_t maxStorage; } SCreateAcctReq, SAlterAcctReq; -int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); -int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); +// int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); +// int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq); typedef struct { char user[TSDB_USER_LEN]; @@ -3446,7 +3446,7 @@ int32_t tDeserializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexR typedef SMDropSmaReq SDropTagIndexReq; -int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); +// int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); int32_t tDeserializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); typedef struct { @@ -3567,8 +3567,8 @@ typedef struct { int8_t igNotExists; } SMDropFullTextReq; -int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); -int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); +// int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); +// int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq); typedef struct { char indexFName[TSDB_INDEX_FNAME_LEN]; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4640c20e07..5eb62c6389 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1009,19 +1009,19 @@ int32_t tDeserializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexR tDecoderClear(&decoder); return 0; } -int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - tEndEncode(&encoder); +// int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); +// if (tStartEncode(&encoder) < 0) return -1; +// tEndEncode(&encoder); - if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -1035,6 +1035,7 @@ int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq * return 0; } + int32_t tSerializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -1059,32 +1060,32 @@ void tFreeSMCreateFullTextReq(SMCreateFullTextReq *pReq) { // impl later return; } -int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; +// if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; - if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; +// if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; - tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} -int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; +// tEndEncode(&encoder); +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } +// int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; +// if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; - tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tEndDecode(&decoder); +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) { SEncoder encoder = {0}; @@ -1474,44 +1475,44 @@ void tFreeSStatisReq(SStatisReq *pReq) { taosMemoryFreeClear(pReq->pCont); } -int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); +// int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +// SEncoder encoder = {0}; +// tEncoderInit(&encoder, buf, bufLen); - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1; - if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1; - if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1; - if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1; - tEndEncode(&encoder); +// if (tStartEncode(&encoder) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; +// if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1; +// if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1; +// if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1; +// tEndEncode(&encoder); - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} +// int32_t tlen = encoder.pos; +// tEncoderClear(&encoder); +// return tlen; +// } -int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); +// int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { +// SDecoder decoder = {0}; +// tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1; - tEndDecode(&decoder); +// if (tStartDecode(&decoder) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; +// if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1; +// if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1; +// if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1; +// tEndDecode(&decoder); - tDecoderClear(&decoder); - return 0; -} +// tDecoderClear(&decoder); +// return 0; +// } int32_t tSerializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) { SEncoder encoder = {0}; @@ -7934,64 +7935,64 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl return 0; } -static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { - if (tStartDecode(pDecoder) < 0) return -1; +// static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) { +// if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; - if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; - pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); - if (NULL == pBlock->tblFName) return -1; - if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; - if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; - if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; +// if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1; +// if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1; +// pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1); +// if (NULL == pBlock->tblFName) return -1; +// if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1; +// if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1; - int32_t meta = 0; - if (tDecodeI32(pDecoder, &meta) < 0) return -1; - if (meta) { - pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); - if (NULL == pBlock->pMeta) return -1; - if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1; - } else { - pBlock->pMeta = NULL; - } +// int32_t meta = 0; +// if (tDecodeI32(pDecoder, &meta) < 0) return -1; +// if (meta) { +// pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); +// if (NULL == pBlock->pMeta) return -1; +// if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1; +// } else { +// pBlock->pMeta = NULL; +// } - tEndDecode(pDecoder); - return 0; -} +// tEndDecode(pDecoder); +// return 0; +// } -int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) { - int32_t nBlocks = taosArrayGetSize(pRsp->pArray); +// int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) { +// int32_t nBlocks = taosArrayGetSize(pRsp->pArray); - if (tStartEncode(pEncoder) < 0) return -1; +// if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1; - if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1; - if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1; - for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { - if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1; - } +// if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1; +// if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1; +// if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1; +// for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { +// if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1; +// } - tEndEncode(pEncoder); - return 0; -} +// tEndEncode(pEncoder); +// return 0; +// } -int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { - if (tStartDecode(pDecoder) < 0) return -1; +// int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) { +// if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1; - if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1; - pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks)); - if (pRsp->pBlocks == NULL) return -1; - for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) { - if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1; - } +// if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1; +// if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1; +// pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks)); +// if (pRsp->pBlocks == NULL) return -1; +// for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) { +// if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1; +// } - tEndDecode(pDecoder); - tDecoderClear(pDecoder); - return 0; -} +// tEndDecode(pDecoder); +// tDecoderClear(pDecoder); +// return 0; +// } // void tFreeSSubmitBlkRsp(void *param) { // if (NULL == param) { From a0c77f5baafeeb44ea9588da2577c35368b2b6ad Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 29 Feb 2024 18:40:21 +0800 Subject: [PATCH 10/54] fix mem leak while taosd quit --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 9 +++--- source/libs/transport/inc/transComm.h | 30 +++++++++---------- source/libs/transport/src/transCli.c | 18 +++++++++-- source/libs/transport/src/transSvr.c | 7 +++-- 4 files changed, 40 insertions(+), 24 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 1a31f08801..c54da860dd 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -87,9 +87,9 @@ static bool dmIsForbiddenIp(int8_t forbidden, char *user, uint32_t clientIp) { } } static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { - SDnodeTrans *pTrans = &pDnode->trans; + SDnodeTrans * pTrans = &pDnode->trans; int32_t code = -1; - SRpcMsg *pMsg = NULL; + SRpcMsg * pMsg = NULL; SMgmtWrapper *pWrapper = NULL; SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; @@ -256,11 +256,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; - SArray *pArray = (*pWrapper->func.getHandlesFp)(); + SArray * pArray = (*pWrapper->func.getHandlesFp)(); if (pArray == NULL) return -1; for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SMgmtHandle *pMgmt = taosArrayGet(pArray, i); + SMgmtHandle * pMgmt = taosArrayGet(pArray, i); SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)]; if (pMgmt->needCheckVgId) { pHandle->needCheckVgId = pMgmt->needCheckVgId; @@ -345,6 +345,7 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.parent = pDnode; rpcInit.rfp = rpcRfp; rpcInit.compressSize = tsCompressMsgSize; + rpcInit.dfp = destroyAhandle; rpcInit.retryMinInterval = tsRedirectPeriod; rpcInit.retryStepFactor = tsRedirectFactor; diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 5b18d56d70..c010e31320 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -256,21 +256,21 @@ void transAsyncPoolDestroy(SAsyncPool* pool); int transAsyncSend(SAsyncPool* pool, queue* mq); bool transAsyncPoolIsEmpty(SAsyncPool* pool); -#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc) \ - do { \ - for (int i = 0; i < pool->nAsync; i++) { \ - uv_async_t* async = &(pool->asyncs[i]); \ - SAsyncItem* item = async->data; \ - while (!QUEUE_IS_EMPTY(&item->qmsg)) { \ - tTrace("destroy msg in async pool "); \ - queue* h = QUEUE_HEAD(&item->qmsg); \ - QUEUE_REMOVE(h); \ - msgType* msg = QUEUE_DATA(h, msgType, q); \ - if (msg != NULL) { \ - freeFunc(msg); \ - } \ - } \ - } \ +#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc, param) \ + do { \ + for (int i = 0; i < pool->nAsync; i++) { \ + uv_async_t* async = &(pool->asyncs[i]); \ + SAsyncItem* item = async->data; \ + while (!QUEUE_IS_EMPTY(&item->qmsg)) { \ + tTrace("destroy msg in async pool "); \ + queue* h = QUEUE_HEAD(&item->qmsg); \ + QUEUE_REMOVE(h); \ + msgType* msg = QUEUE_DATA(h, msgType, q); \ + if (msg != NULL) { \ + freeFunc(msg, param); \ + } \ + } \ + } \ } while (0) #define ASYNC_CHECK_HANDLE(exh1, id) \ diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e937d2e65e..e369bf8f7d 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -219,6 +219,8 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, /// NULL,cliHandleUpdate}; static FORCE_INLINE void destroyCmsg(void* cmsg); + +static FORCE_INLINE void destroyQueuedMsg(void* arg, void* param); static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg); static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst); static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); @@ -1963,7 +1965,19 @@ static FORCE_INLINE void destroyCmsg(void* arg) { transFreeMsg(pMsg->msg.pCont); taosMemoryFree(pMsg); } - +static FORCE_INLINE void destroyQueuedMsg(void* arg, void* param) { + SCliMsg* pMsg = arg; + if (pMsg == NULL) { + return; + } + if (param != NULL) { + SCliThrd* pThrd = param; + if (pThrd->destroyAhandleFp) (*pThrd->destroyAhandleFp)(pMsg->msg.info.ahandle); + } + transDestroyConnCtx(pMsg->ctx); + transFreeMsg(pMsg->msg.pCont); + taosMemoryFree(pMsg); +} static FORCE_INLINE void destroyCmsgAndAhandle(void* param) { if (param == NULL) return; @@ -2057,7 +2071,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosThreadJoin(pThrd->thread, NULL); CLI_RELEASE_UV(pThrd->loop); taosThreadMutexDestroy(&pThrd->msgMtx); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsg); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyQueuedMsg, (void*)pThrd); transAsyncPoolDestroy(pThrd->asyncPool); transDQDestroy(pThrd->delayQueue, destroyCmsgAndAhandle); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index b324ca5f91..5a1ef31b7d 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -159,7 +159,7 @@ static void uvStartSendResp(SSvrMsg* msg); static void uvNotifyLinkBrokenToApp(SSvrConn* conn); -static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); +static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); static FORCE_INLINE SSvrConn* createConn(void* hThrd); static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn); @@ -671,7 +671,8 @@ static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) { transFreeMsg(smsg->msg.pCont); taosMemoryFree(smsg); } -static void destroyAllConn(SWorkThrd* pThrd) { +static FORCE_INLINE void destroySmsgWrapper(void* smsg, void* param) { destroySmsg((SSvrMsg*)smsg); } +static void destroyAllConn(SWorkThrd* pThrd) { tTrace("thread %p destroy all conn ", pThrd); while (!QUEUE_IS_EMPTY(&pThrd->conn)) { queue* h = QUEUE_HEAD(&pThrd->conn); @@ -1394,7 +1395,7 @@ void destroyWorkThrd(SWorkThrd* pThrd) { } taosThreadJoin(pThrd->thread, NULL); SRV_RELEASE_UV(pThrd->loop); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsg); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL); transAsyncPoolDestroy(pThrd->asyncPool); uvWhiteListDestroy(pThrd->pWhiteList); From c3220aec44583595a5c2aae8c55675849c2bb79a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 29 Feb 2024 10:46:09 +0000 Subject: [PATCH 11/54] fix mem leak while taosd quit --- source/libs/transport/src/thttp.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 6de10cbb9e..e76f073d63 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -183,7 +183,8 @@ static void* httpThread(void* arg) { return NULL; } -static void httpDestroyMsg(SHttpMsg* msg) { +static void httpDestroyMsg(void* cont, void* param) { + SHttpMsg* msg = cont; if (msg == NULL) return; taosMemoryFree(msg->server); @@ -554,7 +555,7 @@ void transHttpEnvDestroy() { httpSendQuit(); taosThreadJoin(load->thread, NULL); - TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg); + TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg, NULL); transAsyncPoolDestroy(load->asyncPool); uv_loop_close(load->loop); taosMemoryFree(load->loop); From 881ffacb4f6050ec04fdedca85de2e86a692e9f5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 29 Feb 2024 10:52:50 +0000 Subject: [PATCH 12/54] fix mem leak while taosd quit --- source/libs/transport/src/thttp.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index e76f073d63..c4ca39c323 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -183,8 +183,7 @@ static void* httpThread(void* arg) { return NULL; } -static void httpDestroyMsg(void* cont, void* param) { - SHttpMsg* msg = cont; +static void httpDestroyMsg(SHttpMsg* msg) { if (msg == NULL) return; taosMemoryFree(msg->server); @@ -192,6 +191,15 @@ static void httpDestroyMsg(void* cont, void* param) { taosMemoryFree(msg->cont); taosMemoryFree(msg); } +static void httpDestroyMsgWrapper(void* cont, void* param) { + httpDestroyMsg((SHttpMsg*)cont); + // if (msg == NULL) return; + + // taosMemoryFree(msg->server); + // taosMemoryFree(msg->uri); + // taosMemoryFree(msg->cont); + // taosMemoryFree(msg); +} static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) { SHttpMsg *msg = NULL, *quitMsg = NULL; @@ -555,7 +563,7 @@ void transHttpEnvDestroy() { httpSendQuit(); taosThreadJoin(load->thread, NULL); - TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg, NULL); + TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsgWrapper, NULL); transAsyncPoolDestroy(load->asyncPool); uv_loop_close(load->loop); taosMemoryFree(load->loop); From e0b1b5a1c62958e3f9c9a41d497bf9c107fff3aa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 29 Feb 2024 10:54:35 +0000 Subject: [PATCH 13/54] fix mem leak while taosd quit --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index c54da860dd..77760b16f4 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -87,9 +87,9 @@ static bool dmIsForbiddenIp(int8_t forbidden, char *user, uint32_t clientIp) { } } static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { - SDnodeTrans * pTrans = &pDnode->trans; + SDnodeTrans *pTrans = &pDnode->trans; int32_t code = -1; - SRpcMsg * pMsg = NULL; + SRpcMsg *pMsg = NULL; SMgmtWrapper *pWrapper = NULL; SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; @@ -256,11 +256,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; - SArray * pArray = (*pWrapper->func.getHandlesFp)(); + SArray *pArray = (*pWrapper->func.getHandlesFp)(); if (pArray == NULL) return -1; for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SMgmtHandle * pMgmt = taosArrayGet(pArray, i); + SMgmtHandle *pMgmt = taosArrayGet(pArray, i); SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)]; if (pMgmt->needCheckVgId) { pHandle->needCheckVgId = pMgmt->needCheckVgId; From 867e81c5f5f07b60cec275ad457fe2b8e1075dc2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 29 Feb 2024 10:58:49 +0000 Subject: [PATCH 14/54] fix mem leak while taosd quit --- source/libs/transport/src/transCli.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e369bf8f7d..6ae72eac14 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -220,7 +220,7 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, static FORCE_INLINE void destroyCmsg(void* cmsg); -static FORCE_INLINE void destroyQueuedMsg(void* arg, void* param); +static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param); static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg); static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst); static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); @@ -1965,7 +1965,7 @@ static FORCE_INLINE void destroyCmsg(void* arg) { transFreeMsg(pMsg->msg.pCont); taosMemoryFree(pMsg); } -static FORCE_INLINE void destroyQueuedMsg(void* arg, void* param) { +static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param) { SCliMsg* pMsg = arg; if (pMsg == NULL) { return; @@ -1974,9 +1974,7 @@ static FORCE_INLINE void destroyQueuedMsg(void* arg, void* param) { SCliThrd* pThrd = param; if (pThrd->destroyAhandleFp) (*pThrd->destroyAhandleFp)(pMsg->msg.info.ahandle); } - transDestroyConnCtx(pMsg->ctx); - transFreeMsg(pMsg->msg.pCont); - taosMemoryFree(pMsg); + destroyCmsg(pMsg); } static FORCE_INLINE void destroyCmsgAndAhandle(void* param) { if (param == NULL) return; @@ -2071,7 +2069,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosThreadJoin(pThrd->thread, NULL); CLI_RELEASE_UV(pThrd->loop); taosThreadMutexDestroy(&pThrd->msgMtx); - TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyQueuedMsg, (void*)pThrd); + TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsgWrapper, (void*)pThrd); transAsyncPoolDestroy(pThrd->asyncPool); transDQDestroy(pThrd->delayQueue, destroyCmsgAndAhandle); From 3febcb96a0dab5ed7199d72d308b78f1b6699733 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 1 Mar 2024 10:57:57 +0800 Subject: [PATCH 15/54] reset flush state --- source/libs/stream/src/streamSessionState.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 3d0241df75..723f04c499 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -156,6 +156,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pKey = *pDestWinKey; goto _end; } @@ -167,6 +168,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pKey = *pDestWinKey; goto _end; } @@ -380,6 +382,14 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co (*pWins) = pWinStates; } + if (size > 0 && index == -1) { + SRowBuffPos* pPos = taosArrayGetP(pWinStates, 0); + SSessionKey* pWin = (SSessionKey*)pPos->pKey; + if (pWinKey->win.skey == pWin->win.skey) { + index = 0; + } + } + if (index >= 0) { pCur = createSessionStateCursor(pFileState); pCur->buffIndex = index; @@ -387,6 +397,7 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co *pIndex = index; } } + return pCur; } @@ -666,6 +677,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *key = *pDestWinKey; goto _end; } @@ -679,6 +691,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *key = *pDestWinKey; goto _end; } @@ -771,6 +784,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pWinKey = *pDestWinKey; goto _end; } @@ -799,6 +813,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C (*pVal) = pPos; SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey; pPos->beUsed = true; + pPos->beFlushed = false; *pWinKey = *pDestWinKey; goto _end; } From 9a5219d5943bd315d3f2e27ecb1d5f0d6b2a972f Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 29 Feb 2024 13:19:26 +0800 Subject: [PATCH 16/54] fix: memory free sequence of sub request caused memory use after free --- source/client/inc/clientInt.h | 1 + source/client/src/clientEnv.c | 32 +++++++++++++++++-- source/client/src/clientMain.c | 56 +++++++++++----------------------- 3 files changed, 49 insertions(+), 40 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 989c6614a6..59eee6fd9d 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -354,6 +354,7 @@ SRequestObj* acquireRequest(int64_t rid); int32_t releaseRequest(int64_t rid); int32_t removeRequest(int64_t rid); void doDestroyRequest(void* p); +int64_t removeFromMostPrevReq(SRequestObj* pRequest); char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 1df50a51da..6c20813118 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -385,6 +385,33 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); } +/// return the most previous req ref id +int64_t removeFromMostPrevReq(SRequestObj* pRequest) { + int64_t mostPrevReqRefId = pRequest->self; + SRequestObj* pTmp = pRequest; + while (pTmp->relation.prevRefId) { + pTmp = acquireRequest(pTmp->relation.prevRefId); + if (pTmp) { + mostPrevReqRefId = pTmp->self; + releaseRequest(mostPrevReqRefId); + } else { + break; + } + } + removeRequest(mostPrevReqRefId); + return mostPrevReqRefId; +} + +void destroyNextReq(int64_t nextRefId) { + if (nextRefId) { + SRequestObj* pObj = acquireRequest(nextRefId); + if (pObj) { + releaseRequest(nextRefId); + releaseRequest(nextRefId); + } + } +} + void destroySubRequests(SRequestObj *pRequest) { int32_t reqIdx = -1; SRequestObj *pReqList[16] = {NULL}; @@ -435,7 +462,7 @@ void doDestroyRequest(void *p) { uint64_t reqId = pRequest->requestId; tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest); - destroySubRequests(pRequest); + int64_t nextReqRefId = pRequest->relation.nextRefId; taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); @@ -471,6 +498,7 @@ void doDestroyRequest(void *p) { taosMemoryFreeClear(pRequest->sqlstr); taosMemoryFree(pRequest); tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest); + destroyNextReq(nextReqRefId); } void destroyRequest(SRequestObj *pRequest) { @@ -479,7 +507,7 @@ void destroyRequest(SRequestObj *pRequest) { } taos_stop_query(pRequest); - removeRequest(pRequest->self); + removeFromMostPrevReq(pRequest); } void taosStopQueryImpl(SRequestObj *pRequest) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 275ca0d2aa..e9379946b1 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1254,54 +1254,34 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { } void restartAsyncQuery(SRequestObj *pRequest, int32_t code) { - int32_t reqIdx = 0; - SRequestObj *pReqList[16] = {NULL}; - SRequestObj *pUserReq = NULL; - pReqList[0] = pRequest; - uint64_t tmpRefId = 0; - SRequestObj *pTmp = pRequest; - while (pTmp->relation.prevRefId) { - tmpRefId = pTmp->relation.prevRefId; - pTmp = acquireRequest(tmpRefId); - if (pTmp) { - pReqList[++reqIdx] = pTmp; - releaseRequest(tmpRefId); - } else { - tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId); + tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest); + SRequestObj* pUserReq = pRequest; + acquireRequest(pRequest->self); + while (pUserReq) { + if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) { break; - } - } - - tmpRefId = pRequest->relation.nextRefId; - while (tmpRefId) { - pTmp = acquireRequest(tmpRefId); - if (pTmp) { - tmpRefId = pTmp->relation.nextRefId; - removeRequest(pTmp->self); - releaseRequest(pTmp->self); } else { - tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId); - break; + int64_t nextRefId = pUserReq->relation.nextRefId; + releaseRequest(pUserReq->self); + if (nextRefId) { + pUserReq = acquireRequest(nextRefId); + } } } - - for (int32_t i = reqIdx; i >= 0; i--) { - destroyCtxInRequest(pReqList[i]); - if (pReqList[i]->relation.userRefId == pReqList[i]->self || 0 == pReqList[i]->relation.userRefId) { - pUserReq = pReqList[i]; - } else { - removeRequest(pReqList[i]->self); - } - } - + bool hasSubRequest = pUserReq != pRequest || pRequest->relation.prevRefId != 0; if (pUserReq) { + destroyCtxInRequest(pUserReq); pUserReq->prevCode = code; memset(&pUserReq->relation, 0, sizeof(pUserReq->relation)); } else { - tscError("user req is missing"); + tscError("User req is missing"); + removeFromMostPrevReq(pRequest); return; } - + if (hasSubRequest) + removeFromMostPrevReq(pRequest); + else + releaseRequest(pUserReq->self); doAsyncQuery(pUserReq, true); } From 8df1f09561420ffbb0d98a1ba5248d60d3081783 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 1 Mar 2024 11:27:33 +0800 Subject: [PATCH 17/54] optimze plan and add ci --- source/libs/parser/src/parTranslater.c | 42 +++++++++++------------- tests/script/tsim/query/query_count1.sim | 22 +++++++++++++ 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index cc522c9b5f..3a622eeea6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4009,6 +4009,26 @@ static int32_t translateEventWindow(STranslateContext* pCxt, SSelectStmt* pSelec } static int32_t translateCountWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow; + if (pCountWin->windowCount <= 1) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must exceed 1."); + } + + if (pCountWin->windowSliding <= 0) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must exceed 0."); + } + + if (pCountWin->windowSliding > pCountWin->windowCount) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "sliding value no larger than the count value."); + } + + if (pCountWin->windowCount > INT32_MAX) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Size of Count window must less than 2147483647(INT32_MAX)."); + } if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_QUERY, @@ -7828,29 +7848,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm if (pStmt->pOptions->ignoreExpired != 1) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Ignore expired data of Count window must be 1."); - } - - SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow; - if (pCountWin->windowCount <= 1) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must exceed 1."); } - - if (pCountWin->windowSliding <= 0) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must exceed 0."); - } - - if (pCountWin->windowSliding > pCountWin->windowCount) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "sliding value no larger than the count value."); - } - - if (pCountWin->windowCount > INT32_MAX) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, - "Size of Count window must less than 2147483647(INT32_MAX)."); - } - } return TSDB_CODE_SUCCESS; diff --git a/tests/script/tsim/query/query_count1.sim b/tests/script/tsim/query/query_count1.sim index 0694ab062a..043b604263 100644 --- a/tests/script/tsim/query/query_count1.sim +++ b/tests/script/tsim/query/query_count1.sim @@ -79,5 +79,27 @@ if $data22 != 4 then goto loop3 endi + +print step2 +print =============== create database +sql create database test1 vgroups 1; +sql use test1; + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +#2~INT32_MAX +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(-1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(0); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2147483648); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, 0); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, -1); +sql_error select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(10, 11); + +sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2); +sql select _wstart as s, count(*) c1, sum(b), max(c) from t1 count_window(2147483647); + print query_count0 end system sh/exec.sh -n dnode1 -s stop -x SIGINT From dfae8a9471e3bd3b9ed5697ea070759ae6488d89 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Fri, 1 Mar 2024 11:31:46 +0800 Subject: [PATCH 18/54] optimze plan and add ci --- source/libs/planner/src/planSpliter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index d9a7475f59..9664be1da2 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -274,7 +274,7 @@ static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) { } } - if (WINDOW_TYPE_STATE == pWindow->winType) { + if (WINDOW_TYPE_STATE == pWindow->winType || WINDOW_TYPE_COUNT == pWindow->winType) { if (!streamQuery) { return stbSplHasMultiTbScan(streamQuery, pNode); } else { From 1bf403660f27835fed9dffaaffa1ae3e08c4ce10 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:02:26 +0800 Subject: [PATCH 19/54] Update 14-stream.md --- docs/zh/12-taos-sql/14-stream.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 979fc436b9..c4ab450e79 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -49,10 +49,13 @@ window_clause: { SESSION(ts_col, tol_val) | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] + | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition + | COUNT_WINDOW(count_val[, sliding_val]) } ``` -其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 +其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。COUNT_WINDOW是计数窗口,按固定的数据行数来划分窗口。 +count_val：常量，是正整数，必须大于等于2，最大INT32_MAX。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished) From e20dc784f82895d992fd357f0d86300f0272c752 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:09:57 +0800 Subject: [PATCH 20/54] Update 14-stream.md --- docs/en/12-taos-sql/14-stream.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 6f2343d347..e6aaaf0663 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -41,10 +41,14 @@ window_clause: { SESSION(ts_col, tol_val) | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] + | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition + | COUNT_WINDOW(count_val[, sliding_val]) } ``` `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically. +COUNT_WINDOW is a counting window that is divided by a fixed number of data rows.count_val: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. count_val represents the maximum number of data rows contained in each COUNT_WINDOW. +When the total number of data rows cannot be divided by count_val, the number of rows in the last window will be less than count_val. sliding_val: is a constant that represents the number of window slides, similar to SLIDING in intervals For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable. From 4a53bfb1077a9aea5e4cb9252320cdc03235ef58 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:11:04 +0800 Subject: [PATCH 21/54] Update 14-stream.md --- docs/zh/12-taos-sql/14-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index c4ab450e79..c36f355ffa 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -55,7 +55,7 @@ window_clause: { ``` 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。COUNT_WINDOW是计数窗口,按固定的数据行数来划分窗口。 -count_val：常量，是正整数，必须大于等于2，最大INT32_MAX。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING +count_val：常量，是正整数，必须大于等于2，小于2147483648。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished) From ce379d077d9559dd128aa4c1b08ac8593329e9d8 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:32:45 +0800 Subject: [PATCH 22/54] Update 14-stream.md --- docs/zh/12-taos-sql/14-stream.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index c36f355ffa..5a4772bb6c 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -54,8 +54,10 @@ window_clause: { } ``` -其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。COUNT_WINDOW是计数窗口,按固定的数据行数来划分窗口。 -count_val：常量，是正整数，必须大于等于2，小于2147483648。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING +其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 +EVENT_WINDOW是事件窗口，根据开始条件和结束条件来划定窗口。当start_trigger_condition满足时则窗口开始，直到end_trigger_condition满足时窗口关闭。start_trigger_condition和end_trigger_condition可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。 +COUNT_WINDOW是计数窗口,按固定的数据行数来划分窗口。count_val：常量，是正整数，必须大于等于2，小于2147483648。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。 +sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING。 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished) From dadaa9055f4d1696c57905be0362c510bcbb7b17 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:40:24 +0800 Subject: [PATCH 23/54] Update 14-stream.md --- docs/en/12-taos-sql/14-stream.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index e6aaaf0663..099fa7a963 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -47,8 +47,12 @@ window_clause: { ``` `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically. -COUNT_WINDOW is a counting window that is divided by a fixed number of data rows.count_val: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. count_val represents the maximum number of data rows contained in each COUNT_WINDOW. -When the total number of data rows cannot be divided by count_val, the number of rows in the last window will be less than count_val. sliding_val: is a constant that represents the number of window slides, similar to SLIDING in intervals + +`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` +can be any conditional expressions supported by TDengine and can include multiple columns. + +`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. +When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`. For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable. From 413c1ee4ab86c75a5b1aa81a45f963fe5a1179f3 Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:47:55 +0800 Subject: [PATCH 24/54] Update 14-stream.md --- docs/en/12-taos-sql/14-stream.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 099fa7a963..b8bbd4c154 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -48,11 +48,9 @@ window_clause: { `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically. -`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` -can be any conditional expressions supported by TDengine and can include multiple columns. +`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns. -`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. -When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`. +`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`. For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable. From 0926bbf8fa9f6dec9d59591af2f6511021e81ce4 Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:56:23 +0800 Subject: [PATCH 25/54] Update 14-stream.md --- docs/zh/12-taos-sql/14-stream.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 5a4772bb6c..8868b728f8 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -55,9 +55,8 @@ window_clause: { ``` 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 -EVENT_WINDOW是事件窗口，根据开始条件和结束条件来划定窗口。当start_trigger_condition满足时则窗口开始，直到end_trigger_condition满足时窗口关闭。start_trigger_condition和end_trigger_condition可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。 -COUNT_WINDOW是计数窗口,按固定的数据行数来划分窗口。count_val：常量，是正整数，必须大于等于2，小于2147483648。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。 -sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING。 +EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。 +COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished) @@ -66,6 +65,12 @@ sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLI ```sql CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + +CREATE STREAM streams0 INTO streamt0 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9; + +CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10); ``` ## 流式计算的 partition From 43a3f85f48679a809288b7e8b81033c2c4de93bc Mon Sep 17 00:00:00 2001 From: liuyao <38781207+54liuyao@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:57:24 +0800 Subject: [PATCH 26/54] Update 14-stream.md --- docs/en/12-taos-sql/14-stream.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index b8bbd4c154..e1bf18c854 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -57,6 +57,12 @@ For example, the following SQL statement creates a stream and automatically crea ```sql CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); + +CREATE STREAM streams0 INTO streamt0 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9; + +CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS +SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10); ``` ## Partitions of Stream From 5554362399a92249053b21b72efb8cace06945de Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 3 Mar 2024 16:41:16 +0800 Subject: [PATCH 27/54] fix(stream):remove unused def. --- include/libs/stream/tstream.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 64ce735843..cf980400d6 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -56,7 +56,6 @@ extern "C" { #define STREAM_EXEC_T_RESTART_ALL_TASKS (-4) #define STREAM_EXEC_T_STOP_ALL_TASKS (-5) #define STREAM_EXEC_T_RESUME_TASK (-6) -#define STREAM_EXEC_T_UPDATE_TASK_EPSET (-7) typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; From 550d0fe5aef660aa50eab83d833e62e715a969b0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 3 Mar 2024 16:46:13 +0800 Subject: [PATCH 28/54] fix(stream): add an assert. --- source/libs/stream/src/streamDispatch.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 78b914c3db..9702096b60 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -740,6 +740,8 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + ASSERT(dataStrLen > 0); + void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) return -1; From 8ff0fd346f913d9937569de5f4a86b6d5379f758 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 4 Mar 2024 02:42:03 +0800 Subject: [PATCH 29/54] test: add subscribe and commits in compatibility test --- .../system-test/0-others/com_alltypedata.json | 2 +- tests/system-test/0-others/compatibility.py | 83 +++++++++++++++++-- 2 files changed, 76 insertions(+), 9 deletions(-) diff --git a/tests/system-test/0-others/com_alltypedata.json b/tests/system-test/0-others/com_alltypedata.json index 0e6d8e3a07..1499ca7670 100644 --- a/tests/system-test/0-others/com_alltypedata.json +++ b/tests/system-test/0-others/com_alltypedata.json @@ -22,7 +22,7 @@ "vgroups": 2, "replica": 1, "precision": "ms", - "stt_trigger": 8, + "stt_trigger": 1, "minRows": 100, "maxRows": 4096 }, diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index c936cf1ae4..8163177a3b 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -1,11 +1,13 @@ from urllib.parse import uses_relative import taos +import taosws import sys import os import time import platform import inspect from taos.tmq import Consumer +from taos.tmq import * from pathlib import Path from util.log import * @@ -17,7 +19,7 @@ from util.dnodes import TDDnode from util.cluster import * import subprocess -BASEVERSION = "3.0.2.3" +BASEVERSION = "3.2.0.0" class TDTestCase: def caseDescription(self): f''' @@ -30,7 +32,7 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 4; ;use deldata; + self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 1; ;use deldata; create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); create table deldata.ct1 using deldata.stb1 tags ( 1 ); insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); @@ -104,8 +106,19 @@ class TDTestCase: print(f"{packageName} has been exists") os.system(f" cd {packagePath} && tar xvf {packageName} && cd {packageTPath} && ./install.sh -e no " ) tdDnodes.stop(1) - print(f"start taosd: rm -rf {dataPath}/* && nohup taosd -c {cPath} & ") - os.system(f"rm -rf {dataPath}/* && nohup taosd -c {cPath} & " ) + print(f"start taosd: rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & ") + os.system(f"rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & " ) + os.system(f"killall taosadapter" ) + os.system(f"cp /etc/taos/taosadapter.toml {cPath}/taosadapter.toml " ) + taosadapter_cfg = cPath + "/taosadapter.toml" + taosadapter_log_path = cPath + "/../log/" + print(f"taosadapter_cfg:{taosadapter_cfg},taosadapter_log_path:{taosadapter_log_path} ") + self.alter_string_in_file(taosadapter_cfg,"#path = \"/var/log/taos\"",f"path = \"{taosadapter_log_path}\"") + self.alter_string_in_file(taosadapter_cfg,"taosConfigDir = \"\"",f"taosConfigDir = \"{cPath}\"") + print("/usr/bin/taosadapter --version") + os.system(f" /usr/bin/taosadapter --version" ) + print(f" LD_LIBRARY_PATH=/usr/lib -c {taosadapter_cfg} 2>&1 & ") + os.system(f" LD_LIBRARY_PATH=/usr/lib /usr/bin/taosadapter -c {taosadapter_cfg} 2>&1 & " ) sleep(5) @@ -116,7 +129,24 @@ class TDTestCase: def is_list_same_as_ordered_list(self,unordered_list, ordered_list): sorted_list = sorted(unordered_list) return sorted_list == ordered_list - + + def alter_string_in_file(self,file,old_str,new_str): + """ + replace str in file + :param file + :param old_str + :param new_str + :return: + """ + file_data = "" + with open(file, "r", encoding="utf-8") as f: + for line in f: + if old_str in line: + line = line.replace(old_str,new_str) + file_data += line + with open(file,"w",encoding="utf-8") as f: + f.write(file_data) + def run(self): scriptsPath = os.path.dirname(os.path.realpath(__file__)) distro_id = distro.id() @@ -131,7 +161,7 @@ class TDTestCase: dbname = "test" stb = f"{dbname}.meters" self.installTaosd(bPath,cPath) - os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ") + # os.system(f"echo 'debugFlag 143' >> {cPath}/taos.cfg ") tableNumbers=100 recordNumbers1=100 recordNumbers2=1000 @@ -163,11 +193,46 @@ class TDTestCase: # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show streams;" ') - os.system(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/tmqBasic.json ") + self.alter_string_in_file("0-others/tmqBasic.json", "/etc/taos/", cPath) # os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/tmqBasic.json -y ") os.system('LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists tmq_test_topic as select current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ') os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show topics;" ') + os.system(f" /usr/bin/taosadapter --version " ) + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"}) + try: + consumer.subscribe(["tmq_test_topic"]) + except TmqError: + tdLog.exit(f"subscribe error") + + while True: + message = consumer.poll(timeout=1.0) + if message: + print("message") + id = message.vgroup() + topic = message.topic() + database = message.database() + + for block in message: + nrows = block.nrows() + ncols = block.ncols() + for row in block: + print(row) + values = block.fetchall() + print(nrows, ncols) + + consumer.commit(message) + else: + print("break") + break + + consumer.close() tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") @@ -184,7 +249,8 @@ class TDTestCase: os.system("pkill taosd") # make sure all the data are saved in disk. self.checkProcessPid("taosd") - + os.system("pkill taosadapter") # make sure all the data are saved in disk. + self.checkProcessPid("taosadapter") tdLog.printNoPrefix("==========step2:update new version ") self.buildTaosd(bPath) @@ -193,6 +259,7 @@ class TDTestCase: tdsql=tdCom.newTdSql() print(tdsql) cmd = f" LD_LIBRARY_PATH=/usr/lib taos -h localhost ;" + print(os.system(cmd)) if os.system(cmd) == 0: raise Exception("failed to execute system command. cmd: %s" % cmd) From 0a814e97b554eae97a2e7b39f9ab3cbc937dece2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 4 Mar 2024 09:26:54 +0800 Subject: [PATCH 30/54] enh(stream): reduce the threads requirements. --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c5a26c5c10..55df80ca44 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4; int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 4; -float tsRatioOfVnodeStreamThreads = 1.5F; +float tsRatioOfVnodeStreamThreads = 0.5F; int32_t tsNumOfVnodeFetchThreads = 4; int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfQnodeQueryThreads = 4; From 376872acc6bdd2e18c58711fe3921f7a4765d989 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 4 Mar 2024 09:53:49 +0800 Subject: [PATCH 31/54] fix(test/last_both): remove last_row query --- tests/script/tsim/parser/last_both.sim | 6 +- tests/script/tsim/parser/last_both_query.sim | 496 ++++++++++++++++++ tests/script/tsim/parser/last_cache_query.sim | 106 ---- 3 files changed, 499 insertions(+), 109 deletions(-) create mode 100644 tests/script/tsim/parser/last_both_query.sim diff --git a/tests/script/tsim/parser/last_both.sim b/tests/script/tsim/parser/last_both.sim index d7daf4d333..7b6c40c127 100644 --- a/tests/script/tsim/parser/last_both.sim +++ b/tests/script/tsim/parser/last_both.sim @@ -69,13 +69,13 @@ sql drop table tbf; sql alter table st2 add column c1 int; sql alter table st2 drop column c1; -run tsim/parser/last_cache_query.sim +run tsim/parser/last_both_query.sim sql flush database $db system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -run tsim/parser/last_cache_query.sim +run tsim/parser/last_both_query.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start @@ -145,6 +145,6 @@ sql alter database $db cachemodel 'both' sql alter database $db cachesize 2 sleep 11000 -run tsim/parser/last_cache_query.sim +run tsim/parser/last_both_query.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/last_both_query.sim b/tests/script/tsim/parser/last_both_query.sim new file mode 100644 index 0000000000..775fd482ea --- /dev/null +++ b/tests/script/tsim/parser/last_both_query.sim @@ -0,0 +1,496 @@ + +sql connect + +$db = testdb +sql use $db +print "test tb1" + +sql select last(ts) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != 6 then + print $data00 + return -1 +endi + +sql select last(*) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + print expect 3, actual: $data03 + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql select last(tb1.*,ts,f4) from tb1 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:57.000@ then + return -1 +endi + +print "test tb2" +sql select last(ts) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != -6 then + print $data00 + return -1 +endi + +sql select last(*) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data01 != -6 then + return -1 +endi +if $data02 != -7.000000000 then + print $data02 + return -1 +endi +if $data03 != -8 then + return -1 +endi +if $data04 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi + +sql select last(tb2.*,ts,f4) from tb2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data01 != -6 then + return -1 +endi +if $data02 != -7.000000000 then + print $data02 + return -1 +endi +if $data03 != -8 then + return -1 +endi +if $data04 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi +if $data05 != @21-05-11 10:11:15.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:56.999@ then + if $data04 != @70-01-01 07:59:57.-01@ then + return -1 + endi +endi + +print "test tbd" +sql select last(*) from tbd +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-11 10:12:29.000@ then + print $data00 + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != NULL then + print $data02 + return -1 +endi +if $data03 != NULL then + return -1 +endi +if $data04 != NULL then + return -1 +endi + +print "test tbe" +sql select last(*) from tbe +if $rows != 0 then + return -1 +endi + +print "test stable" +sql select last(ts) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi + +sql select last(f1) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != 6 then + print $data00 + return -1 +endi + +sql select last(*) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 37.000000000 then + print expect 37.000000000 actual: $data02 + return -1 +endi +if $data03 != 27 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + + +sql select last(st2.*,ts,f4) from st2 +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 37.000000000 then + print expect 37.000000000, acutal: $data02 + return -1 +endi +if $data03 != 27 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != @21-05-12 10:10:12.000@ then + print $data00 + return -1 +endi +if $data06 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql select last(*), id from st2 group by id order by id +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 + +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 21 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data10 != @21-05-11 10:12:23.000@ then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data12 != 23.000000000 then + print $data02 + return -1 +endi +if $data13 != -8 then + return -1 +endi +if $data14 != @70-01-01 07:59:58.-04@ then + return -1 +endi +if $data15 != 2 then + return -1 +endi +if $data20 != @21-05-10 10:12:24.000@ then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data22 != 11.000000000 then + print expect 11.000000000 actual: $data22 + return -1 +endi +if $data23 != 25 then + return -1 +endi +if $data24 != @70-01-01 07:59:57.-04@ then = + return -1 +endi +if $data25 != 3 then + return -1 +endi +if $data30 != @21-05-11 10:12:25.000@ then + return -1 +endi +if $data31 != 26 then + return -1 +endi +if $data32 != 17.000000000 then + print $data02 + return -1 +endi +if $data33 != 27 then + return -1 +endi +if $data34 != @70-01-01 07:59:56.-04@ then + return -1 +endi +if $data35 != 4 then + return -1 +endi +if $data40 != @21-05-11 10:12:29.000@ then + return -1 +endi +if $data41 != 36 then + return -1 +endi +if $data42 != 37.000000000 then + print $data02 + return -1 +endi +if $data43 != 35 then + return -1 +endi +if $data44 != @70-01-01 07:59:56.-05@ then + return -1 +endi +if $data45 != 5 then + return -1 +endi + +#sql select last_row(*), id from st2 group by id order by id +#print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +#print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +#print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +#print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +#print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 +# +#if $rows != 5 then +# return -1 +#endi +#if $data00 != @21-05-12 10:10:12.000@ then +# return -1 +#endi +#if $data01 != 6 then +# return -1 +#endi +#if $data02 != NULL then +# print $data02 +# return -1 +#endi +#if $data03 != NULL then +# return -1 +#endi +#if $data04 != @70-01-01 07:59:57.000@ then +# return -1 +#endi +#if $data05 != 1 then +# return -1 +#endi +#if $data10 != @21-05-11 10:12:23.000@ then +# return -1 +#endi +#if $data11 != 22 then +# return -1 +#endi +#if $data12 != 23.000000000 then +# print $data02 +# return -1 +#endi +#if $data13 != NULL then +# return -1 +#endi +#if $data14 != @70-01-01 07:59:58.-04@ then +# return -1 +#endi +#if $data15 != 2 then +# return -1 +#endi +#if $data20 != @21-05-10 10:12:24.000@ then +# return -1 +#endi +#if $data21 != 24 then +# return -1 +#endi +#if $data22 != NULL then +# print expect NULL actual: $data22 +# return -1 +#endi +#if $data23 != 25 then +# return -1 +#endi +#if $data24 != @70-01-01 07:59:57.-04@ then = +# return -1 +#endi +#if $data25 != 3 then +# return -1 +#endi +#if $data30 != @21-05-11 10:12:25.000@ then +# return -1 +#endi +#if $data31 != 26 then +# return -1 +#endi +#if $data32 != NULL then +# print $data02 +# return -1 +#endi +#if $data33 != 27 then +# return -1 +#endi +#if $data34 != @70-01-01 07:59:56.-04@ then +# return -1 +#endi +#if $data35 != 4 then +# return -1 +#endi +#if $data40 != @21-05-11 10:12:29.000@ then +# return -1 +#endi +#if $data41 != NULL then +# return -1 +#endi +#if $data42 != NULL then +# print $data02 +# return -1 +#endi +#if $data43 != NULL then +# return -1 +#endi +#if $data44 != NULL then +# return -1 +#endi +#if $data45 != 5 then +# return -1 +#endi + +print "test tbn" +sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) +sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) +sql insert into tbn values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000) +sql insert into tbn values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000) +sql insert into tbn values ("2021-05-13 10:10:12", NULL,NULL, NULL,NULL) + +sql select last(*) from tbn; +if $rows != 1 then + return -1 +endi +if $data00 != @21-05-13 10:10:12.000@ then + print $data00 + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != 5.000000000 then + print $data02 + return -1 +endi +if $data03 != 3 then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi + +sql alter table tbn add column c1 int; +sql alter table tbn drop column c1; diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim index c5961f2183..30196e0b62 100644 --- a/tests/script/tsim/parser/last_cache_query.sim +++ b/tests/script/tsim/parser/last_cache_query.sim @@ -357,112 +357,6 @@ if $data45 != 5 then return -1 endi -sql select last_row(*), id from st2 group by id order by id -print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 -print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 -print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 -print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 - -if $rows != 5 then - return -1 -endi -if $data00 != @21-05-12 10:10:12.000@ then - return -1 -endi -if $data01 != 6 then - return -1 -endi -if $data02 != NULL then - print $data02 - return -1 -endi -if $data03 != NULL then - return -1 -endi -if $data04 != @70-01-01 07:59:57.000@ then - return -1 -endi -if $data05 != 1 then - return -1 -endi -if $data10 != @21-05-11 10:12:23.000@ then - return -1 -endi -if $data11 != 22 then - return -1 -endi -if $data12 != 23.000000000 then - print $data02 - return -1 -endi -if $data13 != NULL then - return -1 -endi -if $data14 != @70-01-01 07:59:58.-04@ then - return -1 -endi -if $data15 != 2 then - return -1 -endi -if $data20 != @21-05-10 10:12:24.000@ then - return -1 -endi -if $data21 != 24 then - return -1 -endi -if $data22 != NULL then - print expect NULL actual: $data22 - return -1 -endi -if $data23 != 25 then - return -1 -endi -if $data24 != @70-01-01 07:59:57.-04@ then = - return -1 -endi -if $data25 != 3 then - return -1 -endi -if $data30 != @21-05-11 10:12:25.000@ then - return -1 -endi -if $data31 != 26 then - return -1 -endi -if $data32 != NULL then - print $data02 - return -1 -endi -if $data33 != 27 then - return -1 -endi -if $data34 != @70-01-01 07:59:56.-04@ then - return -1 -endi -if $data35 != 4 then - return -1 -endi -if $data40 != @21-05-11 10:12:29.000@ then - return -1 -endi -if $data41 != 36 then - return -1 -endi -if $data42 != 37.000000000 then - print $data02 - return -1 -endi -if $data43 != NULL then - return -1 -endi -if $data44 != @70-01-01 07:59:56.-05@ then - return -1 -endi -if $data45 != 5 then - return -1 -endi - print "test tbn" sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3', -1000) From c7d115d8aaa6f20b26575ec73ccdb269f4ec875b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 4 Mar 2024 14:58:13 +0800 Subject: [PATCH 32/54] fix:add excluded msg for delete in tmq --- include/common/tmsg.h | 3 +++ source/client/src/clientRawBlockWrite.c | 2 ++ source/common/src/tmsg.c | 9 +++++++++ source/dnode/vnode/src/tq/tqUtil.c | 3 +-- source/libs/qworker/src/qwMsg.c | 1 + source/libs/scheduler/src/schRemote.c | 1 + tests/script/sh/deploy.sh | 2 +- utils/test/c/tmq_taosx_ci.c | 1 + 8 files changed, 19 insertions(+), 3 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 587e2f9f3e..e400698f69 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3813,8 +3813,10 @@ typedef struct { uint32_t phyLen; char* sql; char* msg; + int8_t source; } SVDeleteReq; +extern int8_t deleteFromTaosx; int32_t tSerializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq); int32_t tDeserializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq); @@ -3834,6 +3836,7 @@ typedef struct SDeleteRes { char tableFName[TSDB_TABLE_NAME_LEN]; char tsColName[TSDB_COL_NAME_LEN]; int64_t ctimeMs; // fill by vnode + int8_t source; } SDeleteRes; int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index f143624bab..3857e47f57 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1256,7 +1256,9 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName, req.tsColName, req.skey, req.tsColName, req.ekey); + deleteFromTaosx = TD_REQ_FROM_TAOX; TAOS_RES* res = taos_query(taos, sql); + deleteFromTaosx = TD_REQ_FROM_APP; SRequestObj* pRequest = (SRequestObj*)res; code = pRequest->code; if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_PAR_GET_META_ERROR) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 85f5d462c7..e7414c6004 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -7148,6 +7148,7 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { return 0; } +int8_t deleteFromTaosx = TD_REQ_FROM_APP; int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { @@ -7165,6 +7166,7 @@ int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (tEncodeBinary(&encoder, pReq->msg, pReq->phyLen) < 0) return -1; + if (tEncodeI8(&encoder, pReq->source) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -7201,6 +7203,9 @@ int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, &msgLen) < 0) return -1; pReq->phyLen = msgLen; + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->source) < 0) return -1; + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -8400,6 +8405,7 @@ int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) { if (tEncodeCStr(pCoder, pRes->tableFName) < 0) return -1; if (tEncodeCStr(pCoder, pRes->tsColName) < 0) return -1; if (tEncodeI64(pCoder, pRes->ctimeMs) < 0) return -1; + if (tEncodeI8(pCoder, pRes->source) < 0) return -1; return 0; } @@ -8424,6 +8430,9 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) { if (!tDecodeIsEnd(pCoder)) { if (tDecodeI64(pCoder, &pRes->ctimeMs) < 0) return -1; } + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI8(pCoder, &pRes->source) < 0) return -1; + } return 0; } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index dad1211326..6029575e2c 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -263,8 +263,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) { PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq) } else if (pHead->msgType == TDMT_VND_DELETE) { - fetchVer++; - continue; + PROCESS_EXCLUDED_MSG(SDeleteRes, tDecodeDeleteRes) } } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 66ec460861..faa90dcbf8 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -715,6 +715,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD uint64_t tId = req.taskId; int64_t rId = 0; int32_t eId = -1; + pRes->source = req.source; SQWMsg qwMsg = {.node = node, .msg = req.msg, .msgLen = req.phyLen, .connInfo = pMsg->info}; QW_SCH_TASK_DLOG("processDelete start, node:%p, handle:%p, sql:%s", node, pMsg->info.handle, req.sql); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1c0b31109e..f6465316d2 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1086,6 +1086,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.sqlLen = strlen(pJob->sql); req.sql = (char *)pJob->sql; req.msg = pTask->msg; + req.source = deleteFromTaosx; msgSize = tSerializeSVDeleteReq(NULL, 0, &req); msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 3b3d275a07..9af2525c74 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -117,7 +117,7 @@ echo "supportVnodes 1024" >> $TAOS_CFG echo "statusInterval 1" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG -echo "debugFlag 0" >> $TAOS_CFG +echo "debugFlag 135" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "uDebugFlag 143" >> $TAOS_CFG echo "rpcDebugFlag 143" >> $TAOS_CFG diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 2257089f06..5012e50bab 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -574,6 +574,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.consume.excluded", "1"); if (g_conf.snapShot) { tmq_conf_set(conf, "experimental.snapshot.enable", "true"); From eba7b2fa82db2b1a93971990ff846597953a80e2 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 4 Mar 2024 15:58:25 +0800 Subject: [PATCH 33/54] fix(test/last_row): remove random columns' checking --- tests/script/tsim/parser/last_both.sim | 2 +- tests/script/tsim/parser/last_both_query.sim | 190 +++++++++---------- 2 files changed, 96 insertions(+), 96 deletions(-) diff --git a/tests/script/tsim/parser/last_both.sim b/tests/script/tsim/parser/last_both.sim index 7b6c40c127..e01a966744 100644 --- a/tests/script/tsim/parser/last_both.sim +++ b/tests/script/tsim/parser/last_both.sim @@ -6,7 +6,7 @@ sql connect print ======================== dnode1 start $db = testdb sql drop database if exists $db -sql create database $db cachemodel 'both' minrows 10 stt_trigger 1 +sql create database $db cachemodel 'none' minrows 10 stt_trigger 1 sql use $db sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int) diff --git a/tests/script/tsim/parser/last_both_query.sim b/tests/script/tsim/parser/last_both_query.sim index 775fd482ea..5f86412199 100644 --- a/tests/script/tsim/parser/last_both_query.sim +++ b/tests/script/tsim/parser/last_both_query.sim @@ -357,95 +357,95 @@ if $data45 != 5 then return -1 endi -#sql select last_row(*), id from st2 group by id order by id -#print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -#print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 -#print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 -#print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 -#print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 -# -#if $rows != 5 then -# return -1 -#endi -#if $data00 != @21-05-12 10:10:12.000@ then -# return -1 -#endi -#if $data01 != 6 then -# return -1 -#endi -#if $data02 != NULL then -# print $data02 -# return -1 -#endi -#if $data03 != NULL then -# return -1 -#endi -#if $data04 != @70-01-01 07:59:57.000@ then -# return -1 -#endi -#if $data05 != 1 then -# return -1 -#endi -#if $data10 != @21-05-11 10:12:23.000@ then -# return -1 -#endi -#if $data11 != 22 then -# return -1 -#endi -#if $data12 != 23.000000000 then -# print $data02 -# return -1 -#endi -#if $data13 != NULL then -# return -1 -#endi -#if $data14 != @70-01-01 07:59:58.-04@ then -# return -1 -#endi -#if $data15 != 2 then -# return -1 -#endi -#if $data20 != @21-05-10 10:12:24.000@ then -# return -1 -#endi -#if $data21 != 24 then -# return -1 -#endi -#if $data22 != NULL then -# print expect NULL actual: $data22 -# return -1 -#endi -#if $data23 != 25 then -# return -1 -#endi -#if $data24 != @70-01-01 07:59:57.-04@ then = -# return -1 -#endi -#if $data25 != 3 then -# return -1 -#endi -#if $data30 != @21-05-11 10:12:25.000@ then -# return -1 -#endi -#if $data31 != 26 then -# return -1 -#endi -#if $data32 != NULL then -# print $data02 -# return -1 -#endi -#if $data33 != 27 then -# return -1 -#endi -#if $data34 != @70-01-01 07:59:56.-04@ then -# return -1 -#endi -#if $data35 != 4 then -# return -1 -#endi -#if $data40 != @21-05-11 10:12:29.000@ then -# return -1 -#endi +sql select last_row(*), id from st2 group by id order by id +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49 + +if $rows != 5 then + return -1 +endi +if $data00 != @21-05-12 10:10:12.000@ then + return -1 +endi +if $data01 != 6 then + return -1 +endi +if $data02 != NULL then + print $data02 + return -1 +endi +if $data03 != NULL then + return -1 +endi +if $data04 != @70-01-01 07:59:57.000@ then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data10 != @21-05-11 10:12:23.000@ then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data12 != 23.000000000 then + print $data02 + return -1 +endi +if $data13 != NULL then + return -1 +endi +if $data14 != @70-01-01 07:59:58.-04@ then + return -1 +endi +if $data15 != 2 then + return -1 +endi +if $data20 != @21-05-10 10:12:24.000@ then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data22 != NULL then + print expect NULL actual: $data22 + return -1 +endi +if $data23 != 25 then + return -1 +endi +if $data24 != @70-01-01 07:59:57.-04@ then = + return -1 +endi +if $data25 != 3 then + return -1 +endi +if $data30 != @21-05-11 10:12:25.000@ then + return -1 +endi +if $data31 != 26 then + return -1 +endi +if $data32 != NULL then + print $data02 + return -1 +endi +if $data33 != 27 then + return -1 +endi +if $data34 != @70-01-01 07:59:56.-04@ then + return -1 +endi +if $data35 != 4 then + return -1 +endi +if $data40 != @21-05-11 10:12:29.000@ then + return -1 +endi #if $data41 != NULL then # return -1 #endi @@ -453,15 +453,15 @@ endi # print $data02 # return -1 #endi -#if $data43 != NULL then -# return -1 -#endi +if $data43 != NULL then + return -1 +endi #if $data44 != NULL then # return -1 #endi -#if $data45 != 5 then -# return -1 -#endi +if $data45 != 5 then + return -1 +endi print "test tbn" sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) From aaa1c1d4a8dc2097058724f5a2a488d3b1392136 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Mon, 4 Mar 2024 09:35:08 +0800 Subject: [PATCH 34/54] fix: memleak --- source/libs/executor/src/tsort.c | 38 ++++++++++++++------------------ 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 9ff903cdb9..10220426a3 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -1199,6 +1199,18 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) { return code; } +static void freeSSortSource(SSortSource* source) { + if (NULL == source) return; + if (source->param && !source->onlyRef) { + taosMemoryFree(source->param); + } + if (!source->onlyRef && source->src.pBlock) { + blockDataDestroy(source->src.pBlock); + source->src.pBlock = NULL; + } + taosMemoryFree(source); +} + static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { int32_t code = 0; size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; @@ -1231,14 +1243,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { code = blockDataMerge(pHandle->pDataBlock, pBlock); if (code != TSDB_CODE_SUCCESS) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; - } - taosMemoryFree(source); + freeSSortSource(source); return code; } @@ -1248,15 +1253,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { int64_t p = taosGetTimestampUs(); code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); if (code != 0) { - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - if (!source->onlyRef && source->src.pBlock) { - blockDataDestroy(source->src.pBlock); - source->src.pBlock = NULL; - } - - taosMemoryFree(source); + freeSSortSource(source); return code; } @@ -1265,16 +1262,13 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) { if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows); code = doAddToBuf(pHandle->pDataBlock, pHandle); if (code != TSDB_CODE_SUCCESS) { + freeSSortSource(source); return code; } } } - if (source->param && !source->onlyRef) { - taosMemoryFree(source->param); - } - - taosMemoryFree(source); + freeSSortSource(source); if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) { size_t size = blockDataGetSize(pHandle->pDataBlock); From 80fd3e044546a16221816829e768ceaf095671cb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 4 Mar 2024 19:31:47 +0800 Subject: [PATCH 35/54] fix(stream): remove related fill-history if task in stop status. --- include/libs/stream/tstream.h | 2 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 36 +++++++++++++++------- source/libs/stream/src/streamTask.c | 14 +-------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index cf980400d6..0df6b76cf7 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -782,7 +782,7 @@ bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask); bool streamTaskSetSchedStatusWait(SStreamTask* pTask); int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask); int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask); -int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt, bool metaLock); +int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt); int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event); int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index a2d45062b9..8c66ecc6a9 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -142,8 +142,10 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); if (ppHTask == NULL || *ppHTask == NULL) { - tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", - vgId, req.taskId); + tqError( + "vgId:%d failed to acquire fill-history task:0x%x when handling update, may have been dropped already, rel " + "stream task:0x%x", + vgId, (uint32_t)pTask->hTaskInfo.id.taskId, req.taskId); CLEAR_RELATED_FILLHISTORY_TASK(pTask); } else { tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); @@ -612,23 +614,35 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; + int32_t vgId = pMeta->vgId; + STaskId hTaskId = {0}; - int32_t vgId = pMeta->vgId; tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - if (pTask != NULL) { - // drop the related fill-history task firstly + streamMetaWLock(pMeta); + + STaskId id = {.streamId = pReq->streamId, .taskId = pReq->taskId}; + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if ((ppTask != NULL) && ((*ppTask) != NULL)) { + streamMetaAcquireOneTask(*ppTask); + SStreamTask* pTask = *ppTask; + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - STaskId* pHTaskId = &pTask->hTaskInfo.id; - streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); - tqDebug("s-task:0x%x vgId:%d drop fill-history task:0x%x firstly", pReq->taskId, vgId, - (int32_t)pHTaskId->taskId); + hTaskId.streamId = pTask->hTaskInfo.id.streamId; + hTaskId.taskId = pTask->hTaskInfo.id.taskId; + streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); } + streamMetaReleaseTask(pMeta, pTask); } - streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt, true); + streamMetaWUnLock(pMeta); + + // drop the related fill-history task firstly + if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { + streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); + tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); + } // drop the stream task now streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 9639921c77..68b4ba2296 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -766,21 +766,13 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) { return status; } -int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool metaLock) { - if (pTask == NULL) { - return TSDB_CODE_SUCCESS; - } - +int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) { SStreamMeta* pMeta = pTask->pMeta; STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId}; if (pTask->info.fillHistory == 0) { return TSDB_CODE_SUCCESS; } - if (metaLock) { - streamMetaWLock(pMeta); - } - SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId)); if (ppStreamTask != NULL) { stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr, @@ -798,10 +790,6 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool taosThreadMutexUnlock(&(*ppStreamTask)->lock); } - if (metaLock) { - streamMetaWUnLock(pMeta); - } - return TSDB_CODE_SUCCESS; } From ad1780dbdc8211a7fec7414ffecb904c6aa52a89 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 4 Mar 2024 19:32:22 +0800 Subject: [PATCH 36/54] fix(stream): reset the pData after transferring state. --- source/libs/stream/inc/streamInt.h | 1 + source/libs/stream/src/streamTask.c | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 87f63b48ed..d0055d5400 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -99,6 +99,7 @@ void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration); int32_t streamDispatchStreamBlock(SStreamTask* pTask); void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups); int32_t getNumOfDispatchBranch(SStreamTask* pTask); +void clearBufferedDispatchMsg(SStreamTask* pTask); int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 68b4ba2296..8be5b94096 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -412,9 +412,7 @@ void tFreeStreamTask(SStreamTask* pTask) { pTask->pReadyMsgList = taosArrayDestroy(pTask->pReadyMsgList); if (pTask->msgInfo.pData != NULL) { - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - pTask->msgInfo.pData = NULL; - pTask->msgInfo.dispatchMsgType = 0; + clearBufferedDispatchMsg(pTask); } if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { From a38161299b2f64db97459d8f38e572e4624fc4ad Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 4 Mar 2024 19:51:01 +0800 Subject: [PATCH 37/54] fix(stream): add clear msgData buf impl. --- source/libs/stream/src/streamDispatch.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9702096b60..dc790b5b2d 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -315,6 +315,16 @@ int32_t getNumOfDispatchBranch(SStreamTask* pTask) { : taosArrayGetSize(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos); } +void clearBufferedDispatchMsg(SStreamTask* pTask) { + SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; + if (pMsgInfo->pData != NULL) { + destroyDispatchMsg(pMsgInfo->pData, getNumOfDispatchBranch(pTask)); + } + + pMsgInfo->pData = NULL; + pMsgInfo->dispatchMsgType = 0; +} + static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pData) { int32_t code = 0; int32_t numOfBlocks = taosArrayGetSize(pData->blocks); @@ -678,8 +688,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { // todo deal with only partially success dispatch case atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0); if (terrno == TSDB_CODE_APP_IS_STOPPING) { // in case of this error, do not retry anymore - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - pTask->msgInfo.pData = NULL; + clearBufferedDispatchMsg(pTask); return code; } @@ -938,15 +947,12 @@ void streamClearChkptReadyMsg(SStreamTask* pTask) { // this message has been sent successfully, let's try next one. static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId) { stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); - destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); - bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); if (delayDispatch) { pTask->chkInfo.dispatchCheckpointTrigger = true; } - pTask->msgInfo.pData = NULL; - pTask->msgInfo.dispatchMsgType = 0; + clearBufferedDispatchMsg(pTask); int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs; @@ -1086,7 +1092,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } else { // this message has been sent successfully, let's try next one. pTask->msgInfo.retryCount = 0; - // transtate msg has been sent to downstream successfully. let's transfer the fill-history task state + // trans-state msg has been sent to downstream successfully. let's transfer the fill-history task state if (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__TRANS_STATE) { stDebug("s-task:%s dispatch transtate msgId:%d to downstream successfully, start to transfer state", id, msgId); ASSERT(pTask->info.fillHistory == 1); @@ -1095,6 +1101,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (code != TSDB_CODE_SUCCESS) { // todo: do nothing if error happens } + clearBufferedDispatchMsg(pTask); + // now ready for next data output atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); } else { From 09b991d97ad6672a215fd345b08aa414f254a1fc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 4 Mar 2024 12:24:00 +0000 Subject: [PATCH 38/54] fix mem leak --- source/libs/transport/src/transSvr.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 5a1ef31b7d..f47a688e6f 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -159,7 +159,7 @@ static void uvStartSendResp(SSvrMsg* msg); static void uvNotifyLinkBrokenToApp(SSvrConn* conn); -static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); +static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); static FORCE_INLINE SSvrConn* createConn(void* hThrd); static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn); @@ -527,6 +527,10 @@ void uvOnSendCb(uv_write_t* req, int status) { if (!transQueueEmpty(&conn->srvMsgs)) { msg = (SSvrMsg*)transQueueGet(&conn->srvMsgs, 0); if (msg->type == Register && conn->status == ConnAcquire) { + if (conn->regArg.init) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } conn->regArg.notifyCount = 0; conn->regArg.init = 1; conn->regArg.msg = msg->msg; @@ -1350,6 +1354,11 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) { return; } transQueuePop(&conn->srvMsgs); + + if (conn->regArg.init) { + transFreeMsg(conn->regArg.msg.pCont); + conn->regArg.init = 0; + } conn->regArg.notifyCount = 0; conn->regArg.init = 1; conn->regArg.msg = msg->msg; From 2ac803bf3c5975185f25be9fb62350dc945ce935 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 4 Mar 2024 22:16:41 +0800 Subject: [PATCH 39/54] fix(stream): clear and set task status. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 8c66ecc6a9..6a5bd444ef 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -630,9 +630,9 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { hTaskId.streamId = pTask->hTaskInfo.id.streamId; hTaskId.taskId = pTask->hTaskInfo.id.taskId; - streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); } + streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); streamMetaReleaseTask(pMeta, pTask); } From af6d6b69c3380385a6effe88d0c557cc4735e693 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 5 Mar 2024 11:25:43 +0800 Subject: [PATCH 40/54] fix:add excluded msg for delete in tmq --- include/common/tmsg.h | 1 - include/libs/scheduler/scheduler.h | 1 + source/client/inc/clientInt.h | 5 +++-- source/client/src/clientImpl.c | 9 ++++++--- source/client/src/clientMain.c | 6 +++--- source/client/src/clientRawBlockWrite.c | 4 +--- source/common/src/tmsg.c | 1 - source/libs/scheduler/inc/schInt.h | 1 + source/libs/scheduler/src/schJob.c | 1 + source/libs/scheduler/src/schRemote.c | 2 +- 10 files changed, 17 insertions(+), 14 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index e400698f69..4fa244a07c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3816,7 +3816,6 @@ typedef struct { int8_t source; } SVDeleteReq; -extern int8_t deleteFromTaosx; int32_t tSerializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq); int32_t tDeserializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq); diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 958d63349d..952af3c443 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -78,6 +78,7 @@ typedef struct SSchedulerReq { void* chkKillParam; SExecResult* pExecRes; void** pFetchRes; + int8_t source; } SSchedulerReq; int32_t schedulerInit(void); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index f73d121b15..6cc2a62236 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -284,6 +284,7 @@ typedef struct SRequestObj { void* pWrapper; SMetaData parseMeta; char* effectiveUser; + int8_t source; } SRequestObj; typedef struct SSyncQueryParam { @@ -306,10 +307,10 @@ void doFreeReqResultInfo(SReqResultInfo* pResInfo); int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); -TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly); +TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source); TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, int64_t reqid); -void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); +void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source); void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int64_t reqid); void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 971ed407f9..aa18ef1c94 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -729,6 +729,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList .chkKillFp = chkRequestKilled, .chkKillParam = (void*)pRequest->self, .pExecRes = &res, + .source = pRequest->source, }; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob); @@ -1198,6 +1199,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat .chkKillFp = chkRequestKilled, .chkKillParam = (void*)pRequest->self, .pExecRes = NULL, + .source = pRequest->source, }; code = schedulerExecJob(&req, &pRequest->body.queryJob); taosArrayDestroy(pNodeList); @@ -2461,7 +2463,7 @@ void syncQueryFn(void* param, void* res, int32_t code) { tsem_post(&pParam->sem); } -void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) { +void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source) { if (sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; if (fp) { @@ -2487,6 +2489,7 @@ void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, return; } + pRequest->source = source; pRequest->body.queryFp = fp; doAsyncQuery(pRequest, false); } @@ -2521,7 +2524,7 @@ void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_ doAsyncQuery(pRequest, false); } -TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { +TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; @@ -2536,7 +2539,7 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { } tsem_init(¶m->sem, 0, 0); - taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly); + taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, source); tsem_wait(¶m->sem); SRequestObj* pRequest = NULL; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index f6aef5aa26..eacad0f18f 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -401,7 +401,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return pResInfo->userFields; } -TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false); } +TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false, TD_REQ_FROM_APP); } TAOS_RES *taos_query_with_reqid(TAOS *taos, const char *sql, int64_t reqid) { return taosQueryImplWithReqid(taos, sql, false, reqid); } @@ -827,7 +827,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { } int taos_validate_sql(TAOS *taos, const char *sql) { - TAOS_RES *pObj = taosQueryImpl(taos, sql, true); + TAOS_RES *pObj = taosQueryImpl(taos, sql, true, TD_REQ_FROM_APP); int code = taos_errno(pObj); @@ -1125,7 +1125,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { int64_t connId = *(int64_t *)taos; tscDebug("taos_query_a start with sql:%s", sql); - taosAsyncQueryImpl(connId, sql, fp, param, false); + taosAsyncQueryImpl(connId, sql, fp, param, false, TD_REQ_FROM_APP); tscDebug("taos_query_a end with sql:%s", sql); } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 3857e47f57..adc8c361cd 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1256,9 +1256,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName, req.tsColName, req.skey, req.tsColName, req.ekey); - deleteFromTaosx = TD_REQ_FROM_TAOX; - TAOS_RES* res = taos_query(taos, sql); - deleteFromTaosx = TD_REQ_FROM_APP; + TAOS_RES* res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX); SRequestObj* pRequest = (SRequestObj*)res; code = pRequest->code; if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_PAR_GET_META_ERROR) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index e7414c6004..467300352c 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -7148,7 +7148,6 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { return 0; } -int8_t deleteFromTaosx = TD_REQ_FROM_APP; int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 1c90e61ea3..276b668717 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -304,6 +304,7 @@ typedef struct SSchJob { SSchResInfo userRes; char *sql; SQueryProfileSummary summary; + int8_t source; } SSchJob; typedef struct SSchTaskCtx { diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index e50ec64d54..48aab63ba3 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -746,6 +746,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->chkKillParam = pReq->chkKillParam; pJob->userRes.execFp = pReq->execFp; pJob->userRes.cbParam = pReq->cbParam; + pJob->source = pReq->source; if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) { qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index f6465316d2..39273ffa50 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1086,7 +1086,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, req.sqlLen = strlen(pJob->sql); req.sql = (char *)pJob->sql; req.msg = pTask->msg; - req.source = deleteFromTaosx; + req.source = pJob->source; msgSize = tSerializeSVDeleteReq(NULL, 0, &req); msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { From cdfbbc298535858df1a8a65de45803cee3855b6e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 5 Mar 2024 13:28:34 +0800 Subject: [PATCH 41/54] fix(stream): set the dest epset from mnode, and set the retry error code for trans. --- source/dnode/mnode/impl/inc/mndStream.h | 2 +- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndStreamUtil.c | 16 ++++++++++++---- source/libs/stream/src/streamTask.c | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 57fd187da3..5307ff4b05 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -111,7 +111,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo); -int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans); +int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans); SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId); int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3ef2f64df7..fca757006d 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1811,7 +1811,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid, pStream->name, pTrans->id); - int32_t code = mndStreamSetUpdateEpsetAction(pStream, pChangeInfo, pTrans); + int32_t code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans); // todo: not continue, drop all and retry again if (code != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 1ae85a2cc6..2b8fcee9fd 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -462,14 +462,22 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha return TSDB_CODE_SUCCESS; } -static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { +static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { void *pBuf = NULL; int32_t len = 0; streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); - int32_t code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0); + SEpSet epset = {0}; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS || !hasEpset) { + terrno = code; + return code; + } + + code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(pBuf); } @@ -478,14 +486,14 @@ static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroup } // build trans to update the epset -int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) { +int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) { mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid); taosWLockLatch(&pStream->lock); SStreamTaskIter *pIter = createStreamTaskIter(pStream); while (streamTaskIterNextTask(pIter)) { SStreamTask *pTask = streamTaskIterGetCurrent(pIter); - int32_t code = doSetUpdateTaskAction(pTrans, pTask, pInfo); + int32_t code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo); if (code != TSDB_CODE_SUCCESS) { destroyStreamTaskIter(pIter); taosWUnLockLatch(&pStream->lock); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 8be5b94096..6fd96f651e 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -39,7 +39,7 @@ static int32_t doUpdateTaskEpset(SStreamTask* pTask, int32_t nodeId, SEpSet* pEp stDebug("s-task:0x%x (vgId:%d) self node epset is updated %s", pTask->id.taskId, nodeId, buf); } - // check for the dispath info and the upstream task info + // check for the dispatch info and the upstream task info int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__SOURCE) { streamTaskUpdateDownstreamInfo(pTask, nodeId, pEpSet); From 254afc070c8050cfd3fb5f8fddff4eff7841ffec Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Tue, 5 Mar 2024 15:20:11 +0800 Subject: [PATCH 42/54] fix: dynmaic config supportvnodes in ent --- source/common/src/tglobal.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 28 +++++--------------------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 55df80ca44..fde8313228 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -586,7 +586,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfSupportVnodes = tsNumOfCores * 2; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); - if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 79a5f5fd83..befb6d3521 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -610,7 +610,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { for(int32_t j = 0; j < tagSize; j++){ SJson* item = tjsonGetArrayItem(arrayTag, j); - *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); + *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); tjsonGetStringValue(item, "name", *(labels + j)); *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); @@ -626,7 +626,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { for(int32_t j = 0; j < metricLen; j++){ SJson *item = tjsonGetArrayItem(metrics, j); - char name[MONITOR_METRIC_NAME_LEN] = {0}; + char name[MONITOR_METRIC_NAME_LEN] = {0}; tjsonGetStringValue(item, "name", name); double value = 0; @@ -636,7 +636,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { tjsonGetDoubleValue(item, "type", &type); int32_t metricNameLen = strlen(name) + strlen(tableName) + 2; - char* metricName = taosMemoryMalloc(metricNameLen); + char* metricName = taosMemoryMalloc(metricNameLen); memset(metricName, 0, metricNameLen); sprintf(metricName, "%s:%s", tableName, name); @@ -669,7 +669,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { else{ mTrace("get metric from registry:%p", metric); } - + if(type == 0){ taos_counter_add(metric, value, (const char**)sample_labels); } @@ -689,7 +689,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { taosMemoryFreeClear(labels); } } - + } code = 0; @@ -1409,24 +1409,6 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { if (strcasecmp(cfgReq.config, "resetlog") == 0) { strcpy(dcfgReq.config, "resetlog"); #ifdef TD_ENTERPRISE - } else if (strncasecmp(cfgReq.config, "supportvnodes", 13) == 0) { - int32_t optLen = strlen("supportvnodes"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag < 0 || flag > 4096) { - mError("dnode:%d, failed to config supportVnodes since value:%d. Valid range: [0, 4096]", cfgReq.dnodeId, flag); - terrno = TSDB_CODE_OUT_OF_RANGE; - goto _err_out; - } - if (flag == 0) { - flag = tsNumOfCores * 2; - } - flag = TMAX(flag, 2); - - strcpy(dcfgReq.config, "supportvnodes"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { int32_t optLen = strlen("s3blocksize"); int32_t flag = -1; From 35c5dca341820f37ac9d519ddbd66651bd757f4a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 5 Mar 2024 07:42:29 +0000 Subject: [PATCH 43/54] fix count error --- source/libs/transport/src/transCli.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 6ae72eac14..2e35aecd08 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -584,8 +584,8 @@ void* destroyConnPool(SCliThrd* pThrd) { static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) { void* pool = pThrd->pool; - SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1); STrans* pTranInst = pThrd->pTransInst; + SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1); if (plist == NULL) { SConnList list = {0}; taosHashPut((SHashObj*)pool, key, strlen(key) + 1, (void*)&list, sizeof(list)); @@ -867,17 +867,18 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { QUEUE_INIT(&conn->q); conn->broken = true; + if (conn->list == NULL) { + conn->list = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr)); + } - if (conn->list != NULL) { - SConnList* connList = conn->list; - connList->list->numOfConn--; - connList->size--; - } else { - if (pThrd->pool) { - SConnList* connList = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr) + 1); - if (connList != NULL) connList->list->numOfConn--; + if (conn->list) { + SConnList* list = conn->list; + list->list->numOfConn--; + if (conn->status == ConnInPool) { + list->size--; } } + conn->list = NULL; pThrd->newConnCount--; From 45e779d464cf41fb60aa7e51bf940d6255c0171f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 5 Mar 2024 15:48:58 +0800 Subject: [PATCH 44/54] refactor: do some internal refactor. --- source/libs/stream/src/streamTask.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 6fd96f651e..4dc01ee9b4 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -622,6 +622,7 @@ void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDo void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet) { char buf[512] = {0}; EPSET_TO_STR(pEpSet, buf); + int32_t id = pTask->id.taskId; int8_t type = pTask->outputInfo.type; if (type == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -633,8 +634,8 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE if (pVgInfo->vgId == nodeId) { epsetAssign(&pVgInfo->epSet, pEpSet); - stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", pTask->id.taskId, - pVgInfo->taskId, nodeId, buf); + stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pVgInfo->taskId, nodeId, + buf); break; } } @@ -642,8 +643,8 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE STaskDispatcherFixed* pDispatcher = &pTask->outputInfo.fixedDispatcher; if (pDispatcher->nodeId == nodeId) { epsetAssign(&pDispatcher->epSet, pEpSet); - stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpSet:%s", pTask->id.taskId, - pDispatcher->taskId, nodeId, buf); + stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pDispatcher->taskId, nodeId, + buf); } } else { // do nothing From 4d1b556bc50d51584464042d4dbad9b8d8fae1d5 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 5 Mar 2024 18:54:07 +0800 Subject: [PATCH 45/54] load operator checkpoint --- source/libs/executor/src/streamcountwindowoperator.c | 4 ++-- source/libs/executor/src/streameventwindowoperator.c | 4 ++-- source/libs/executor/src/streamtimewindowoperator.c | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index f9c7b51316..d9ce10d38a 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -694,6 +694,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->recvGetAll = false; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT; + setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, true, + OP_NOT_OPENED, pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -704,8 +706,6 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys doStreamCountDecodeOpState(buff, len, pOperator, true); taosMemoryFree(buff); } - setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, true, - OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAgg, NULL, destroyStreamCountAggOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState); diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 4a2fe2416f..10b4a5c344 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -735,6 +735,8 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->reCkBlock = false; pInfo->recvGetAll = false; + setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED, + pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -746,8 +748,6 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys taosMemoryFree(buff); } - setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED, - pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 2838d005ab..00877ae1b1 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2992,6 +2992,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->recvGetAll = false; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; + setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, + OP_NOT_OPENED, pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -3002,8 +3004,6 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh doStreamSessionDecodeOpState(buff, len, pOperator, true); taosMemoryFree(buff); } - setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, - OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); @@ -3873,6 +3873,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); pInfo->recvGetAll = false; + setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, + pInfo, pTaskInfo); // for stream void* buff = NULL; int32_t len = 0; @@ -3884,8 +3886,6 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys taosMemoryFree(buff); } - setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, - pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState); From 57baceeeaf2bc939f1f2e743c05140c005878d90 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 5 Mar 2024 19:14:20 +0800 Subject: [PATCH 46/54] load operator checkpoint --- source/libs/executor/src/streamcountwindowoperator.c | 8 +++++--- source/libs/executor/src/streamtimewindowoperator.c | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index d9ce10d38a..720734431f 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -390,10 +390,12 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato buf = taosDecodeFixedI32(buf, &mapSize); for (int32_t i = 0; i < mapSize; i++) { SSessionKey key = {0}; - SResultWindowInfo winfo = {0}; + SCountWindowInfo curWin = {0}; buf = decodeSSessionKey(buf, &key); - buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); - tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; + setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); + buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); } // 2.twAggSup diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 00877ae1b1..3b62cc80e3 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2533,7 +2533,6 @@ int32_t encodeSResultWindowInfo(void** buf, SResultWindowInfo* key, int32_t outL void* decodeSResultWindowInfo(void* buf, SResultWindowInfo* key, int32_t outLen) { buf = taosDecodeFixedBool(buf, &key->isOutput); - key->pStatePos->pRowBuff = NULL; buf = decodeSSessionKey(buf, &key->sessionWin); return buf; } From bd430a031ab147f3759e9f02bd327dabd4979af4 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Tue, 5 Mar 2024 19:37:54 +0800 Subject: [PATCH 47/54] load operator checkpoint --- source/libs/executor/src/streameventwindowoperator.c | 3 +++ source/libs/executor/src/streamtimewindowoperator.c | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 10b4a5c344..11d8d1487a 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -406,6 +406,7 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato if (!pInfo) { return buf; } + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; // 4.checksum int32_t dataLen = len - sizeof(uint32_t); @@ -423,6 +424,8 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); + pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap, + (void**)&winfo.pStatePos, &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 3b62cc80e3..7a1bb2729c 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2590,6 +2590,7 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera if (!pInfo) { return buf; } + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; // 5.checksum if (isParent) { @@ -2608,6 +2609,8 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); + pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap, + (void**)&winfo.pStatePos, &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } @@ -3537,6 +3540,7 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato if (!pInfo) { return buf; } + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; // 5.checksum if (isParent) { @@ -3555,6 +3559,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato SSessionKey key = {0}; SResultWindowInfo winfo = {0}; buf = decodeSSessionKey(buf, &key); + pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, + pAggSup->stateKeySize, compareStateKey, + (void**)&winfo.pStatePos, &pAggSup->resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); } From 7d3bfb0da91abe6236df97d8cdc8d8438a3467ad Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 5 Mar 2024 12:05:31 +0000 Subject: [PATCH 48/54] fix mem leak --- source/libs/scheduler/src/schRemote.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1c0b31109e..be21858e6a 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -940,6 +940,10 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery SCH_ERR_JRET(schGenerateCallBackInfo(pJob, pTask, msg, msgSize, msgType, trans, isHb, &pMsgSendInfo)); SCH_ERR_JRET(schUpdateSendTargetInfo(pMsgSendInfo, addr, pTask)); + if (isHb && persistHandle && trans->pHandle == 0) { + trans->pHandle = rpcAllocHandle(); + } + if (pJob && pTask) { SCH_TASK_DLOG("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId, epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle); From 3416d684335b92c67f32f0057233ce58a5a004c7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 5 Mar 2024 21:02:20 +0800 Subject: [PATCH 49/54] add coverity scan --- source/libs/stream/src/streamBackendRocksdb.c | 2 + source/libs/stream/test/backendTest.cpp | 56 +++++++++++++------ 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 910fd93989..5dafad8951 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -906,6 +906,7 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { return 0; } +#ifdef BUILD_NO_CALL static int32_t chkpIdComp(const void* a, const void* b) { int64_t x = *(int64_t*)a; int64_t y = *(int64_t*)b; @@ -964,6 +965,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) { taosMemoryFree(chkpPath); return 0; } +#endif #ifdef BUILD_NO_CALL int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) { diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp index a949748eb5..c9b981e5f9 100644 --- a/source/libs/stream/test/backendTest.cpp +++ b/source/libs/stream/test/backendTest.cpp @@ -29,7 +29,7 @@ class BackendEnv : public ::testing::Test { void *backendCreate() { const char *streamPath = "/tmp"; - void *p = NULL; + void * p = NULL; // char *absPath = NULL; // // SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(streamPath, -1, 2); @@ -52,7 +52,7 @@ SStreamState *stateCreate(const char *path) { } void *backendOpen() { streamMetaInit(); - const char *path = "/tmp/backend"; + const char * path = "/tmp/backend"; SStreamState *p = stateCreate(path); ASSERT(p != NULL); @@ -79,7 +79,7 @@ void *backendOpen() { const char *val = "value data"; int32_t len = 0; - char *newVal = NULL; + char * newVal = NULL; streamStateGet_rocksdb(p, &key, (void **)&newVal, &len); ASSERT(len == strlen(val)); } @@ -100,7 +100,7 @@ void *backendOpen() { const char *val = "value data"; int32_t len = 0; - char *newVal = NULL; + char * newVal = NULL; int32_t code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len); ASSERT(code != 0); } @@ -130,7 +130,7 @@ void *backendOpen() { winkey.groupId = 0; winkey.ts = tsArray[0]; - char *val = NULL; + char * val = NULL; int32_t len = 0; pCurr = streamStateSeekKeyNext_rocksdb(p, &winkey); @@ -157,7 +157,7 @@ void *backendOpen() { key.ts = tsArray[i]; key.exprIdx = i; - char *val = NULL; + char * val = NULL; int32_t len = 0; streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len); ASSERT(len == strlen("Value")); @@ -168,7 +168,7 @@ void *backendOpen() { key.ts = tsArray[i]; key.exprIdx = i; - char *val = NULL; + char * val = NULL; int32_t len = 0; streamStateFuncDel_rocksdb(p, &key); } @@ -213,7 +213,7 @@ void *backendOpen() { { SSessionKey key; memset(&key, 0, sizeof(key)); - char *val = NULL; + char * val = NULL; int32_t vlen = 0; code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen); ASSERT(code == 0); @@ -260,7 +260,7 @@ void *backendOpen() { SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; key.groupId = (uint64_t)(i); key.ts = tsArray[i]; - char *val = NULL; + char * val = NULL; int32_t vlen = 0; ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0); taosMemoryFreeClear(val); @@ -272,7 +272,7 @@ void *backendOpen() { SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key); ASSERT(pCurr != NULL); - char *val = NULL; + char * val = NULL; int32_t vlen = 0; ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen)); ASSERT(vlen == strlen("Value")); @@ -296,7 +296,7 @@ void *backendOpen() { SWinKey key = {0}; // {.groupId = (uint64_t)(i), .ts = tsArray[i]}; key.groupId = (uint64_t)(i); key.ts = tsArray[i]; - char *val = NULL; + char * val = NULL; int32_t vlen = 0; ASSERT(streamStateFillDel_rocksdb(p, &key) == 0); taosMemoryFreeClear(val); @@ -338,7 +338,7 @@ void *backendOpen() { char key[128] = {0}; sprintf(key, "tbname_%d", i); - char *val = NULL; + char * val = NULL; int32_t len = 0; code = streamDefaultGet_rocksdb(p, key, (void **)&val, &len); ASSERT(code == 0); @@ -354,7 +354,7 @@ TEST_F(BackendEnv, checkOpen) { SStreamState *p = (SStreamState *)backendOpen(); int64_t tsStart = taosGetTimestampMs(); { - void *pBatch = streamStateCreateBatch(); + void * pBatch = streamStateCreateBatch(); int32_t size = 0; for (int i = 0; i < size; i++) { char key[128] = {0}; @@ -368,7 +368,7 @@ TEST_F(BackendEnv, checkOpen) { streamStateDestroyBatch(pBatch); } { - void *pBatch = streamStateCreateBatch(); + void * pBatch = streamStateCreateBatch(); int32_t size = 0; char valBuf[256] = {0}; for (int i = 0; i < size; i++) { @@ -385,7 +385,7 @@ TEST_F(BackendEnv, checkOpen) { // do checkpoint 2 taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2); { - void *pBatch = streamStateCreateBatch(); + void * pBatch = streamStateCreateBatch(); int32_t size = 0; char valBuf[256] = {0}; for (int i = 0; i < size; i++) { @@ -407,12 +407,22 @@ TEST_F(BackendEnv, checkOpen) { // taosMkDir(dump); taosMulMkDir(dump); SBkdMgt *mgt = bkdMgtCreate((char *)path); - SArray *result = taosArrayInit(4, sizeof(void *)); + SArray * result = taosArrayInit(4, sizeof(void *)); bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump); + taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 4); + + taosArrayClear(result); + bkdMgtGetDelta(mgt, p->pTdbState->idstr, 4, result, (char *)dump); bkdMgtDestroy(mgt); streamStateClose((SStreamState *)p, true); + // { + // taosRemoveDir("/tmp/backend"); + // const char * path = "/tmp/backend"; + // SStreamState *p = stateCreate(path); + // } taosRemoveDir(path); + // streamStateClose((SStreamState *)p, true); } TEST_F(BackendEnv, backendChkp) { const char *path = "/tmp"; } @@ -430,6 +440,20 @@ TEST_F(BackendEnv, backendUtil) { ASSERT_EQ(nextPow2((uint32_t)(kvDict[i].k)), kvDict[i].v); } } +TEST_F(BackendEnv, oldBackendInit) { + const char *path = "/tmp/backend1"; + taosMulMkDir(path); + { + SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(path, 10, 10); + streamBackendCleanup((void *)p); + } + { + SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(path, 10, 10); + streamBackendCleanup((void *)p); + } + + taosRemoveDir(path); +} int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); From 3afec5e224d1ac496a09de4fd6eca793c0e05583 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 6 Mar 2024 10:44:36 +0800 Subject: [PATCH 50/54] fix: join table fetch error --- source/libs/executor/src/dynqueryctrloperator.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index 9e7f1144f8..e7c9c17a0b 100755 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -456,7 +456,8 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS code = pInfo->stbJoin.basic.srcScan[1] ? buildSingleTableScanOperatorParam(&pSrcParam1, 1, rightVg, rightUid) : buildExchangeOperatorParam(&pSrcParam1, 1, rightVg, rightUid); } } - + + bool initParam = pSrcParam0 ? true : false; if (TSDB_CODE_SUCCESS == code) { code = buildGroupCacheOperatorParam(&pGcParam0, 0, *leftVg, *leftUid, pPost->leftNeedCache, pSrcParam0); pSrcParam0 = NULL; @@ -466,7 +467,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS pSrcParam1 = NULL; } if (TSDB_CODE_SUCCESS == code) { - code = buildMergeJoinOperatorParam(ppParam, pSrcParam0 ? true : false, pGcParam0, pGcParam1); + code = buildMergeJoinOperatorParam(ppParam, initParam, pGcParam0, pGcParam1); } if (TSDB_CODE_SUCCESS != code) { if (pSrcParam0) { From f092eead38d726863dff778f1ae5b9ee5ba5f096 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 6 Mar 2024 17:22:25 +0800 Subject: [PATCH 51/54] test:comment tmqparamstest test with restful --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c32790022a..a2348bdedd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -115,7 +115,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ins_topics_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxTopic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py From 02de597e69aa427f816cdd195ef82a59dc79626d Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 7 Mar 2024 09:00:40 +0800 Subject: [PATCH 52/54] opt ci --- tests/script/tsim/stream/state0.sim | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 0d96d0841b..37081ad1ae 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -21,6 +21,8 @@ print create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); +sleep 1000 + sql insert into t1 values(1648791213000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,3,1.0,2); $loop_count = 0 @@ -457,6 +459,8 @@ print create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); +sleep 1000 + sql insert into t1 values(1648791212000,2,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,1,2,4,1.0,2); @@ -504,6 +508,9 @@ sql create table t1(ts timestamp, a int, b int , c int, d double, id int); print create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1, sum(b) c3 from t1 state_window(a); sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart, count(*) c1, sum(b) c3 from t1 state_window(a); + +sleep 1000 + sql insert into t1 values(1648791212000,1,2,3,1.0,1); sql insert into t1 values(1648791213000,2,2,3,1.0,1); sql insert into t1 values(1648791214000,3,2,4,1.0,2); @@ -557,6 +564,8 @@ print create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 sql create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1); +sleep 1000 + sql insert into t1 (ts, c1) values (1668073288209, 11); sql insert into t1 (ts, c1) values (1668073288210, 11); sql insert into t1 (ts, c1) values (1668073288211, 11); @@ -745,6 +754,9 @@ sql create table b (c timestamp, d int, e int , f int, g double); print create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a); + +sleep 1000 + sql insert into b values(1648791213000,NULL,NULL,NULL,NULL); sql select * from streamt order by c1, c2, c3; From f26b7f78f5adbfe6c8ca6ecc85428bb5942c14eb Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 7 Mar 2024 14:23:30 +0800 Subject: [PATCH 53/54] calc stream tag size --- source/libs/parser/src/parTranslater.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3a622eeea6..491768e9ab 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8135,7 +8135,9 @@ static int32_t adjustTagsForCreateTable(STranslateContext* pCxt, SCreateStreamSt SColumnDefNode* pDef = (SColumnDefNode*)pTagDef; if (!dataTypeEqual(&pDef->dataType, &((SExprNode*)pTagExpr)->resType)) { SNode* pFunc = NULL; - int32_t code = createCastFunc(pCxt, pTagExpr, pDef->dataType, &pFunc); + SDataType defType = pDef->dataType; + defType.bytes = calcTypeBytes(defType); + int32_t code = createCastFunc(pCxt, pTagExpr, defType, &pFunc); if (TSDB_CODE_SUCCESS != code) { return code; } From 7c319adf46e9bbbde24c5ff98afea028d1ef2b8e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 7 Mar 2024 17:59:02 +0800 Subject: [PATCH 54/54] fix:[TS-4544]do not get vgroup if subscribe query --- source/libs/parser/src/parTranslater.c | 2 +- tests/system-test/7-tmq/tmq_taosx.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3a622eeea6..291c04d8c4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4547,7 +4547,7 @@ static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = getQueryTimeRange(pCxt, pSelect->pWhere, &pSelect->timeRange); } - if (pSelect->pWhere != NULL) { + if (pSelect->pWhere != NULL && pCxt->pParseCxt->topicQuery == false) { setTableVgroupsFromEqualTbnameCond(pCxt, pSelect); } return code; diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index c547385d70..b8f3c944b0 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -359,8 +359,34 @@ class TDTestCase: finally: consumer.close() + def consume_ts_4544(self): + tdSql.execute(f'create database if not exists d1') + tdSql.execute(f'use d1') + tdSql.execute(f'create table stt(ts timestamp, i int) tags(t int)') + tdSql.execute(f'insert into tt1 using stt tags(1) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into tt2 using stt tags(2) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into tt3 using stt tags(3) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into tt1 using stt tags(1) values(now+5s, 11) (now+10s, 12)') + + tdSql.execute(f'create topic topic_in as select * from stt where tbname in ("tt2")') + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + } + consumer = Consumer(consumer_dict) + + try: + consumer.subscribe(["topic_in"]) + except TmqError: + tdLog.exit(f"subscribe error") + + consumer.close() + def run(self): self.consumeTest() + self.consume_ts_4544() tdSql.prepare() self.checkWal1VgroupOnlyMeta()