From 616539a70083f495140070c6c09cd8ae2fa6970e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 23 Jul 2022 17:30:00 +0800 Subject: [PATCH 01/27] feat: super table order by primary key optimization --- source/libs/planner/src/planOptimizer.c | 24 +++++++++---------- source/libs/planner/src/planPhysiCreater.c | 1 - source/libs/planner/src/planSpliter.c | 14 ++++++++++- source/libs/planner/src/planUtil.c | 3 ++- source/libs/planner/test/planOptimizeTest.cpp | 2 ++ 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index b006ac2b0a..f107b2343c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -997,10 +997,7 @@ static int32_t sortPriKeyOptGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimi switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - if (TSDB_SUPER_TABLE != ((SScanLogicNode*)pNode)->tableType) { - return nodesListMakeAppend(pScanNodes, (SNode*)pNode); - } - break; + return nodesListMakeAppend(pScanNodes, (SNode*)pNode); case QUERY_NODE_LOGIC_PLAN_JOIN: code = sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); @@ -1040,13 +1037,16 @@ static EOrder sortPriKeyOptGetPriKeyOrder(SSortLogicNode* pSort) { static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort, SNodeList* pScanNodes) { EOrder order = sortPriKeyOptGetPriKeyOrder(pSort); - if (ORDER_DESC == order) { - SNode* pScanNode = NULL; - FOREACH(pScanNode, pScanNodes) { - SScanLogicNode* pScan = (SScanLogicNode*)pScanNode; - if (pScan->scanSeq[0] > 0) { - TSWAP(pScan->scanSeq[0], pScan->scanSeq[1]); - } + SNode* pScanNode = NULL; + FOREACH(pScanNode, pScanNodes) { + SScanLogicNode* pScan = (SScanLogicNode*)pScanNode; + if (ORDER_DESC == order && pScan->scanSeq[0] > 0) { + TSWAP(pScan->scanSeq[0], pScan->scanSeq[1]); + } + if (TSDB_SUPER_TABLE == pScan->tableType) { + pScan->scanType = SCAN_TYPE_TABLE_MERGE; + pScan->node.resultDataOrder = DATA_ORDER_LEVEL_GLOBAL; + pScan->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; } } @@ -2191,7 +2191,7 @@ static bool tagScanMayBeOptimized(SLogicNode* pNode) { !planOptNodeListHasTbname(pAgg->pGroupKeys)) { return false; } - + SNode* pGroupKey = NULL; FOREACH(pGroupKey, pAgg->pGroupKeys) { SNode* pGroup = NULL; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 0a1f8bbd0b..fd359fce45 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -415,7 +415,6 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS SScanPhysiNode* pScanPhysiNode, SPhysiNode** pPhyNode) { int32_t code = createScanCols(pCxt, pScanPhysiNode, pScanLogicNode->pScanCols); if (TSDB_CODE_SUCCESS == code) { - // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t code = addDataBlockSlots(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 8586234b7e..10604fac19 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -913,13 +913,25 @@ static int32_t stbSplSplitScanNodeWithPartTags(SSplitContext* pCxt, SStableSplit } static SNode* stbSplFindPrimaryKeyFromScan(SScanLogicNode* pScan) { + bool find = false; SNode* pCol = NULL; FOREACH(pCol, pScan->pScanCols) { if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pCol)->colId) { + find = true; + break; + } + } + if (!find) { + return NULL; + } + SNode* pTarget = NULL; + FOREACH(pTarget, pScan->node.pTargets) { + if (nodesEqualNode(pTarget, pCol)) { return pCol; } } - return NULL; + nodesListStrictAppend(pScan->node.pTargets, nodesCloneNode(pCol)); + return pCol; } static int32_t stbSplSplitMergeScanNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SScanLogicNode* pScan, diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index bfa6079cb1..7aab8a7ca3 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -124,7 +124,8 @@ int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* } static int32_t adjustScanDataRequirement(SScanLogicNode* pScan, EDataOrderLevel requirement) { - if (SCAN_TYPE_TABLE != pScan->scanType && SCAN_TYPE_TABLE_MERGE != pScan->scanType) { + if ((SCAN_TYPE_TABLE != pScan->scanType && SCAN_TYPE_TABLE_MERGE != pScan->scanType) || + DATA_ORDER_LEVEL_GLOBAL == pScan->node.requireDataOrder) { return TSDB_CODE_SUCCESS; } // The lowest sort level of scan output data is DATA_ORDER_LEVEL_IN_BLOCK diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 770ac94e5b..058705403b 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -53,6 +53,8 @@ TEST_F(PlanOptimizeTest, sortPrimaryKey) { run("SELECT c1 FROM t1 ORDER BY ts"); + run("SELECT c1 FROM st1 ORDER BY ts"); + run("SELECT c1 FROM t1 ORDER BY ts DESC"); run("SELECT COUNT(*) FROM t1 INTERVAL(10S) ORDER BY _WSTART DESC"); From 0a65932afe1d12da23f7d8ad1fe7e7d8a8c5b688 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 23 Jul 2022 18:57:38 +0800 Subject: [PATCH 02/27] test: add test cases for tmq --- .../7-tmq/dataFromTsdbNWal-multiCtb.py | 250 ++++++++++++++++++ tests/system-test/7-tmq/dataFromTsdbNWal.py | 51 ++-- tests/system-test/fulltest.sh | 3 +- 3 files changed, 276 insertions(+), 28 deletions(-) create mode 100644 tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py new file mode 100644 index 0000000000..2216000214 --- /dev/null +++ b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py @@ -0,0 +1,250 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 4 + self.ctbNum = 100 + self.rowsPerTbl = 1000 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("flush db to let data falls into the disk") + tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 500, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + # after start consume, continue insert some data + paraDict['batchNum'] = 100 + paraDict['startTs'] = paraDict['startTs'] + self.rowsPerTbl + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + pInsertThread.join() + + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + if expectRowsList[0] != resultList[0]: + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + tdSql.query("flush database %s"%(paraDict['dbName'])) + + for i in range(len(topicNameList)): + tmqCom.waitSubscriptionExit(tdSql,topicNameList[i]) + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 500, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + totalRowsInserted = expectRowsList[0] + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 1 + expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3) + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor 0") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + actConsumeRows = resultList[0] + + tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(actConsumeRows, expectrowcnt, totalRowsInserted)) + if not (expectrowcnt <= actConsumeRows and totalRowsInserted >= actConsumeRows): + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + consumerId = 2 + expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3) + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor 1") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + actConsumeRows = resultList[0] + tdLog.info("act consume rows: %d, expect rows: %d, act insert rows: %d"%(actConsumeRows, expectrowcnt, totalRowsInserted)) + if not ((actConsumeRows >= expectrowcnt) and (totalRowsInserted > actConsumeRows)): + tdLog.exit("%d tmq consume rows error!"%consumerId) + + for i in range(len(topicNameList)): + tmqCom.waitSubscriptionExit(tdSql,topicNameList[i]) + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + tdSql.prepare() + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal.py b/tests/system-test/7-tmq/dataFromTsdbNWal.py index 227ce9d5a5..faa70f4820 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal.py @@ -17,8 +17,8 @@ from tmqCommon import * class TDTestCase: def __init__(self): - self.vgroups = 1 - self.ctbNum = 100 + self.vgroups = 4 + self.ctbNum = 1 self.rowsPerTbl = 10000 def init(self, conn, logSql): @@ -38,9 +38,9 @@ class TDTestCase: 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, - 'ctbNum': 100, + 'ctbNum': 1, 'rowsPerTbl': 10000, - 'batchNum': 3000, + 'batchNum': 100, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 10, 'showMsg': 1, @@ -85,7 +85,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 100, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 3, + 'pollDelay': 5, 'showMsg': 1, 'showRow': 1, 'snapshot': 1} @@ -117,17 +117,16 @@ class TDTestCase: keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - # after start consume, continue insert some data paraDict['batchNum'] = 100 paraDict['startTs'] = paraDict['startTs'] + self.rowsPerTbl - tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], - ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], - startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - # + pInsertThread.join() + tdSql.query(queryString) expectRowsList.append(tdSql.getRows()) @@ -135,15 +134,16 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) if expectRowsList[0] != resultList[0]: tdLog.exit("%d tmq consume rows error!"%consumerId) tmqCom.checkFileContent(consumerId, queryString) - time.sleep(10) + tdSql.query("flush database %s"%(paraDict['dbName'])) + for i in range(len(topicNameList)): + tmqCom.waitSubscriptionExit(tdSql,topicNameList[i]) tdSql.query("drop topic %s"%topicNameList[i]) tdLog.printNoPrefix("======== test case 1 end ...... ") @@ -204,13 +204,12 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - - if not (expectrowcnt <= resultList[0] and totalRowsInserted >= resultList[0]): - tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(resultList[0], expectrowcnt, totalRowsInserted)) + actConsumeRows = resultList[0] + + tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(actConsumeRows, expectrowcnt, totalRowsInserted)) + if not (expectrowcnt <= actConsumeRows and totalRowsInserted >= actConsumeRows): tdLog.exit("%d tmq consume rows error!"%consumerId) - - firstConsumeRows = resultList[0] - + # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() consumerId = 2 @@ -224,15 +223,13 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - actConsumeTotalRows = firstConsumeRows + resultList[0] - - if not (expectrowcnt >= resultList[0] and totalRowsInserted == actConsumeTotalRows): - tdLog.info("act consume rows, first: %d, second: %d "%(firstConsumeRows, resultList[0])) - tdLog.info("and sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) + actConsumeRows = resultList[0] + tdLog.info("act consume rows: %d, expect rows: %d, act insert rows: %d"%(actConsumeRows, expectrowcnt, totalRowsInserted)) + if not ((actConsumeRows >= expectrowcnt) and (totalRowsInserted > actConsumeRows)): tdLog.exit("%d tmq consume rows error!"%consumerId) - time.sleep(10) for i in range(len(topicNameList)): + tmqCom.waitSubscriptionExit(tdSql,topicNameList[i]) tdSql.query("drop topic %s"%topicNameList[i]) tdLog.printNoPrefix("======== test case 2 end ...... ") @@ -241,7 +238,7 @@ class TDTestCase: tdSql.prepare() self.prepareTestEnv() self.tmqCase1() - # self.tmqCase2() + self.tmqCase2() def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 5cc7aca675..242628ce70 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -224,7 +224,8 @@ python3 ./test.py -f 7-tmq/tmqUdf.py python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py - +python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py +python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py # python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py #------------querPolicy 2----------- From 34e4f79557603abcbd2b8d33414a63a8736f2c58 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 23 Jul 2022 19:57:10 +0800 Subject: [PATCH 03/27] os: multi-level storage mount error --- source/common/src/tglobal.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a73808f2ed..36900e3dfa 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -213,10 +213,6 @@ static int32_t taosSetTfsCfg(SConfig *pCfg) { memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg)); if (pCfg->level == 0 && pCfg->primary == 1) { tstrncpy(tsDataDir, pCfg->dir, PATH_MAX); - if (taosMulMkDir(tsDataDir) != 0) { - uError("failed to create dataDir:%s since %s", tsDataDir, terrstr()); - return -1; - } } if (taosMulMkDir(pCfg->dir) != 0) { uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr()); @@ -227,12 +223,13 @@ static int32_t taosSetTfsCfg(SConfig *pCfg) { if (tsDataDir[0] == 0) { if (pItem->str != NULL) { - taosAddDataDir(0, pItem->str, 0, 1); + taosAddDataDir(tsDiskCfgNum, pItem->str, 0, 1); tstrncpy(tsDataDir, pItem->str, PATH_MAX); if (taosMulMkDir(tsDataDir) != 0) { - uError("failed to create dataDir:%s since %s", tsDataDir, terrstr()); + uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr()); return -1; } + tsDiskCfgNum++; } else { uError("datadir not set"); return -1; From dbd71d247c627520cefc3bf1d914ec90195eb0d6 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 23 Jul 2022 20:46:39 +0800 Subject: [PATCH 04/27] feat: super table order by primary key optimization --- source/libs/planner/src/planSpliter.c | 40 +++++++++++++++++++--- source/libs/planner/test/planBasicTest.cpp | 7 ++-- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 10604fac19..879c04743d 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -934,18 +934,48 @@ static SNode* stbSplFindPrimaryKeyFromScan(SScanLogicNode* pScan) { return pCol; } +static int32_t stbSplCreateMergeScanNode(SScanLogicNode* pScan, SLogicNode** pOutputMergeScan, + SNodeList** pOutputMergeKeys) { + SNodeList* pChildren = pScan->node.pChildren; + pScan->node.pChildren = NULL; + + int32_t code = TSDB_CODE_SUCCESS; + SScanLogicNode* pMergeScan = (SScanLogicNode*)nodesCloneNode((SNode*)pScan); + if (NULL == pMergeScan) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + SNodeList* pMergeKeys = NULL; + if (TSDB_CODE_SUCCESS == code) { + pMergeScan->scanType = SCAN_TYPE_TABLE_MERGE; + pMergeScan->node.pChildren = pChildren; + splSetParent((SLogicNode*)pMergeScan); + code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pMergeScan), &pMergeKeys); + } + + if (TSDB_CODE_SUCCESS == code) { + *pOutputMergeScan = (SLogicNode*)pMergeScan; + *pOutputMergeKeys = pMergeKeys; + } else { + nodesDestroyNode((SNode*)pMergeScan); + nodesDestroyList(pMergeKeys); + } + + return code; +} + static int32_t stbSplSplitMergeScanNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SScanLogicNode* pScan, bool groupSort) { - SNodeList* pMergeKeys = NULL; - int32_t code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pScan), &pMergeKeys); + SLogicNode* pMergeScan = NULL; + SNodeList* pMergeKeys = NULL; + int32_t code = stbSplCreateMergeScanNode(pScan, &pMergeScan, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan, groupSort); + code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, pMergeScan, groupSort); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pSubplan->pChildren, - (SNode*)splCreateScanSubplan(pCxt, (SLogicNode*)pScan, SPLIT_FLAG_STABLE_SPLIT)); + (SNode*)splCreateScanSubplan(pCxt, pMergeScan, SPLIT_FLAG_STABLE_SPLIT)); } - pScan->scanType = SCAN_TYPE_TABLE_MERGE; ++(pCxt->groupId); return code; } diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 8f9cd94c19..9cfae68d34 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -24,9 +24,10 @@ TEST_F(PlanBasicTest, selectClause) { useDb("root", "test"); run("SELECT * FROM t1"); - run("SELECT 1 FROM t1"); - run("SELECT * FROM st1"); - run("SELECT 1 FROM st1"); + + run("SELECT MAX(c1) c2, c2 FROM t1"); + + run("SELECT MAX(c1) c2, c2 FROM st1"); } TEST_F(PlanBasicTest, whereClause) { From dfe4ed92e3f8f013331c5faa7df34230094d45d9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 25 Jul 2022 09:01:40 +0800 Subject: [PATCH 05/27] fix: fix disk page direty issue --- source/libs/executor/inc/executil.h | 5 +++++ source/libs/executor/src/timewindowoperator.c | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 23732a6f9a..2f96482643 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -96,6 +96,11 @@ static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo return pRow; } +static FORCE_INLINE void setResultBufPageDirty(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { + void* pPage = getBufPage(pBuf, pos->pageId); + setBufPageDirty(pPage, true); +} + void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d1ecab8e4a..f6afebb3f7 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -905,6 +905,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { saveResultRow(pResult, tableGroupId, pUpdated); + setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); } } @@ -961,6 +962,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { saveResultRow(pResult, tableGroupId, pUpdated); + setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); } ekey = ascScan ? nextWin.ekey : nextWin.skey; @@ -2507,6 +2509,7 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr } if (find && pUpdated) { saveResultRow(pCurResult, pWinRes->groupId, pUpdated); + setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur); } } } @@ -2627,6 +2630,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) { saveResultRow(pResult, tableGroupId, pUpdated); + setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, From f64f0323b5421f2f46e2c173f54a9f649c67a224 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 25 Jul 2022 11:14:47 +0800 Subject: [PATCH 06/27] test: add test case for tmq --- tests/system-test/7-tmq/tmqDnodeRestart.py | 37 +--------------------- tests/system-test/fulltest.sh | 2 +- 2 files changed, 2 insertions(+), 37 deletions(-) diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index cec6985a4e..5117ee3d24 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -151,41 +151,6 @@ class TDTestCase: if not (totalConsumeRows == totalRowsFromQury): tdLog.exit("tmq consume rows error!") - - - - # tdLog.info("****************************************************************************") - # tmqCom.initConsumerTable() - # consumerId = 1 - # expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 - # topicList = topicFromStb1 - # ifcheckdata = 0 - # ifManualCommit = 0 - # keyList = 'group.id:cgrp2,\ - # enable.auto.commit:true,\ - # auto.commit.interval.ms:3000,\ - # auto.offset.reset:earliest' - # tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - # tdLog.info("start consume processor") - # tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - - # expectRows = 1 - # resultList = tmqCom.selectConsumeResult(expectRows) - # totalConsumeRows = 0 - # for i in range(expectRows): - # totalConsumeRows += resultList[i] - - # tdSql.query(queryString) - # totalRowsFromQury = tdSql.getRows() - - # tdLog.info("act consume rows: %d, act query rows: %d"%(totalConsumeRows, totalRowsFromQury)) - # if not (totalConsumeRows == totalRowsFromQury): - # tdLog.exit("tmq consume rows error!") - - - # tdLog.info("****************************************************************************") - tmqCom.waitSubscriptionExit(tdSql, topicFromStb1) tdSql.query("drop topic %s"%topicFromStb1) @@ -259,7 +224,7 @@ class TDTestCase: tdLog.info("create some new child table and insert data ") paraDict["batchNum"] = 100 paraDict["ctbPrefix"] = 'newCtb' - # tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) + tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) tdLog.info("insert process end, and start to check consume result") expectRows = 1 diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 242628ce70..ba104ddf22 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -210,7 +210,7 @@ python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py -#python3 ./test.py -f 7-tmq/tmqDnodeRestart.py +python3 ./test.py -f 7-tmq/tmqDnodeRestart.py python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py From 9bddf99d7d3c6184bd29f999b594bb5d97925b19 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Jul 2022 11:17:45 +0800 Subject: [PATCH 07/27] refactor(query): do some internal refactor. --- source/dnode/mnode/impl/src/mndProfile.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 763 ++++++------------ source/libs/executor/inc/executil.h | 2 - source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executil.c | 16 +- source/libs/executor/src/executorimpl.c | 62 +- source/libs/executor/src/timewindowoperator.c | 11 +- 7 files changed, 266 insertions(+), 592 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 5871d56a8f..f6ecd4493d 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -312,7 +312,7 @@ static int32_t mndSaveQueryList(SConnObj *pConn, SQueryHbReqBasic *pBasic) { pConn->numOfQueries = pBasic->queryDesc ? taosArrayGetSize(pBasic->queryDesc) : 0; pBasic->queryDesc = NULL; - mDebug("queries updated in conn %d, num:%d", pConn->id, pConn->numOfQueries); + mDebug("queries updated in conn %u, num:%d", pConn->id, pConn->numOfQueries); taosWUnLockLatch(&pConn->queryLock); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 26ced6cf6a..d6a93e1909 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -16,6 +16,12 @@ #include "tsdb.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +typedef enum { + EXTERNAL_ROWS_PREV = 0x1, + EXTERNAL_ROWS_MAIN = 0x2, + EXTERNAL_ROWS_NEXT = 0x3, +} EContentData; + typedef struct { STbDataIter* iter; int32_t index; @@ -70,9 +76,9 @@ typedef struct SFilesetIter { } SFilesetIter; typedef struct SFileDataBlockInfo { - int32_t - tbBlockIdx; // index position in STableBlockScanInfo in order to check whether neighbor block overlaps with it + // index position in STableBlockScanInfo in order to check whether neighbor block overlaps with it uint64_t uid; + int32_t tbBlockIdx; } SFileDataBlockInfo; typedef struct SDataBlockIter { @@ -99,12 +105,11 @@ typedef struct SReaderStatus { SHashObj* pTableMap; // SHash STableBlockScanInfo* pTableIter; // table iterator used in building in-memory buffer data blocks. SFileBlockDumpInfo fBlockDumpInfo; - - SDFileSet* pCurrentFileset; // current opened file set - SBlockData fileBlockData; - SFilesetIter fileIter; - SDataBlockIter blockIter; - bool composedDataBlock; // the returned data block is a composed block or not + SDFileSet* pCurrentFileset; // current opened file set + SBlockData fileBlockData; + SFilesetIter fileIter; + SDataBlockIter blockIter; + bool composedDataBlock; // the returned data block is a composed block or not } SReaderStatus; struct STsdbReader { @@ -115,15 +120,17 @@ struct STsdbReader { SSDataBlock* pResBlock; int32_t capacity; SReaderStatus status; - char* idStr; // query info handle, for debug purpose - int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows + char* idStr; // query info handle, for debug purpose + int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; + SIOCostSummary cost; + STSchema* pSchema; + SDataFReader* pFileReader; + SVersionRange verRange; - SIOCostSummary cost; - STSchema* pSchema; - SDataFReader* pFileReader; - SVersionRange verRange; + int32_t step; + STsdbReader* innerReader[2]; }; static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); @@ -200,6 +207,9 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK pTsdbReader->idStr); } + tsdbDebug("%p create %d tables scan-info, size:%.2f Kb, %s", pTsdbReader, numOfTables, (sizeof(STableBlockScanInfo)*numOfTables)/1024.0, + pTsdbReader->idStr); + return pTableMap; } @@ -328,7 +338,7 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { continue; } - tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pReader, fid, pReader->window.skey, + tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->window.skey, pReader->window.ekey, pReader->idStr); return true; } @@ -378,7 +388,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) return pResBlock; } -static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, const char* idstr) { +static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, int32_t capacity, const char* idstr) { int32_t code = 0; int8_t level = 0; STsdbReader* pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader)); @@ -392,7 +402,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level); pReader->suid = pCond->suid; pReader->order = pCond->order; - pReader->capacity = 4096; + pReader->capacity = capacity; pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; @@ -483,95 +493,6 @@ _end: // return res; // } -// static TSKEY extractFirstTraverseKey(STableBlockScanInfo* pCheckInfo, int32_t order, int32_t update, TDRowVerT -// maxVer) { -// TSDBROW row = {0}; -// STSRow *rmem = NULL, *rimem = NULL; - -// if (pCheckInfo->iter) { -// if (tsdbTbDataIterGet(pCheckInfo->iter, &row)) { -// rmem = row.pTSRow; -// } -// } - -// if (pCheckInfo->iiter) { -// if (tsdbTbDataIterGet(pCheckInfo->iiter, &row)) { -// rimem = row.pTSRow; -// } -// } - -// if (rmem == NULL && rimem == NULL) { -// return TSKEY_INITIAL_VAL; -// } - -// if (rmem != NULL && rimem == NULL) { -// pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM; -// return TD_ROW_KEY(rmem); -// } - -// if (rmem == NULL && rimem != NULL) { -// pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; -// return TD_ROW_KEY(rimem); -// } - -// TSKEY r1 = TD_ROW_KEY(rmem); -// TSKEY r2 = TD_ROW_KEY(rimem); - -// if (r1 == r2) { -// if (TD_SUPPORT_UPDATE(update)) { -// pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH; -// } else { -// pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; -// tsdbTbDataIterNext(pCheckInfo->iter); -// } -// return r1; -// } else if (r1 < r2 && ASCENDING_TRAVERSE(order)) { -// pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM; -// return r1; -// } else { -// pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM; -// return r2; -// } -// } - -// static bool moveToNextRowInMem(STableBlockScanInfo* pCheckInfo) { -// bool hasNext = false; -// if (pCheckInfo->chosen == CHECKINFO_CHOSEN_MEM) { -// if (pCheckInfo->iter != NULL) { -// hasNext = tsdbTbDataIterNext(pCheckInfo->iter); -// } - -// if (hasNext) { -// return hasNext; -// } - -// if (pCheckInfo->iiter != NULL) { -// return tsdbTbDataIterGet(pCheckInfo->iiter, NULL); -// } -// } else if (pCheckInfo->chosen == CHECKINFO_CHOSEN_IMEM) { -// if (pCheckInfo->iiter != NULL) { -// hasNext = tsdbTbDataIterNext(pCheckInfo->iiter); -// } - -// if (hasNext) { -// return hasNext; -// } - -// if (pCheckInfo->iter != NULL) { -// return tsdbTbDataIterGet(pCheckInfo->iter, NULL); -// } -// } else { -// if (pCheckInfo->iter != NULL) { -// hasNext = tsdbTbDataIterNext(pCheckInfo->iter); -// } -// if (pCheckInfo->iiter != NULL) { -// hasNext = tsdbTbDataIterNext(pCheckInfo->iiter) || hasNext; -// } -// } - -// return hasNext; -// } - // static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { // int32_t firstSlot = 0; // int32_t lastSlot = numOfBlocks - 1; @@ -602,18 +523,22 @@ _end: static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, SArray* pIndexList) { SArray* aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + int64_t st = taosGetTimestampUs(); int32_t code = tsdbReadBlockIdx(pFileReader, aBlockIdx, NULL); if (code != TSDB_CODE_SUCCESS) { goto _end; } - if (taosArrayGetSize(aBlockIdx) == 0) { + size_t num = taosArrayGetSize(aBlockIdx); + if (num == 0) { taosArrayClear(aBlockIdx); return TSDB_CODE_SUCCESS; } - SBlockIdx* pBlockIdx; - for (int32_t i = 0; i < taosArrayGetSize(aBlockIdx); ++i) { + int64_t et1 = taosGetTimestampUs(); + + SBlockIdx* pBlockIdx = NULL; + for (int32_t i = 0; i < num; ++i) { pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i); // uid check @@ -627,17 +552,6 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, continue; } - // todo: not valid info in bockIndex - // time range check - // if (pBlockIdx->minKey > pReader->window.ekey || pBlockIdx->maxKey < pReader->window.skey) { - // continue; - // } - - // version check - // if (pBlockIdx->minVersion > pReader->verRange.maxVer || pBlockIdx->maxVersion < pReader->verRange.minVer) { - // continue; - // } - STableBlockScanInfo* pScanInfo = p; if (pScanInfo->pBlockList == NULL) { pScanInfo->pBlockList = taosArrayInit(16, sizeof(SBlock)); @@ -647,6 +561,9 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, taosArrayPush(pIndexList, pBlockIdx); } + int64_t et2 = taosGetTimestampUs(); + tsdbDebug("load block index for %d tables completed, elapsed time:%.2f ms, set blockIdx:%.2f ms, size:%d bytes %s", + (int32_t)num, (et1 - st)/1000.0, (et2-et1)/1000.0, num * sizeof(SBlockIdx), pReader->idStr); _end: taosArrayDestroy(aBlockIdx); return code; @@ -655,9 +572,11 @@ _end: static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_t* numOfValidTables, int32_t* numOfBlocks) { size_t numOfTables = taosArrayGetSize(pIndexList); - *numOfValidTables = 0; + int64_t st = taosGetTimestampUs(); + size_t size = 0; + STableBlockScanInfo* px = NULL; while (1) { px = taosHashIterate(pReader->status.pTableMap, px); @@ -675,6 +594,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_ tMapDataReset(&mapData); tsdbReadBlock(pReader->pFileReader, pBlockIdx, &mapData, NULL); + size += mapData.nData; + STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(int64_t)); for (int32_t j = 0; j < mapData.nItem; ++j) { SBlock block = {0}; @@ -706,6 +627,10 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, uint32_ } } + int64_t et = taosGetTimestampUs(); + tsdbDebug("load block of %d tables completed, blocks:%d in %d tables, size:%.2f Kb, elapsed time:%.2f ms %s", + numOfTables, *numOfBlocks, *numOfValidTables, size/1000.0, (et-st)/1000.0, pReader->idStr); + return TSDB_CODE_SUCCESS; } @@ -816,7 +741,6 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn return TSDB_CODE_SUCCESS; } -// todo consider the output buffer size static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { int64_t st = taosGetTimestampUs(); @@ -853,346 +777,6 @@ _error: return code; } -// static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { -// int firstPos, lastPos, midPos = -1; -// int numOfRows; -// TSKEY* keyList; - -// assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - -// if (num <= 0) return -1; - -// keyList = (TSKEY*)pValue; -// firstPos = 0; -// lastPos = num - 1; - -// if (order == TSDB_ORDER_DESC) { -// // find the first position which is smaller than the key -// while (1) { -// if (key >= keyList[lastPos]) return lastPos; -// if (key == keyList[firstPos]) return firstPos; -// if (key < keyList[firstPos]) return firstPos - 1; - -// numOfRows = lastPos - firstPos + 1; -// midPos = (numOfRows >> 1) + firstPos; - -// if (key < keyList[midPos]) { -// lastPos = midPos - 1; -// } else if (key > keyList[midPos]) { -// firstPos = midPos + 1; -// } else { -// break; -// } -// } - -// } else { -// // find the first position which is bigger than the key -// while (1) { -// if (key <= keyList[firstPos]) return firstPos; -// if (key == keyList[lastPos]) return lastPos; - -// if (key > keyList[lastPos]) { -// lastPos = lastPos + 1; -// if (lastPos >= num) -// return -1; -// else -// return lastPos; -// } - -// numOfRows = lastPos - firstPos + 1; -// midPos = (numOfRows >> 1) + firstPos; - -// if (key < keyList[midPos]) { -// lastPos = midPos - 1; -// } else if (key > keyList[midPos]) { -// firstPos = midPos + 1; -// } else { -// break; -// } -// } -// } - -// return midPos; -// } - -// static void doCheckGeneratedBlockRange(STsdbReader* pTsdbReadHandle) { -// SQueryFilePos* cur = &pTsdbReadHandle->cur; - -// if (cur->rows > 0) { -// if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { -// assert(cur->win.skey >= pTsdbReadHandle->window.skey && cur->win.ekey <= pTsdbReadHandle->window.ekey); -// } else { -// assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey); -// } - -// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0); -// assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && -// cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows - 1]); -// } else { -// cur->win = pTsdbReadHandle->window; - -// int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; -// cur->lastKey = pTsdbReadHandle->window.ekey + step; -// } -// } - -// static void copyAllRemainRowsFromFileBlock(STsdbReader* pTsdbReadHandle, STableBlockScanInfo* pCheckInfo, -// SDataBlockInfo* pBlockInfo, int32_t endPos) { -// SQueryFilePos* cur = &pTsdbReadHandle->cur; - -// SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; -// TSKEY* tsArray = pCols->cols[0].pData; - -// bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order); - -// int32_t step = ascScan ? 1 : -1; - -// int32_t start = cur->pos; -// int32_t end = endPos; - -// if (!ascScan) { -// TSWAP(start, end); -// } - -// assert(pTsdbReadHandle->outputCapacity >= (end - start + 1)); -// int32_t numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, start, end); - -// // the time window should always be ascending order: skey <= ekey -// cur->win = (STimeWindow){.skey = tsArray[start], .ekey = tsArray[end]}; -// cur->mixBlock = (numOfRows != pBlockInfo->rows); -// cur->lastKey = tsArray[endPos] + step; -// cur->blockCompleted = (ascScan ? (endPos == pBlockInfo->rows - 1) : (endPos == 0)); - -// // The value of pos may be -1 or pBlockInfo->rows, and it is invalid in both cases. -// int32_t pos = endPos + step; -// updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); -// doCheckGeneratedBlockRange(pTsdbReadHandle); - -// tsdbDebug("%p uid:%" PRIu64 ", data block created, mixblock:%d, brange:%" PRIu64 "-%" PRIu64 " rows:%d, %s", -// pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, -// pTsdbReadHandle->idStr); -// } - -// // only return the qualified data to client in terms of query time window, data rows in the same block but do not -// // be included in the query time window will be discarded -// static void doMergeTwoLevelData(STsdbReader* pTsdbReadHandle, STableBlockScanInfo* pCheckInfo, SBlock* pBlock) { -// SQueryFilePos* cur = &pTsdbReadHandle->cur; -// SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); -// STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); - -// initTableMemIterator(pTsdbReadHandle, pCheckInfo); - -// SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; -// assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID && -// cur->pos >= 0 && cur->pos < pBlock->numOfRows); -// // Even Multi-Version supported, the records with duplicated TSKEY would be merged inside of tsdbLoadData -// interface. TSKEY* tsArray = pCols->cols[0].pData; assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == -// pBlock->minKey.ts && -// tsArray[pBlock->numOfRows - 1] == pBlock->maxKey.ts); - -// bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order); -// int32_t step = ascScan ? 1 : -1; - -// // for search the endPos, so the order needs to reverse -// int32_t order = ascScan ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; - -// int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); -// int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo); - -// STimeWindow* pWin = &blockInfo.window; -// tsdbDebug("%p uid:%" PRIu64 " start merge data block, file block range:%" PRIu64 "-%" PRIu64 -// " rows:%d, start:%d, end:%d, %s", -// pTsdbReadHandle, pCheckInfo->tableId, pWin->skey, pWin->ekey, blockInfo.rows, cur->pos, endPos, -// pTsdbReadHandle->idStr); - -// // compared with the data from in-memory buffer, to generate the correct timestamp array list -// int32_t numOfRows = 0; -// int32_t curRow = 0; - -// int16_t rv1 = -1; -// int16_t rv2 = -1; -// STSchema* pSchema1 = NULL; -// STSchema* pSchema2 = NULL; - -// int32_t pos = cur->pos; -// cur->win = TSWINDOW_INITIALIZER; -// bool adjustPos = false; - -// // no data in buffer, load data from file directly -// if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) { -// copyAllRemainRowsFromFileBlock(pTsdbReadHandle, pCheckInfo, &blockInfo, endPos); -// return; -// } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { -// SSkipListNode* node = NULL; -// TSKEY lastKeyAppend = TSKEY_INITIAL_VAL; - -// do { -// STSRow* row2 = NULL; -// STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2, TD_VER_MAX); -// if (row1 == NULL) { -// break; -// } - -// TSKEY key = TD_ROW_KEY(row1); -// if ((key > pTsdbReadHandle->window.ekey && ascScan) || (key < pTsdbReadHandle->window.ekey && !ascScan)) { -// break; -// } - -// if (adjustPos) { -// if (key == lastKeyAppend) { -// pos -= step; -// } -// adjustPos = false; -// } - -// if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ascScan) || -// ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ascScan)) { -// break; -// } - -// if ((key < tsArray[pos] && ascScan) || (key > tsArray[pos] && !ascScan)) { -// if (rv1 != TD_ROW_SVER(row1)) { -// // pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); -// rv1 = TD_ROW_SVER(row1); -// } -// if (row2 && rv2 != TD_ROW_SVER(row2)) { -// // pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); -// rv2 = TD_ROW_SVER(row2); -// } - -// numOfRows += -// mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, -// pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend); -// if (cur->win.skey == TSKEY_INITIAL_VAL) { -// cur->win.skey = key; -// } - -// cur->win.ekey = key; -// cur->lastKey = key + step; -// cur->mixBlock = true; -// moveToNextRowInMem(pCheckInfo); -// } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it -// if (TD_SUPPORT_UPDATE(pCfg->update)) { -// if (lastKeyAppend != key) { -// if (lastKeyAppend != TSKEY_INITIAL_VAL) { -// ++curRow; -// } -// lastKeyAppend = key; -// } -// // load data from file firstly -// numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos); - -// if (rv1 != TD_ROW_SVER(row1)) { -// rv1 = TD_ROW_SVER(row1); -// } -// if (row2 && rv2 != TD_ROW_SVER(row2)) { -// rv2 = TD_ROW_SVER(row2); -// } - -// // still assign data into current row -// numOfRows += -// mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, -// pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend); - -// if (cur->win.skey == TSKEY_INITIAL_VAL) { -// cur->win.skey = key; -// } - -// cur->win.ekey = key; -// cur->lastKey = key + step; -// cur->mixBlock = true; - -// moveToNextRowInMem(pCheckInfo); - -// pos += step; -// adjustPos = true; -// } else { -// // discard the memory record -// moveToNextRowInMem(pCheckInfo); -// } -// } else if ((key > tsArray[pos] && ascScan) || (key < tsArray[pos] && !ascScan)) { -// if (cur->win.skey == TSKEY_INITIAL_VAL) { -// cur->win.skey = tsArray[pos]; -// } - -// int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order); -// assert(end != -1); - -// if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it -// #if 0 -// if (pCfg->update == TD_ROW_DISCARD_UPDATE) { -// moveToNextRowInMem(pCheckInfo); -// } else { -// end -= step; -// } -// #endif -// if (!TD_SUPPORT_UPDATE(pCfg->update)) { -// moveToNextRowInMem(pCheckInfo); -// } else { -// end -= step; -// } -// } - -// int32_t qstart = 0, qend = 0; -// getQualifiedRowsPos(pTsdbReadHandle, pos, end, numOfRows, &qstart, &qend); - -// if ((lastKeyAppend != TSKEY_INITIAL_VAL) && (lastKeyAppend != (ascScan ? tsArray[qstart] : tsArray[qend]))) { -// ++curRow; -// } - -// numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, qstart, qend); -// pos += (qend - qstart + 1) * step; -// if (numOfRows > 0) { -// curRow = numOfRows - 1; -// } - -// cur->win.ekey = ascScan ? tsArray[qend] : tsArray[qstart]; -// cur->lastKey = cur->win.ekey + step; -// lastKeyAppend = cur->win.ekey; -// } -// } while (numOfRows < pTsdbReadHandle->outputCapacity); - -// if (numOfRows < pTsdbReadHandle->outputCapacity) { -// /** -// * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT -// * copy them all to result buffer, since it may be overlapped with file data block. -// */ -// if (node == NULL || ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && ascScan) -// || -// ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && !ascScan)) { -// // no data in cache or data in cache is greater than the ekey of time window, load data from file block -// if (cur->win.skey == TSKEY_INITIAL_VAL) { -// cur->win.skey = tsArray[pos]; -// } - -// int32_t start = -1, end = -1; -// getQualifiedRowsPos(pTsdbReadHandle, pos, endPos, numOfRows, &start, &end); - -// numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, start, end); -// pos += (end - start + 1) * step; - -// cur->win.ekey = ascScan ? tsArray[end] : tsArray[start]; -// cur->lastKey = cur->win.ekey + step; -// cur->mixBlock = true; -// } -// } -// } - -// cur->blockCompleted = (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ascScan) || -// ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ascScan)); - -// if (!ascScan) { -// TSWAP(cur->win.skey, cur->win.ekey); -// } - -// updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); -// doCheckGeneratedBlockRange(pTsdbReadHandle); - -// tsdbDebug("%p uid:%" PRIu64 ", data block created, mixblock:%d, brange:%" PRIu64 "-%" PRIu64 " rows:%d, %s", -// pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, -// pTsdbReadHandle->idStr); -// } - static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) { taosMemoryFreeClear(pSup->numOfBlocksPerTable); taosMemoryFreeClear(pSup->indexPerTable); @@ -1252,8 +836,9 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte // access data blocks according to the offset of each block in asc/desc order. int32_t numOfTables = (int32_t)taosHashGetSize(pReader->status.pTableMap); - SBlockOrderSupporter sup = {0}; + int64_t st = taosGetTimestampUs(); + SBlockOrderSupporter sup = {0}; int32_t code = initBlockOrderSupporter(&sup, numOfTables); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1302,11 +887,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i}; taosArrayPush(pBlockIter->blockList, &blockInfo); } - tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted %s", pReader, cnt, - pReader->idStr); + + int64_t et = taosGetTimestampUs(); + tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s", pReader, cnt, + (et - st)/1000.0, pReader->idStr); pBlockIter->index = asc ? 0 : (numOfBlocks - 1); - cleanupBlockOrderSupporter(&sup); return TSDB_CODE_SUCCESS; } @@ -1340,7 +926,8 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); } - tsdbDebug("%p %d data blocks sort completed, %s", pReader, cnt, pReader->idStr); + int64_t et = taosGetTimestampUs(); + tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, cnt, (et-st)/1000.0, pReader->idStr); cleanupBlockOrderSupporter(&sup); taosMemoryFree(pTree); @@ -1813,6 +1400,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader, STableBlockScanInfo* SBlockData* pBlockData = &pReader->status.fileBlockData; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; + int64_t st = taosGetTimestampUs(); + while (1) { // todo check the validate of row in file block { @@ -1851,10 +1440,11 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader, STableBlockScanInfo* blockDataUpdateTsWindow(pResBlock, 0); setComposedBlockFlag(pReader, true); + int64_t et = taosGetTimestampUs(); - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%d, %s", pReader, + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%d, elapsed time:%.2f ms %s", pReader, pBlockScanInfo->uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, - pReader->idStr); + (et - st)/1000.0, pReader->idStr); return TSDB_CODE_SUCCESS; } @@ -2031,7 +1621,9 @@ static TSDBKEY getCurrentKeyInBuf(SDataBlockIter* pBlockIter, STsdbReader* pRead static int32_t moveToNextFile(STsdbReader* pReader, int32_t* numOfBlocks) { SReaderStatus* pStatus = &pReader->status; - SArray* pIndexList = taosArrayInit(4, sizeof(SBlockIdx)); + + size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); while (1) { bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader); @@ -2803,24 +2395,57 @@ int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list) { // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTableList, STsdbReader** ppReader, const char* idstr) { - int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, idstr); - if (code) { + int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, 4096, idstr); + if (code != TSDB_CODE_SUCCESS) { goto _err; } - if (pCond->suid != 0) { - (*ppReader)->pSchema = metaGetTbTSchema((*ppReader)->pTsdb->pVnode->pMeta, (*ppReader)->suid, -1); - } else if (taosArrayGetSize(pTableList) > 0) { - STableKeyInfo* pKey = taosArrayGet(pTableList, 0); - (*ppReader)->pSchema = metaGetTbTSchema((*ppReader)->pTsdb->pVnode->pMeta, pKey->uid, -1); - } - + // check for query time window STsdbReader* pReader = *ppReader; if (isEmptyQueryTimeWindow(&pReader->window)) { tsdbDebug("%p query window not overlaps with the data set, no result returned, %s", pReader, pReader->idStr); return TSDB_CODE_SUCCESS; } + if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { + // update the SQueryTableDataCond to create inner reader + STimeWindow w = pCond->twindows; + int32_t order = pCond->order; + if (order == TSDB_ORDER_ASC) { + pCond->twindows.ekey = pCond->twindows.skey; + pCond->twindows.skey = INT64_MIN; + pCond->order = TSDB_ORDER_DESC; + } else { + pCond->twindows.skey = pCond->twindows.ekey; + pCond->twindows.ekey = INT64_MAX; + pCond->order = TSDB_ORDER_ASC; + } + + code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, idstr); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + if (order == TSDB_ORDER_ASC) { + pCond->twindows.skey = w.ekey; + pCond->twindows.ekey = INT64_MAX; + } else { + pCond->twindows.skey = INT64_MIN; + pCond->twindows.ekey = w.ekey; + } + code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, idstr); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + } + + if (pCond->suid != 0) { + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1); + } else if (taosArrayGetSize(pTableList) > 0) { + STableKeyInfo* pKey = taosArrayGet(pTableList, 0); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1); + } + int32_t numOfTables = taosArrayGetSize(pTableList); pReader->status.pTableMap = createDataBlockScanInfo(pReader, pTableList->pData, numOfTables); if (pReader->status.pTableMap == NULL) { @@ -2831,21 +2456,41 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl goto _err; } - SDataBlockIter* pBlockIter = &pReader->status.blockIter; - code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap); - if (code) goto _err; + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } - initFilesetIterator(&pReader->status.fileIter, (*ppReader)->pReadSnap->fs.aDFileSet, pReader->order, pReader->idStr); - resetDataBlockIterator(&pReader->status.blockIter, pReader->order); + if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { + SDataBlockIter* pBlockIter = &pReader->status.blockIter; - // no data in files, let's try buffer in memory - if (pReader->status.fileIter.numOfFiles == 0) { - pReader->status.loadFromFile = false; + initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader->order, pReader->idStr); + resetDataBlockIterator(&pReader->status.blockIter, pReader->order); + + // no data in files, let's try buffer in memory + if (pReader->status.fileIter.numOfFiles == 0) { + pReader->status.loadFromFile = false; + } else { + code = initForFirstBlockInFile(pReader, pBlockIter); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } } else { - code = initForFirstBlockInFile(pReader, pBlockIter); - if (code != TSDB_CODE_SUCCESS) { - return code; + STsdbReader* pPrevReader = pReader->innerReader[0]; + SDataBlockIter* pBlockIter = &pPrevReader->status.blockIter; + + initFilesetIterator(&pPrevReader->status.fileIter, pPrevReader->pReadSnap->fs.aDFileSet, pPrevReader->order, pPrevReader->idStr); + resetDataBlockIterator(&pPrevReader->status.blockIter, pPrevReader->order); + + // no data in files, let's try buffer in memory + if (pPrevReader->status.fileIter.numOfFiles == 0) { + pPrevReader->status.loadFromFile = false; + } else { + code = initForFirstBlockInFile(pPrevReader, pBlockIter); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } @@ -2885,20 +2530,6 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbDataFReaderClose(&pReader->pFileReader); } -#if 0 -// if (pReader->status.pTableScanInfo != NULL) { -// pReader->status.pTableScanInfo = destroyTableCheckInfo(pReader->status.pTableScanInfo); -// } - -// tsdbDestroyReadH(&pReader->rhelper); - -// tdFreeDataCols(pReader->pDataCols); -// pReader->pDataCols = NULL; -// -// pReader->prev = doFreeColumnInfoData(pReader->prev); -// pReader->next = doFreeColumnInfoData(pReader->next); -#endif - SIOCostSummary* pCost = &pReader->cost; tsdbDebug("%p :io-cost summary: head-file read cnt:%" PRIu64 ", head-file time:%" PRIu64 " us, statis-info:%" PRId64 @@ -2911,55 +2542,100 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFreeClear(pReader); } -bool tsdbNextDataBlock(STsdbReader* pReader) { - if (isEmptyQueryTimeWindow(&pReader->window)) { - return false; - } - +static bool doTsdbNextDataBlock(STsdbReader* pReader) { // cleanup the data that belongs to the previous data block SSDataBlock* pBlock = pReader->pResBlock; blockDataCleanup(pBlock); int64_t stime = taosGetTimestampUs(); - int64_t elapsedTime = stime; SReaderStatus* pStatus = &pReader->status; - if (pReader->type == BLOCK_LOAD_OFFSET_ORDER) { - if (pStatus->loadFromFile) { - int32_t code = buildBlockFromFiles(pReader); - if (code != TSDB_CODE_SUCCESS) { - return false; - } + if (pStatus->loadFromFile) { + int32_t code = buildBlockFromFiles(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } - if (pBlock->info.rows > 0) { - return true; - } else { - buildBlockFromBufferSequentially(pReader); - return pBlock->info.rows > 0; - } - } else { // no data in files, let's try the buffer + if (pBlock->info.rows > 0) { + return true; + } else { buildBlockFromBufferSequentially(pReader); return pBlock->info.rows > 0; } - } else if (pReader->type == BLOCK_LOAD_TABLESEQ_ORDER) { - } else if (pReader->type == BLOCK_LOAD_EXTERN_ORDER) { - } else { - ASSERT(0); + } else { // no data in files, let's try the buffer + buildBlockFromBufferSequentially(pReader); + return pBlock->info.rows > 0; } + return false; } -void tsdbRetrieveDataBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockInfo) { +bool tsdbNextDataBlock(STsdbReader* pReader) { + if (isEmptyQueryTimeWindow(&pReader->window)) { + return false; + } + + if (pReader->innerReader[0] != NULL) { + bool ret = doTsdbNextDataBlock(pReader->innerReader[0]); + if (ret) { + pReader->step = EXTERNAL_ROWS_PREV; + return ret; + } + + tsdbReaderClose(pReader->innerReader[0]); + pReader->innerReader[0] = NULL; + } + + pReader->step = EXTERNAL_ROWS_MAIN; + bool ret = doTsdbNextDataBlock(pReader); + if (ret) { + return ret; + } + + if (pReader->innerReader[1] != NULL) { + bool ret1 = doTsdbNextDataBlock(pReader->innerReader[1]); + if (ret1) { + pReader->step = EXTERNAL_ROWS_NEXT; + return ret1; + } + + tsdbReaderClose(pReader->innerReader[1]); + pReader->innerReader[1] = NULL; + } + + return false; +} + +static void setBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockInfo) { ASSERT(pDataBlockInfo != NULL && pReader != NULL); pDataBlockInfo->rows = pReader->pResBlock->info.rows; pDataBlockInfo->uid = pReader->pResBlock->info.uid; pDataBlockInfo->window = pReader->pResBlock->info.window; } +void tsdbRetrieveDataBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockInfo) { + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + if (pReader->step == EXTERNAL_ROWS_MAIN) { + setBlockInfo(pReader, pDataBlockInfo); + } else if (pReader->step == EXTERNAL_ROWS_PREV) { + setBlockInfo(pReader->innerReader[0], pDataBlockInfo); + } else { + setBlockInfo(pReader->innerReader[1], pDataBlockInfo); + } + } else { + setBlockInfo(pReader, pDataBlockInfo); + } +} + int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) { int32_t code = 0; *allHave = false; + if(pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + *pBlockStatis = NULL; + return TSDB_CODE_SUCCESS; + } + // there is no statistics data for composed block if (pReader->status.composedDataBlock) { *pBlockStatis = NULL; @@ -3029,7 +2705,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS return code; } -SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { +static SArray* doRetrieveDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; if (pStatus->composedDataBlock) { @@ -3058,16 +2734,27 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { return pReader->pResBlock->pDataBlock; } +SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { + if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { + if (pReader->step == EXTERNAL_ROWS_PREV) { + return doRetrieveDataBlock(pReader->innerReader[0]); + } else if (pReader->step == EXTERNAL_ROWS_NEXT) { + return doRetrieveDataBlock(pReader->innerReader[1]); + } + } + + return doRetrieveDataBlock(pReader); +} + int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { if (isEmptyQueryTimeWindow(&pReader->window)) { return TSDB_CODE_SUCCESS; } pReader->order = pCond->order; - pReader->type = BLOCK_LOAD_OFFSET_ORDER; + pReader->type = TIMEWINDOW_RANGE_CONTAINED; pReader->status.loadFromFile = true; pReader->status.pTableIter = NULL; - pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); // allocate buffer in order to load data blocks from file @@ -3077,10 +2764,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; tsdbDataFReaderClose(&pReader->pFileReader); - // todo set the correct numOfTables - int32_t numOfTables = 1; - SDataBlockIter* pBlockIter = &pReader->status.blockIter; - + int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); tsdbDataFReaderClose(&pReader->pFileReader); initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader->order, pReader->idStr); @@ -3088,18 +2772,23 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { resetDataBlockScanInfo(pReader->status.pTableMap); int32_t code = 0; + SDataBlockIter* pBlockIter = &pReader->status.blockIter; + // no data in files, let's try buffer in memory if (pReader->status.fileIter.numOfFiles == 0) { pReader->status.loadFromFile = false; } else { code = initForFirstBlockInFile(pReader, pBlockIter); if (code != TSDB_CODE_SUCCESS) { + tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", + pReader, numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr); return code; } } tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader, pReader->suid, numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr); + return code; } @@ -3190,7 +2879,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { STbData* d = NULL; if (pReader->pTsdb->mem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pTsdb->mem, pReader->suid, pBlockScanInfo->uid, &d); + tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid, &d); if (d != NULL) { rows += tsdbGetNRowsInTbData(d); } @@ -3198,7 +2887,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { STbData* di = NULL; if (pReader->pTsdb->imem != NULL) { - tsdbGetTbDataFromMemTable(pReader->pTsdb->imem, pReader->suid, pBlockScanInfo->uid, &di); + tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid, &di); if (di != NULL) { rows += tsdbGetNRowsInTbData(di); } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 23732a6f9a..efe2ad6e83 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -82,8 +82,6 @@ size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); void initResultRowInfo(SResultRowInfo* pResultRowInfo); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -void closeAllResultRows(SResultRowInfo* pResultRowInfo); - void initResultRow(SResultRow* pResultRow); void closeResultRow(SResultRow* pResultRow); bool isResultRowClosed(SResultRow* pResultRow); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1ad17bbc76..f4d0eb3b5e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -108,7 +108,6 @@ typedef struct STaskCostInfo { SFileBlockLoadRecorder* pRecoder; uint64_t elapsedTime; - uint64_t firstStageMergeTime; uint64_t winInfoSize; uint64_t tableInfoSize; uint64_t hashSize; @@ -549,6 +548,7 @@ typedef struct SProjectOperatorInfo { SLimitInfo limitInfo; bool mergeDataBlocks; SSDataBlock* pFinalRes; + SNode* pCondition; } SProjectOperatorInfo; typedef struct SIndefOperatorInfo { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a76253ab20..ec8e3c4abb 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -43,10 +43,6 @@ void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo) { } } -void closeAllResultRows(SResultRowInfo* pResultRowInfo) { - // do nothing -} - bool isResultRowClosed(SResultRow* pRow) { return (pRow->closed == true); } void closeResultRow(SResultRow* pResultRow) { pResultRow->closed = true; } @@ -160,11 +156,13 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { SArray* createSortInfo(SNodeList* pNodeList) { size_t numOfCols = 0; + if (pNodeList != NULL) { numOfCols = LIST_LENGTH(pNodeList); } else { numOfCols = 0; } + SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo)); if (pList == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -196,10 +194,6 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { for (int32_t i = 0; i < numOfCols; ++i) { SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); - /*if (!pDescNode->output) { // todo disable it temporarily*/ - /*continue;*/ - /*}*/ - SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); idata.info.scale = pDescNode->dataType.scale; @@ -701,9 +695,6 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu } } -#ifdef BUF_PAGE_DEBUG - qDebug("page_setSelect num:%d", num); -#endif if (p != NULL) { p->subsidiaries.pCtx = pValCtx; p->subsidiaries.num = num; @@ -852,7 +843,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi // TODO: get it from stable scan node pCond->twindows = pTableScanNode->scanRange; pCond->suid = pTableScanNode->scan.suid; - pCond->type = BLOCK_LOAD_OFFSET_ORDER; + pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = -1; // pCond->type = pTableScanNode->scanFlag; @@ -947,6 +938,7 @@ STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInter } // get the correct time window according to the handled timestamp +// todo refactor STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t order) { STimeWindow w = {0}; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7bac828a53..906952be9c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1665,9 +1665,6 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) { // hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); // pSummary->hashSize = hashSize; - // add the merge time - pSummary->elapsedTime += pSummary->firstStageMergeTime; - // SResultRowPool* p = pTaskInfo->pool; // if (p != NULL) { // pSummary->winInfoSize = getResultRowPoolMemSize(p); @@ -1676,17 +1673,16 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) { // pSummary->winInfoSize = 0; // pSummary->numOfTimeWindows = 0; // } - // - // calculateOperatorProfResults(pQInfo); SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; if (pSummary->pRecoder != NULL) { - qDebug("%s :cost summary: elapsed time:%" PRId64 " us, first merge:%" PRId64 - " us, total blocks:%d, " - "load block statis:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, - GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pRecorder->totalBlocks, - pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, pRecorder->totalCheckedRows); + qDebug( + "%s :cost summary: elapsed time:%.2f ms, total blocks:%d, load block SMA:%d, load data block:%d, total rows:%" + PRId64 ", check rows:%" PRId64, GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, + pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, + pRecorder->totalCheckedRows); } + // qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, // hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, // pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); @@ -3031,7 +3027,6 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { } } - closeAllResultRows(&pAggInfo->binfo.resultRowInfo); initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); OPTR_SET_OPENED(pOperator); @@ -3400,27 +3395,33 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { break; } - // no results generated - if (pInfo->pRes->info.rows == 0 || (!pProjectInfo->mergeDataBlocks)) { - break; - } - if (pProjectInfo->mergeDataBlocks) { - pFinalRes->info.groupId = pInfo->pRes->info.groupId; - pFinalRes->info.version = pInfo->pRes->info.version; + if (pRes->info.rows > 0) { + pFinalRes->info.groupId = pRes->info.groupId; + pFinalRes->info.version = pRes->info.version; - // continue merge data, ignore the group id - blockDataMerge(pFinalRes, pInfo->pRes); - - if (pFinalRes->info.rows + pInfo->pRes->info.rows <= pOperator->resultInfo.threshold) { - continue; + // continue merge data, ignore the group id + blockDataMerge(pFinalRes, pRes); + if (pFinalRes->info.rows + pRes->info.rows <= pOperator->resultInfo.threshold) { + continue; + } } - } - // do apply filter - SSDataBlock* p = pProjectInfo->mergeDataBlocks ? pFinalRes : pRes; - doFilter(pProjectInfo->pFilterNode, p, NULL); - if (p->info.rows > 0) { + // do apply filter + doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL); + if (pFinalRes->info.rows > 0 || pRes->info.rows == 0) { + break; + } + } else { + // do apply filter + if (pRes->info.rows > 0) { + doFilter(pProjectInfo->pFilterNode, pRes, NULL); + if (pRes->info.rows == 0) { + continue; + } + } + + // no results generated break; } } @@ -3884,8 +3885,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo); pInfo->binfo.pRes = pResBlock; - pInfo->pFinalRes = createOneDataBlock(pResBlock, false); - + pInfo->pFinalRes = createOneDataBlock(pResBlock, false); pInfo->pFilterNode = pProjPhyNode->node.pConditions; pInfo->mergeDataBlocks = pProjPhyNode->mergeDataBlock; @@ -4416,7 +4416,7 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; pCond->suid = uid; - pCond->type = BLOCK_LOAD_OFFSET_ORDER; + pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = -1; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 43750cb967..e045b17af0 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1057,7 +1057,6 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL); } - closeAllResultRows(&pInfo->binfo.resultRowInfo); initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->order); OPTR_SET_OPENED(pOperator); @@ -1213,7 +1212,6 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); @@ -2008,7 +2006,6 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pBInfo->resultRowInfo); initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); @@ -2172,8 +2169,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { SSDataBlock* pResBlock = pSliceInfo->pRes; SExprSupp* pSup = &pOperator->exprSupp; - blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity); - // if (pOperator->status == OP_RES_TO_RETURN) { // // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); // if (pResBlock->info.rows == 0 || !hasDataInGroupInfo(&pSliceInfo->groupResInfo)) { @@ -2313,10 +2308,10 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, (SNodeListNode*)pInterpPhyNode->pFillValues); - pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pInfo->win = pInterpPhyNode->timeRange; + pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + pInfo->win = pInterpPhyNode->timeRange; pInfo->interval.interval = pInterpPhyNode->interval; - pInfo->current = pInfo->win.skey; + pInfo->current = pInfo->win.skey; pOperator->name = "TimeSliceOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC; From 93af7aaaf77b1426c722aad4dfadd08f1448d5f1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Jul 2022 11:18:30 +0800 Subject: [PATCH 08/27] fix(query):fix limit/offset bug. --- include/common/tcommon.h | 9 ++++----- source/dnode/vnode/inc/vnode.h | 5 ++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 9e7aea03ea..e1aadd4486 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -153,11 +153,10 @@ typedef struct SQueryTableDataCond { int32_t order; // desc|asc order to iterate the data block int32_t numOfCols; SColumnInfo* colList; - int32_t type; // data block load type: - // int32_t numOfTWindows; - STimeWindow twindows; - int64_t startVersion; - int64_t endVersion; + int32_t type; // data block load type: + STimeWindow twindows; + int64_t startVersion; + int64_t endVersion; } SQueryTableDataCond; int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f2b791def6..277aa2d08d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -117,9 +117,8 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur); // typedef struct STsdb STsdb; typedef struct STsdbReader STsdbReader; -#define BLOCK_LOAD_OFFSET_ORDER 1 -#define BLOCK_LOAD_TABLESEQ_ORDER 2 -#define BLOCK_LOAD_EXTERN_ORDER 3 +#define TIMEWINDOW_RANGE_CONTAINED 1 +#define TIMEWINDOW_RANGE_EXTERNAL 2 #define LASTROW_RETRIEVE_TYPE_ALL 0x1 #define LASTROW_RETRIEVE_TYPE_SINGLE 0x2 From 86c4d343390d3a0ffa45967f5317a496399428b0 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 25 Jul 2022 11:19:52 +0800 Subject: [PATCH 09/27] fix(stream): recover the overwritten code --- source/libs/executor/src/executorimpl.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7bac828a53..b063d0b3bd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3328,6 +3328,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { if (pLimitInfo->remainGroupOffset > 0) { if (pLimitInfo->currentGroupId == 0 || pLimitInfo->currentGroupId == pBlock->info.groupId) { // it is the first group pLimitInfo->currentGroupId = pBlock->info.groupId; + ASSERT(pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM); continue; } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { // now it is the data from a new group @@ -3336,6 +3337,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // ignore data block in current group if (pLimitInfo->remainGroupOffset > 0) { + ASSERT(pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM); continue; } } @@ -3380,10 +3382,12 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { if (pLimitInfo->remainOffset >= pInfo->pRes->info.rows) { pLimitInfo->remainOffset -= pInfo->pRes->info.rows; blockDataCleanup(pInfo->pRes); + ASSERT(pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM); continue; } else if (pLimitInfo->remainOffset < pInfo->pRes->info.rows && pLimitInfo->remainOffset > 0) { blockDataTrimFirstNRows(pInfo->pRes, pLimitInfo->remainOffset); pLimitInfo->remainOffset = 0; + ASSERT(pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM); } // check for the limitation in each group @@ -3391,6 +3395,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { pLimitInfo->numOfOutputRows + pInfo->pRes->info.rows >= pLimitInfo->limit.limit) { int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); blockDataKeepFirstNRows(pInfo->pRes, keepRows); + ASSERT(pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM); if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { pOperator->status = OP_EXEC_DONE; } @@ -3412,7 +3417,8 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // continue merge data, ignore the group id blockDataMerge(pFinalRes, pInfo->pRes); - if (pFinalRes->info.rows + pInfo->pRes->info.rows <= pOperator->resultInfo.threshold) { + if (pFinalRes->info.rows + pInfo->pRes->info.rows <= pOperator->resultInfo.threshold && + pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM) { continue; } } From f60bd675d1299cd61380b75eab20978255731af1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Jul 2022 11:22:31 +0800 Subject: [PATCH 10/27] other: merge 3.0 --- source/libs/transport/src/.transSvr.c.swo | Bin 0 -> 53248 bytes tools/CMakeLists.txt | 80 +++++++++++----------- 2 files changed, 39 insertions(+), 41 deletions(-) create mode 100644 source/libs/transport/src/.transSvr.c.swo diff --git a/source/libs/transport/src/.transSvr.c.swo b/source/libs/transport/src/.transSvr.c.swo new file mode 100644 index 0000000000000000000000000000000000000000..c9486c50867994926ea395577110ee4d15de5632 GIT binary patch literal 53248 zcmeI53wT^tb?-%a7AT~=pY4fLyOJWyj|7L=0wbx#I@3q!md!Mh)P)+$?C_2p){vSqP4*V@u< zmTIj=z1iNJmtv`m@8|cE|nugzLw{-#a?)|8&Rw zrBI=m|9v~|f34&G{lfjl@b~cD{Y!lQ{^9y_!{3i|+z&td&I$LQ6c`qt50}^ftaD(U z1M3`E=fFA#);X}wfpre7b6}kV>l|3;z&Z#1OE^%P@9Wz@$vb2Zv;H6L|NnSkU*8A8 zGB_9f!vp&Iz6M?n_JLmnzX1N~XZreH0$vPW1Wtmdg9?}j=Ym1-K=1(Y9~jbK0q+5C z0j~tV2~L9Rz>~ox;9M{W9tysLasDOn2jJ!4Bv=3yFbM|10JuN+E{6W6!Kc9c!Mng~ zK?Pg_?gxI1v*1VI@4+X)AA`4mmw=nVb3qkc0nP_M2kr~Lj}zjH;5P6!@O$8O;8(zn zpazz}72qQ9i{K&P!Qek|aC{B?Irts$Yv5Nw3(SK3;6m_y93}q*z6L%IJ_k;K$AVAd zWO)O)4r~K=;A40*Xn{+?eZalJJ;0}NNPHaJ0-g%0;5=|2aBuJ#+WVD2ev(CS0gwzl z4y@tRYM0v8a&oL*ollmJ9jLdf3n%tfYu8=ctY25D&D2L4js9tZH%9BV+F(-V|AAc) zT6k)@U23+cE4BHlO3PF4Zygy-mRd(hTTD9HtTal^iW2WBwM&+DOgsbuvvbP}v+cp; z_*~bcg;K_A?Mky&TFf$Nd2t=(UDm8rM(6s;_-Ls%zgSV;{;ezjL#4&*dM6z+_6#Pa zrYh7?!XctyMi!uErqsGl1t}~>#WX9|4<*%FI}rz#TixZ6Zcn|(Q>07=w}7NwU8-z` z=dIL&pVftY zC+*b)mbcpVjO5*SGMS#PG>=u96LU|cTO@P!`XZuF)6CNo6Ujp=myMjr?n;s^Tau?P zBQkW8>zAwTWT9DK^8ZRE+OgA+34Q6j?T8v5MXjlz{IzPst{j_fAR@HVV4^RDYkHDv z`>HUiW|U29V3;}DoL6%GBE+E%8WZBDw!Ad4pp=#76e&(9)2{pebgT28dVZxjK7WI5 zP}|%{KpqSAz@No>y&=)k-#~M_%oRvz8EbWMX5I_oV%uol)z?1JsLWRu7`~dz<#w(e zk{g-1leB(%+?vJMtY4Z;RB1&4j?-&(?fT`)a>eEk<4!DZ|8l!>gJz2TZL5YSg9`g_ zUC&P2pfX9*4|3VsTR(J*_K)@BR<4Tm*6hN(H|YYRVktxOkucwY3@jRils0~4=++~V zNIkGJUt&84wlG&Rp&8l60DjEC$wchb)nW~;oava*%88XuCtY1_s7v)|6Duq$*;7%G zqot-ED_>VRad>8W&!w}IBh%A|CZ_hx?i)LBt@n;{wQ;o4)WkxnzU-k^rQEEvdp(v* zw2bU{l7k1vbD8RTGQKBSs5g6RFQ!oa+n#E(zICVeihc4N-;oWKwsO2P&5 zklHg)iq+hs z{BlW;(*vxEx~{|;Fz9@ZnX-)`GqY1;m(A`O8{IcDHP%1SJyy8Fmj7>=>(VOw}NDgh4$|`sCtdLumu`mn5L` z#_l~)E7LtSSnaRRFE3WcRqDmbEgvDt%HGVZ$iR$8pnxsF%cN9hXHR?=wJk2Fh5!^!x9NMeSqRFaF1 zw%d)JTeciOeth^yZF#ufJVFq4xl(IYT3dDxOAWlLzMQm<)|VIO(Or*KXzX&OdJG+~ z#E8{6k<=Hgt6hHJU^1c!y`EfNsa2Y##bk1Mj=Yk6o*#KByUWIB4vbAtCznl3B_qk? z$kfdE=s_AXnLIc(IWawEWsV%UDmgSVH8pZz=Biy*2b7hha;#FbvZ_ms#VQmWFEwFY z+i2K7HZ{7J^dpy!?;D@FiiT5ZGr5{4Fkzl4vRR#D+?Nh(nH{Zbd^}oeCoukab#XD7 zt0W903(Jc`$ZQYt8%8dl+_zz1h&dr?sH&}_Ls;f%_Ik~#M?w514~X7$L?cG6!SNVad?c40CzGBKUVRvJpiYvtkO zqNg5hm+F^*7WF9&m+MQrrTEMKe+72)ZtQH?|K|kb?=#r^?*K0V3!nn_fo)(bcp$g~ z`~L^v1K^$DRp6DN2JQ#Gg1!Gva0_@7cqMoW7zGak_XPh&xqk}e3wRZfFW?p6GB5^2 z*TrD{v(ABa4y@xaKp+6?{~xs^5yC6ybF^5m=+uzhD| zSyq#&%3`I|s*K&>6!*vKO;r~5A8GevJ(|bKDEP?{w{54~v4Im?t+w(woDOkDxVb{L z;LtCp^8DuAsXKHi*|fBI_w@x%Zfu!PQ>?`4(KKgx4^CpAE-ucM%3`UNvad8(S)_#M z!G%e%dh>2?87so2An)eIXA)@29yD(gWXVCpecV9SimE?XX@4%ZgCj`(w*9tnsEgRQ z`zuTJ=7~{gtC#z?`GQ1!Aujn1KnB^2OkZ_ibarfNYW9g!V>8YbCono@-TyvMrd9`| z(H>R7!}3TDn*@*`_#W}CS=FJ&sN>F{5t)`Zx_4~!Nwau6_Us$$C)>Ob(TB{DnW(+- zW%4u+Z{D4iWPY$Hkb@$tkjA;ZE9;DrZP3$Swso+SK3`_V(I)$!CGo!RX(++|kMaF~ zg?;~4@Juia{uO)vU%)58$HB|Mlfln}zr&8d9y|m*7H#7 z8@>Ua0UiY20s%XL*5c>D9juqX5d0jtl{N9pz?WG6u7RHduVC$a5c~=2+*9Co*0Y}m z9su6N`nA@v|AN5#rvTOKyQ*9IfxmoC`k$ZZ|JU@z?3XG_txCIp6Rl9&yqm6sTX-nh zn#yMWIszIWJB`0tS^a}qKia&zywDiP^+?7|+*&o9h?^u5DKW!{8ol+v!F~HqU3^wn z&))3$W<=bX>ZCQ#%8J9hve(Ge^Z|M(x@mn?Dk(J@3`a_I>WZYL_?K8#bpbzGo;h+T zYmPdDpIFw(GnnT}S;eyAL~@rJWTisF^;nj8c(zRnwjeOM+&UV?KP-PB{I(7me6w4D`7)Xm1xMt~U;-5#v4Ns9dobF?@?R!qOP|gr~ONaN*;&UvP3)63a?w zyf`DD(O&G;k7b<}Gwx*KSeA7yX~de`?Z+tk@MQ!m>}46Rzu#7{29EOm&dnzxuTa(~ zg5LCH!`P9->fvdRv%RVw%kpv87hP#qDn$&-8SZ1TEXjt+Rgfym6T?tD{rZ{c)v>Hz zbw1PAVp-iYWmxzNQzlaxX7;B+(Yk^Y)={o0}!2zYS2mQYO#Yf=nMu-H9bF|2lX zrj~VNY12TSU?-t-*`m3kWe${fvKZLf zKWHPj8Rwq-@U;WcP&E?`w@GG?QlWW-JGn8-ywy--oj-d8;cD5hFlr9Pum!O(O&X~> zMOj5^V>w;ny4^&s^q}?HJ5{JEcghE2UFDAI7?@TMMz-di&m;r>V4+!g5|lK8Idlr+ zW#=;;sR=<)&|z8AX0m0_{~k5(SBOm++(I7-Ip)ZNaDquCo-X-Ek@tD;P35DN@^#7a zqZPsrnn`I<@_j-zok!;-w4&0kBn|vAm3GY6C=C^xMvu%`7y_TwK?ptR`@A-)WB~zK3t&GvHm|CEzMB4JN=j;79lzJ_LRX zycs+Wd<_3X30w)jkN@EX;341(_#kcr9{}$MzYP>?zX{wMh~J+R&wnBxoT3_9(G6X> z?$VPRdri@ap=Es*H+9~yZ0TgFdV@lA6o1rE$Vy95RE)6Ohm*Quo?5sO6NU9Ckc44> zvFhH!Fe#hgwT%1{6sxG)r6qz_5a~q)3zE&Y;dHES5Diqe=$aU)r1CBQCVK)XE=(T& zWFWd$*9gQoxNS(DJh_T3LIRh;ME;1cp66jPOj>1LSjUEr+VQjLR%M$5@frvvro67hU!$OVy?g1i;C3mHcn zh+QL+2YS_}kko!Fq&B*>6-AwS^+*|t%&sY|cSrvps`pgpmXGvru<-z=U8zh^J<)86 zwu>5@IqWR2v=Ob#u25O1h$Y9$nwdD_S&IgOKE)9qiHKtIB$A zp{MymfX3K@T^t_LjEt#Fbs)y-qD~q-9w8(B60Pa&kaLfsa*0%cqD<3F*w92YYd)Va z%8|&j)&mTB@(e&kWD14JQqqIbHysJd`=#SGEt*6zAeWSAt6(qV#TXF=T4w_zze0j7 zTG%!__4oH@y6r&VT#SJp*t}a-#!#Y=Vkx!Kf_>Xl{FUyWGTM~AQM-cFgyPjQv#mC^ zj+B2a0pBnw^T?yjzo;7!Dtaq_>Gi^^lFB*k&aXt5av$eqVPA_Gg;XIS3Uq z6MH85=Q;iOMD=K?KCp95yS3cpku=d-<;IEF+Z8sSPG|muIxSXev7L3zi_YYihKJ&# z?Rt&1x>Ty8IHZzg?zuv$bb3skYI+*`Pk+d=zj%EBAu|EVZ;H{HmJ`sRhtSH}_u`uf zG``q}B;vT;pCdZG5786*xj>jQC zDh-o6gCLqjiT!2As_hf0k3>YIdn@Q;#zRx*T}FBrKN7N0v^@Ds=t2a2x}gwi5DPRH zbgiV-sFbS5}Kh`y#hZ+&fxYhwSaVl<40MjPx`kG@Jqi>{}cUCN9u4ZtmqR$u!c{3GJ4Aj zUW3%Q(jG8&03NB;i3Ue7H{jj!CpC-6B4_cOGziri%e-^mrOa2c5}($7}kZv3?f0sur5gZ;&33TbIMC z*_vehRE)XC*8eft`#v7*|7x&Dwf6s9a5cCJd|Bj9H6dhl#84}J;Ef^qOL@HsMj zJva=m1{=WH;CAf)Uj;7$F9er@3&A76_ptqM1HTVm1l7zY4q<41rDHQQ%?V zr@$T9{NDrL1+NCr2Q_dx=m-B3oBtN@WRQSIfCqseVC#Pud>gz9{0h+i`zL`Xg0sOR zz#Z89p9Y@W^0TicjKDZnB1N;NW z!FAwLa1Pi2ZpSb1QE)T(4e)GmHJAi@!KL6F@Ccyx`pZuLfttHL)@;_BnZ}*PLfJ0k zV6<;+g^5kt)NG~MoUP%r?RSOR6^4QZXR6gf_EhKW4^0*}(a;O~T+-s-?24|hB*_e? z;d!Mhl&1^DOxBu>>;6(9&c4{=8O+Y{4AGxVw@BS!16DCxq9$}LE^K)hYoQ54UtfF_ z-lkffR^AkcmZLxDpO)7o3T6oQ2zeC}W^7X4Q2~!r^C^Nua`Yq^IWtlMv|aYj#d;KS zbNx*Y{g^3gmXnoT^7%uAJFES^?|*dD$?v%~5u;t?q98N=w`H1qCi!~fuCiTp+g42A zoAPWEANg5GoR_Eh*s}Mf`7CNo=C79#?L{|^gTa`m*@-+I>B!2?MLUcwHQFaS1t2R2 zb8 zG;&~P${*(@Q(h}pNf~3!tw3#b)$+sDU{OVAxC-!8yLRc(R-dV`cXW#Iy?*1s zbk8*%mXNCT9V4K0X89{2+V%e;4R#Htx^F%z^JeJP8|{pOpu+b)c_EgK?!9^U9C4#+ zrF69Dibd2?*#H~&EIB5SHFW&JqubM(o7O!_!(4jT4_$yrjXXa3xSBgRYydpKPs#A!c_Qk{-!Au`Np%x-4gK`qxY z&L~4<#5RyyUtTspke+6K#gM)z`sUh)O3hlqLu>b$t?NxZvB>jkL>Fgv%UTmVqeLTsW zDV|W(;EttAy&dGKyUJ~`y7B;dSI(M=<0n&N>LQ^RHBrfem&XqrkWl+-cZX*MMou-# zL`$>RKqEDS>$a_YT!%wZT4PkKwLepBv|rUd3SKxsR`nb;48G}vqZsdr-XhH<|9~>EIrW@g%zdZ`h*%UmzO^_t_8FH=!qjF z*a=9V^e?meV?A*iZpNY!lK7QOHf76BS`xyo9R^5KYuF&)aB0!aY_y4dEe6iB+XG@H z(828(*Y;T4Q5HCzz!#hi?GIGwLbcdAgA zH1yH(2y@NarBv(xn7V!EVpD4U|JlL*{|9XTKL@t~odfuM&;+}H_W%Dow*EhZkAhpl zyMWFBXo5+w704g(O>F@;7ZodpA(&_C9Pck*Of8xq9kLvSIZ0ke##x;pw zoOr6eoTt*i>A3Gv3NGWb{nq&O?AZRvnXBw}7%`-G`+|ub6LqkLgZ#SwO&tEEy+iFA zY<%;_t@Lo`dg{fth)_&NCi{A1!FV@UYL|(UXx7V>R!fRb$qW|0sLUfRL%}e+uDW3 zB}S7DBBL{tiejTvEPDWA@%jfx-cc8R8}WDgf( zrGO|ewHm6PJqHOg0OIkOAsvRC>XTdZ45gu9Om+zxJ=$R%->gY(UxTu6>mcdG%J4-J1_bNQ+cJOTz3;0@GCN#I?3~VQ<;d0*ZPFk&dDmh# zWZ2D4KIY}TgL~=vNtlZUi!y(F&WtcJ3@g&QFz1dO2savo-o{T~!S==R=)Ot9M`Cqm zg2F_KF)L=BMKxultUc~)HJd>pTX{vftO1Qd4(ADnZ{e-9J;1BQgF&mse!2McaHLp+ zsT>)r(uDINNBh&Au1J+`s!wEAZDzmHKtU|BYt}f}&UP3lbVL~@(|sQKqQSDiVTDjL z#lsk1Jz_mw6!1MW#@c*)hq#>(3K231fr6nO()|dbTW2n4->2{dwkhnCvzl7@K;{24 zsiy%LYDg&W->QMzX}2A6%Drs5`V`Rk2YVK~o|+Lgy$VN!0SUTHSz zOUXW3YK=9fD;9EPEpH}oysA0|+Je|hJhOB;Sq{l^fw;28wc&dp_dqTl*M*7=mbqsLZ zHDTqZnSNAKsh; z{glz@a4j-rV$G*Y!xD`oKAQCE&#H!z9hc*xc4ErOE?fUkFz~ddEc-vcUr*=%eF(e{ zyaqf2=={Hnz!q>{@YCSS*!bETumCOscL#roU4JWh1JId&$HArGEO1|NFK`d=aqRoI zgO`FB8!*fFtAXAD5YPU*Ki_p`z@LCu0-aa*a&R>m1NQ`X10TcQe>YIv!0&_Sfd!y_ z1K+{cza4xHd=>mIcp11BTm^Q5v%!zB^*;vQ4Q>U$51tORpWtdBpTNVx!@%dT`TrYu zH+U9!BKSJC{D;7+!HdE3z)e6l{{Z*~cKp}DZ-cjh5ukX1Phzj%23`kV1%4Ae2@HWp zf*)b8e+>K?xEVCTG#CcjZ=iGi?*SA;@FDPe@GS65Fb7ItH&8o29GvRoBf#18DLekS zp@Uv(M`p`9Jhfk!f=fImKzr0x4Jz5C0nA&ZGiSt_fkdLvCGB=ccc#C)=4E0V+<)oC z!&|S_{OGH|{KzNr&=eQ)%ORoCRu?aFO~>W zy$Qi2ELkdXW-Wefj({D?rWj_Ijq~iXaSArOYe8NFBQd2bmeO%qU9IyHis3-AF=Swe z#JeMmv!;zuSVQfwy7-^x!%r<@#vS9{XE>X|!-(@;2zOP}XHp)89hwX?`_qG=MO5TJ zR|c{^)EyV?Z%PPjl5q}8W9%&R6?vdAVs&0<7&?;F!gJ&);gi_FR zmpbK^`r5YU7+yho9jS3;xsY22$diKPe)lD$BIk)0~3%~3HC zWL`PJ&gw{KmJXpY2{!^dN7SOuv)NX6(gDM_0C&EpB9?%jj5ylMP)?x%#(t>bN!DwO zVF?ae8k4q5^+BVP!PO4gURbV`?K~?I(O0d*ily+9BrJ<@+%iI9$?OST?ici#}dd>rRS=PVjz((*WEgRT9IIN8^i^eXqJ+dL*+^Ejy(6>6=nQnbaTs@UC3+c&C{l)uC+K#c&Qq4+ymQq@6SxX~O z{UHO?(+-po`e*o#-RfZ0-3`MtKu<1p5(yDHeB&-cvNpV-IJ2M@+7tFPIzw4Sm& zqmj&UZrkEfd2zam^qBEx(KPW+>YJG+la5H#J4Fkk6hZh)iSdiN3|q zEGUimYZ&6I+Lj%AOg9-((^;;ChOC!ukoI?CnJh_Z*ZTisFa{qYn=ohpe+AqAv*1(U zli-!$M({K+4W_^Vco6toY<|51;Qipu;1=*Ap!Wox1MUUx3BG{M|9PPO0eV;9m0%WdFY%`(A7RQ(zC+4YdCMEo}R5g0BGC{4WK+3SIP#{0Rx4~Dy{{lCI7lLPir-2>dtJwB`3-oTkYry$n1JHSdj|Jbs&Q~13d%?}% z_24<+N^lTd4$cLGK=A;I0T=>$hu@>Yqrf+@`M(T42>uk@3XX#v;1@sweja=s+h6Pa zPX*&(FX#t9#MXZUSOSN^As`>XXJ|*AHSluqQlLEq@&OzMJAvBy!C%{tnH=Hsumq^?xD4{ix#uj?kkAr&P$TwExzjokY($GA4x znRjtwZzdX@*fTaeH90yvIWaXeON79lsqq86NhPEWR&X|v^Og@nn7e0jGIbAu`e8r6 zD1ZsZlXvVA-DJ~gwBsJPZM(0gMfDYzNYvI=sYMrMbJZH5gPq#N3UoZ8GdO-gG~T?&ND)2B(aalYN^->`Mq{^b=08N%^d8|ypr)@zm01U7uS zu8~t4Qe%-jY8Hxg9m32OOYT-n*OTIFE~DVF?MkM>JYB@?3bIa%YkLFRSQ?H4=EW*T zik%iAH?Q{aMfHid?UeNv8IRZSIk<;7b~U7`70mvS)oC?rCs*<0=#TlMLaiYywu96`fi1k%X`bIajO&Jgla<~JhblVOE@mEkw#Vbc`yR&otGo}oN4U73dT*2;SAn`A}t zDyGxv!YI{fSN!DPs9x~)lx&r?tES7q8$B-vcpAGtar^Kpd zlqfePIhzPNFP=dbCYYaq%=YkU)BKrxB*<>=Xe^6}w^s%t&(;!o-l}E9RaKd{&&7Ep zb&+(+xOf@LkQJ2{S2X=qLg>OCQO0jEgH64;;kR>VFNwN`6rxY!8#M=4RRBun_w=!_N?f(+OITx#h_?A1tJ~5 z@^3u#@x}w)y??|W68Wrl)aO)HAVruz)O1D1&=J;~wLHP$NyJbyDVvFCBI805yPlK* z5^K7*?VFYk=k`n)gH0Z~4rL(Rx4cJ)^k$A%z&9Zip}!Jt?Z4@*`KWY_kIQsplXt3q=E35OTM*fThu8#=$c0hFYoehQ37f3V#ak%e zE107vn`ZQEm@@QSj~PQLT;QzMxm?0{AIi(rw61jkOOa}5DnmYNMI1S|Y!dWx;^{Et zU6=yXV(~3DhkKmZ0te-!)w@8MhYOQlaes-`Vhe$^MuvK(J*O8l#u%I|?kJKW2E+}s zHfeT8skyYM7GAnDJuGg9#Z}q<48=2}lSTzJM{!YPP?|^sxJ}@3O;*81z9icG%H-~yoc0DceK{=4AA z;LpHo!E3+~Z~$m8z!vaB?0mfw@GroxgKL1!3D^eo?tp*B&e!<>Hv*j*cmvQm0Ye}G zj{%PcKfvz49o!0@1D*)v6SxG(FYo~HkJ$RRf!_r>NAMPK99#nK3qFgT|61@oa0KWK z!GFQF{~Pev;8Q?*1LOnfT>szBwQb;EvF~pK9|Eri*MP&|=YZk>be3Q|JK!yRe-?Nq zm;-x&;sJgZoCW>{TmOCFW#A>?*T5vW5Ii2-fzAKN;00hOxDeb6d&YQuV`}9%_vaa1_fn_R#`nLre`NTL*_i}Y2-IA14nX}^l zB=cZLZEjI-74b!l!irns87}age`|Svk?Lv23)%;-FCLqk=|?}-GK#tb7lXFEq8YQP z+`eHLO%iK_axGL~wmhemBkZ2?ocG-ZRNGY^ft4MEg*mD? zLAent()O7HrgH-8B05&nhBzCpbDF9yvNrFvjN>5cZ;UQCD&VOcw7 zlNQd_p&FUjUVgW5WI~GUM z;6kixChx4^UT1$)ZMe`2$+a&x(JK1ZUHrKy_uMq6j7K_6EK6{yEbwM6?M+cFBf_z#Sc&5Z73GY*1Ck+PUugQ&XOvYF}&d zF<6|leMJpKd6KA}slB`z-1gLO38GSINV3&D^ABW-lHcptxm4^-K{`gIMCAv*jwQrk z@49Qu35&}_EJua^ajsd-BV&^+6%r7m3BiD;1c! zJBxaBn$U+vCMVyC#)eikPP!3aGm$On5ujA99V;zzD7cqOY5$2*6k1jti+Lvc38yOl znl3eQICDFAa9o^XYwCoYcv4IVY&P08UeO0l(9$Cs=v=k3thtAo)Si2 z_rzJf>FFFha${`j(4pKr8;3DX`&^+QlC7l|ufQ|gtt94oL*|b|PZhM#Rn2^}9y#OT zg$$RWD`@3e^_GJqwYaT2hYi~Et8LJxQw8Tq_|s$b>#=<)HT8Or8$E~R=sTqbejHOa zF8|jIF~LAiv<4r;3K&3AUZ*2Qe39Imh8AQZ7~5)g9F_TVhSh}OZW?8tuHVH|u0n!l z*E`S(H#CezF*Vgd7DIX#M3`z*4>~Db$Cx}gy*G8UOKx!ZgnV-fN5S1L5!DfnzR5Hl z1h4NdwL{U!<}ND%PGlmV3$p8B6dh}-&``zPWEy7&JMlmDerM(6oCt34_^Scv&}Qze z;OGsK=_+KGt5@s)nDBjH4(I|z+YpZzZbk2+yp9M9y}I&1Ka%TKx_GL1g`;?f?@DO zZ1it{_X64N$3O$<9RTNn`-Aslm%jtN6`TZha2nXHW&c|;1k&Ee+b?UPJ*X^dxEcEtG^4p3A`4Z0EfUu;L+f(X*adqmpab^q#YAJ zQ3H*GDE-b}rI~vnYWf<_M7vmu6OA5Ar%KnF`9F7rTOonZrAauvt#Hc&i93#Mvq{@J z+KT6x#d$T;B*W~Nd$!lR<)AY7w6hsM4p}{pe~WC&De9|ns;Iqps^kQbtL`>e=Xx)f z7$l5+He#d^^Ls7A3?;)_Cfnvq(%kwH3n(btnmfglj@_+k1Xz!XP;C$JH095B1@>>> zvVGu`mB>i7s-RS+>Lc2TJ85Iof2>1`RXIQGVWs!iIQ>DfDHv$p);($v<9I_T6i)XL zjTSC!hSF2kCT)_iAheQPgdjeJvAF~n#U{V!ytwGo(In&Xz*dMe{zY}F_>{=#+A$vr zc8hXIGLtUQbg0W|tyZc|&^}!4I~{2>vv6&VnW+xuF^CoR_6#jO63#y`A(p0)uFGkd zTBEx|l{|$5gCWl>Wb=2(#@F#`!5!m5s`$E7*QyUx$WJC#|C;9t;uzYx|RbOtFDrxNGf@N2SS?JgAnieC$wnw}PSSqdUL%;z-KexHvF9v47;rdZCEC-Fff1Q_+#Bu{akE zsoyatefP%S)prhFuj+y|C*!F;YC4?0)0M@AN%%rzkI#oOpjB%$k>>?lQdJHqOVTWd z5?-6Mv8CU`riTs_l0Cd}>*JriId^JYYo@Z)*i$9wU}n$kp7E*au}S2ff~QLhl__+- z{sHY8Q^=C7EvT%+lOr>GXW@Zl?Z&OVFYp|_M~P%>BkLMIPq^k9QFhHWIno+=!ipp$ z@L|-q?@anZRw*lrr0qEWTF9v`@XpHQ(D;Ep6Njdut5=h#2jv5dx)*_nMKuGTd0!pY z*X2zb^wY*RmkowfIH9pRn^YTIsW#Hzs;uOiy!4fx*`sTA$!glF*<&taUT)eEm2=t` zPA})|v0Tobi+tp7E=()dhh6rf7Ov*I7&r1iQznRQA~iC!Fgz7~#c2{zCHwyuu@$GW zCAI$l++hFzAME}A0d(e{_WbVvj|Gne4+sB>4e;yWTCfjX0Cs@C#b@w`K<5LT z0Clh(+zosme}T>lkT2m4;Ps#Zo&+|6yMzCW-{4!|GvJTFd%!EeN$_-VIncR)8^GD% zAMq#L3Z4(H1WyE)fV034Xn(!0@IBz|;3Uww0SAF(V+ySCqd7b1!hXIdgQM;IVjaSF zy2$+X!%Cvp`>2_~#W1+kT*b8WRh004m+d)i+e@&KE*NgBti_l=*9Mm}P13d;N!FcD zn<9I_qK&7StzwK)8orDplhX@5ZZtipNyeXe*2}`*<`#a=`)d`K^SWF(e97M0WS0gu zVY$g@;Z#@1(KQ>SDcBD*O0uJUbd;hmqVh+hP-gWr40UNnIlaQY17ZB{c$tm)DxFU0 zaty-r>^{~XGPSDM5BCjIaPc%*YhI&;O4<=M@ulyBMr1ZyxntjgY?a=r7~MNI`lQ*t zBM0{E8`F-qa1yMMgF>LQdJ;ZyQGhH1ud;HZ>_wk?mc|8qSD!RxYihOf6mL!>G_Vo^ z1k)pb%ykkyJ%z4x%Kn0J7@Q(u3FRlydG0z9|GC}8M80s2dfH(EuZqs2;4#nE*s)rQ z{Ro-KIz^nAp@MHsE>6G!5@TkeLbb9y{jCra_eAz^l6!O6f*<%z?{s z)KKlA*Uv%SGd9{?prv%eGAFl+_T2IT6XwRU(W4_czGr1)H@%a@jvqKv#7^bk8q}Lq zOp^U}GLJbB(S_kkpPOmL>nv)c>nnMOtxF#dq(g#rr`7c?P4q$YiVic23iRa=^T`ByWxwU94LkgU@}%~& zfiJf{!qAXOOV6d^{qp?>4|WeJ$ZSm4lcLs;=R3@a=EbiWxh!M=nH&bZMiO zbY^Pgz%=gzoSvDQxN6oXhRKPEeY3ouy6Xu)(V5C{{*g!dY3axE=@7s$`=%n;A$;L> zm-zJ>K`m|902<0qJJL!l%w#_|1Vy5wGn*!^EV1V=3!hor>$}$aK8mhbrr<#}&S~E4 X8I7VLU*SJlFp83@@A;kVNZtPjn%Mi$ literal 0 HcmV?d00001 diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index e6cd47b91d..58095940f3 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,46 +1,44 @@ IF (TD_WEBSOCKET) MESSAGE("${Green} use libtaos-ws${ColourReset}") - IF (TD_LINUX) - IF (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/libtaosws.so" OR "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/libtaosws.so" IS_NEWER_THAN "${CMAKE_SOURCE_DIR}/.git/modules/tools/taosws-rs/FETCH_HEAD") - include(ExternalProject) - ExternalProject_Add(taosws-rs - PREFIX "taosws-rs" - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs - BUILD_ALWAYS off - DEPENDS taos - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" - PATCH_COMMAND - COMMAND git clean -f -d - BUILD_COMMAND - COMMAND cargo build --release -p taos-ws-sys - COMMAND ./taos-ws-sys/ci/package.sh - INSTALL_COMMAND - COMMAND cmake -E copy target/libtaosws/libtaosws.so ${CMAKE_BINARY_DIR}/build/lib - COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include - COMMAND cmake -E copy target/libtaosws/taosws.h ${CMAKE_BINARY_DIR}/build/include - ) - ELSE() - include(ExternalProject) - ExternalProject_Add(taosws-rs - PREFIX "taosws-rs" - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs - BUILD_ALWAYS on - DEPENDS taos - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" - PATCH_COMMAND - COMMAND git clean -f -d - BUILD_COMMAND - COMMAND cargo build --release -p taos-ws-sys - COMMAND ./taos-ws-sys/ci/package.sh - INSTALL_COMMAND - COMMAND cmake -E copy target/libtaosws/libtaosws.so ${CMAKE_BINARY_DIR}/build/lib - COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include - COMMAND cmake -E copy target/libtaosws/taosws.h ${CMAKE_BINARY_DIR}/build/include - ) - ENDIF () - ENDIF() + IF (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/libtaosws.so" OR "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/libtaosws.so" IS_NEWER_THAN "${CMAKE_SOURCE_DIR}/.git/modules/tools/taosws-rs/FETCH_HEAD") + include(ExternalProject) + ExternalProject_Add(taosws-rs + PREFIX "taosws-rs" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs + BUILD_ALWAYS off + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND cargo build --release -p taos-ws-sys + COMMAND ./taos-ws-sys/ci/package.sh + INSTALL_COMMAND + COMMAND cmake -E copy target/libtaosws/libtaosws.so ${CMAKE_BINARY_DIR}/build/lib + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include + COMMAND cmake -E copy target/libtaosws/taosws.h ${CMAKE_BINARY_DIR}/build/include + ) + ELSE() + include(ExternalProject) + ExternalProject_Add(taosws-rs + PREFIX "taosws-rs" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs + BUILD_ALWAYS on + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND cargo build --release -p taos-ws-sys + COMMAND ./taos-ws-sys/ci/package.sh + INSTALL_COMMAND + COMMAND cmake -E copy target/libtaosws/libtaosws.so ${CMAKE_BINARY_DIR}/build/lib + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include + COMMAND cmake -E copy target/libtaosws/taosws.h ${CMAKE_BINARY_DIR}/build/include + ) + ENDIF () ENDIF () IF (TD_TAOS_TOOLS) From 700039c798fd61b101c46c49146e723ca4366dd0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Jul 2022 11:23:16 +0800 Subject: [PATCH 11/27] other: add ignore file type. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d5c7f763cf..d7fcb019ae 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ mac/ .mypy_cache *.tmp *.swp +*.swo *.orig src/connector/nodejs/node_modules/ src/connector/nodejs/out/ From 5eb32ea2a37955f44aea78da7c989a21de5cb2df Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 25 Jul 2022 12:01:44 +0800 Subject: [PATCH 12/27] test:split case for timeout --- .../system-test/7-tmq/tmqDropNtb-snapshot0.py | 225 ++++++++++++++++++ .../system-test/7-tmq/tmqDropNtb-snapshot1.py | 225 ++++++++++++++++++ tests/system-test/fulltest.sh | 3 +- 3 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/7-tmq/tmqDropNtb-snapshot0.py create mode 100644 tests/system-test/7-tmq/tmqDropNtb-snapshot1.py diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py new file mode 100644 index 0000000000..650d918828 --- /dev/null +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py @@ -0,0 +1,225 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.snapshot = 0 + self.vgroups = 4 + self.ctbNum = 1000 + self.rowsPerTbl = 10 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + # drop some ntbs + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ntb', + 'ctbStartIdx': 0, + 'ctbNum': 1000, + 'rowsPerTbl': 100, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'endTs': 0, + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdLog.info("start create database....") + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("start create normal tables....") + tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) + tdLog.info("start insert data into normal tables....") + tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) + + tdLog.info("create topics from database") + topicFromDb = 'topic_dbt' + tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName'])) + + if self.snapshot == 0: + consumerId = 0 + elif self.snapshot == 1: + consumerId = 1 + + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) + topicList = topicFromDb + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("drop some ntables") + # drop 1/4 ctbls from half offset + paraDict["ctbStartIdx"] = paraDict["ctbStartIdx"] + int(paraDict["ctbNum"] * 1 / 2) + paraDict["ctbNum"] = int(paraDict["ctbNum"] / 4) + tmqCom.drop_ctable(tdSql, dbname=paraDict['dbName'], count=paraDict["ctbNum"], default_ctbname_prefix=paraDict["ctbPrefix"], ctbStartIdx=paraDict["ctbStartIdx"]) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + + if not ((totalConsumeRows >= expectrowcnt * 3/4) and (totalConsumeRows < expectrowcnt)): + tdLog.exit("tmq consume rows error with snapshot = 0!") + + tdLog.info("wait subscriptions exit ....") + tmqCom.waitSubscriptionExit(tdSql, topicFromDb) + + tdSql.query("drop topic %s"%topicFromDb) + tdLog.info("success dorp topic: %s"%topicFromDb) + tdLog.printNoPrefix("======== test case 1 end ...... ") + + + + # drop some ntbs and create some new ntbs + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ntb', + 'ctbStartIdx': 0, + 'ctbNum': 1000, + 'rowsPerTbl': 100, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'endTs': 0, + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdLog.info("start create database....") + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("start create normal tables....") + tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) + tdLog.info("start insert data into normal tables....") + tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) + + tdLog.info("create topics from database") + topicFromDb = 'topic_dbt' + tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName'])) + + if self.snapshot == 0: + consumerId = 2 + elif self.snapshot == 1: + consumerId = 3 + + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + topicList = topicFromDb + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("drop some ntables") + # drop 1/4 ctbls from half offset + paraDict["ctbStartIdx"] = paraDict["ctbStartIdx"] + int(paraDict["ctbNum"] * 1 / 2) + paraDict["ctbNum"] = int(paraDict["ctbNum"] / 4) + tmqCom.drop_ctable(tdSql, dbname=paraDict['dbName'], count=paraDict["ctbNum"], default_ctbname_prefix=paraDict["ctbPrefix"], ctbStartIdx=paraDict["ctbStartIdx"]) + + tdLog.info("start create some new normal tables....") + paraDict["ctbPrefix"] = 'newCtb' + paraDict["ctbNum"] = self.ctbNum + tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) + tdLog.info("start insert data into these new normal tables....") + tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + + if not ((totalConsumeRows >= expectrowcnt / 2 * (1 + 3/4)) and (totalConsumeRows < expectrowcnt)): + tdLog.exit("tmq consume rows error with snapshot = 0!") + + tdLog.info("wait subscriptions exit ....") + tmqCom.waitSubscriptionExit(tdSql, topicFromDb) + + tdSql.query("drop topic %s"%topicFromDb) + tdLog.info("success dorp topic: %s"%topicFromDb) + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + tdLog.printNoPrefix("=============================================") + tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") + self.snapshot = 0 + self.tmqCase1() + self.tmqCase2() + + # tdLog.printNoPrefix("====================================================================") + # tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + # self.snapshot = 1 + # self.tmqCase1() + # self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py new file mode 100644 index 0000000000..b23f422585 --- /dev/null +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py @@ -0,0 +1,225 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.snapshot = 0 + self.vgroups = 4 + self.ctbNum = 1000 + self.rowsPerTbl = 10 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + # drop some ntbs + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ntb', + 'ctbStartIdx': 0, + 'ctbNum': 1000, + 'rowsPerTbl': 100, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'endTs': 0, + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdLog.info("start create database....") + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("start create normal tables....") + tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) + tdLog.info("start insert data into normal tables....") + tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) + + tdLog.info("create topics from database") + topicFromDb = 'topic_dbt' + tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName'])) + + if self.snapshot == 0: + consumerId = 0 + elif self.snapshot == 1: + consumerId = 1 + + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) + topicList = topicFromDb + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("drop some ntables") + # drop 1/4 ctbls from half offset + paraDict["ctbStartIdx"] = paraDict["ctbStartIdx"] + int(paraDict["ctbNum"] * 1 / 2) + paraDict["ctbNum"] = int(paraDict["ctbNum"] / 4) + tmqCom.drop_ctable(tdSql, dbname=paraDict['dbName'], count=paraDict["ctbNum"], default_ctbname_prefix=paraDict["ctbPrefix"], ctbStartIdx=paraDict["ctbStartIdx"]) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + + if not ((totalConsumeRows >= expectrowcnt * 3/4) and (totalConsumeRows < expectrowcnt)): + tdLog.exit("tmq consume rows error with snapshot = 0!") + + tdLog.info("wait subscriptions exit ....") + tmqCom.waitSubscriptionExit(tdSql, topicFromDb) + + tdSql.query("drop topic %s"%topicFromDb) + tdLog.info("success dorp topic: %s"%topicFromDb) + tdLog.printNoPrefix("======== test case 1 end ...... ") + + + + # drop some ntbs and create some new ntbs + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ntb', + 'ctbStartIdx': 0, + 'ctbNum': 1000, + 'rowsPerTbl': 100, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'endTs': 0, + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdLog.info("start create database....") + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("start create normal tables....") + tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) + tdLog.info("start insert data into normal tables....") + tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) + + tdLog.info("create topics from database") + topicFromDb = 'topic_dbt' + tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName'])) + + if self.snapshot == 0: + consumerId = 2 + elif self.snapshot == 1: + consumerId = 3 + + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + topicList = topicFromDb + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("drop some ntables") + # drop 1/4 ctbls from half offset + paraDict["ctbStartIdx"] = paraDict["ctbStartIdx"] + int(paraDict["ctbNum"] * 1 / 2) + paraDict["ctbNum"] = int(paraDict["ctbNum"] / 4) + tmqCom.drop_ctable(tdSql, dbname=paraDict['dbName'], count=paraDict["ctbNum"], default_ctbname_prefix=paraDict["ctbPrefix"], ctbStartIdx=paraDict["ctbStartIdx"]) + + tdLog.info("start create some new normal tables....") + paraDict["ctbPrefix"] = 'newCtb' + paraDict["ctbNum"] = self.ctbNum + tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) + tdLog.info("start insert data into these new normal tables....") + tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + + if not ((totalConsumeRows >= expectrowcnt / 2 * (1 + 3/4)) and (totalConsumeRows < expectrowcnt)): + tdLog.exit("tmq consume rows error with snapshot = 0!") + + tdLog.info("wait subscriptions exit ....") + tmqCom.waitSubscriptionExit(tdSql, topicFromDb) + + tdSql.query("drop topic %s"%topicFromDb) + tdLog.info("success dorp topic: %s"%topicFromDb) + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + # tdLog.printNoPrefix("=============================================") + # tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") + # self.snapshot = 0 + # self.tmqCase1() + # self.tmqCase2() + + tdLog.printNoPrefix("====================================================================") + tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + self.snapshot = 1 + self.tmqCase1() + self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index ba104ddf22..f074bd8850 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -219,7 +219,8 @@ python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py python3 ./test.py -f 7-tmq/tmqDropStb.py python3 ./test.py -f 7-tmq/tmqDropStbCtb.py -python3 ./test.py -f 7-tmq/tmqDropNtb.py +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py python3 ./test.py -f 7-tmq/tmqUdf.py python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py From d9917e2f67f1cc571241090bc41bab0835c31301 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Jul 2022 12:17:49 +0800 Subject: [PATCH 13/27] other: disable some unit test. --- source/client/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 0b259169dc..129e20e5de 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -51,6 +51,6 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) -#if(${BUILD_TEST}) +if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -#endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) \ No newline at end of file From 4d8fc705d8e91f559684ad037c5cc9338de4bb44 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Mon, 25 Jul 2022 09:50:44 +0530 Subject: [PATCH 14/27] Added 2 acceptable error codes to crash_gen tool --- tests/pytest/crash_gen/crash_gen_main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 8990c24305..d5ffc1b7c1 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1327,6 +1327,8 @@ class Task(): # TDengine 3.0 Error Codes: 0x0333, # Object is creating # TODO: this really is NOT an acceptable error + 0x0369, # Tag already exists + 0x0388, # Database not exist 0x03A0, # STable already exists 0x03A1, # STable [does] not exist 0x03AA, # Tag already exists From a1d71f8fa73181a9b813c260dbbf82bfc51dfa52 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 25 Jul 2022 13:28:31 +0800 Subject: [PATCH 15/27] ci(stream): recover stream ci --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a606311f3c..0c19e4a2fe 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -224,7 +224,7 @@ # ---- stream ./test.sh -f tsim/stream/basic0.sim -#./test.sh -f tsim/stream/basic1.sim +./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/drop_stream.sim ./test.sh -f tsim/stream/distributeInterval0.sim From cd53c7e5735abdccbc025bd1d181d7ad1282c6ab Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Mon, 25 Jul 2022 13:38:54 +0800 Subject: [PATCH 16/27] Delete tmqDropNtb.py test: split two cases for timeout --- tests/system-test/7-tmq/tmqDropNtb.py | 225 -------------------------- 1 file changed, 225 deletions(-) delete mode 100644 tests/system-test/7-tmq/tmqDropNtb.py diff --git a/tests/system-test/7-tmq/tmqDropNtb.py b/tests/system-test/7-tmq/tmqDropNtb.py deleted file mode 100644 index e1f5794ce2..0000000000 --- a/tests/system-test/7-tmq/tmqDropNtb.py +++ /dev/null @@ -1,225 +0,0 @@ - -import taos -import sys -import time -import socket -import os -import threading -from enum import Enum - -from util.log import * -from util.sql import * -from util.cases import * -from util.dnodes import * -sys.path.append("./7-tmq") -from tmqCommon import * - -class TDTestCase: - def __init__(self): - self.snapshot = 0 - self.vgroups = 4 - self.ctbNum = 100 - self.rowsPerTbl = 10 - - def init(self, conn, logSql): - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) - - # drop some ntbs - def tmqCase1(self): - tdLog.printNoPrefix("======== test case 1: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 4, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ntb', - 'ctbStartIdx': 0, - 'ctbNum': 100, - 'rowsPerTbl': 1000, - 'batchNum': 1000, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'endTs': 0, - 'pollDelay': 5, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - paraDict['snapshot'] = self.snapshot - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - tmqCom.initConsumerTable() - tdLog.info("start create database....") - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdLog.info("start create normal tables....") - tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) - tdLog.info("start insert data into normal tables....") - tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) - - tdLog.info("create topics from database") - topicFromDb = 'topic_dbt' - tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName'])) - - if self.snapshot == 0: - consumerId = 0 - elif self.snapshot == 1: - consumerId = 1 - - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) - topicList = topicFromDb - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1,\ - enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ - auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - - tmqCom.getStartConsumeNotifyFromTmqsim() - tdLog.info("drop some ntables") - # drop 1/4 ctbls from half offset - paraDict["ctbStartIdx"] = paraDict["ctbStartIdx"] + int(paraDict["ctbNum"] * 1 / 2) - paraDict["ctbNum"] = int(paraDict["ctbNum"] / 4) - tmqCom.drop_ctable(tdSql, dbname=paraDict['dbName'], count=paraDict["ctbNum"], default_ctbname_prefix=paraDict["ctbPrefix"], ctbStartIdx=paraDict["ctbStartIdx"]) - - tdLog.info("start to check consume result") - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - totalConsumeRows = 0 - for i in range(expectRows): - totalConsumeRows += resultList[i] - - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) - - if not ((totalConsumeRows >= expectrowcnt * 3/4) and (totalConsumeRows < expectrowcnt)): - tdLog.exit("tmq consume rows error with snapshot = 0!") - - tdLog.info("wait subscriptions exit ....") - tmqCom.waitSubscriptionExit(tdSql, topicFromDb) - - tdSql.query("drop topic %s"%topicFromDb) - tdLog.info("success dorp topic: %s"%topicFromDb) - tdLog.printNoPrefix("======== test case 1 end ...... ") - - - - # drop some ntbs and create some new ntbs - def tmqCase2(self): - tdLog.printNoPrefix("======== test case 2: ") - paraDict = {'dbName': 'dbt', - 'dropFlag': 1, - 'event': '', - 'vgroups': 4, - 'stbName': 'stb', - 'colPrefix': 'c', - 'tagPrefix': 't', - 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'TIMESTAMP', 'count':1}], - 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], - 'ctbPrefix': 'ntb', - 'ctbStartIdx': 0, - 'ctbNum': 100, - 'rowsPerTbl': 1000, - 'batchNum': 1000, - 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'endTs': 0, - 'pollDelay': 10, - 'showMsg': 1, - 'showRow': 1, - 'snapshot': 0} - paraDict['snapshot'] = self.snapshot - paraDict['vgroups'] = self.vgroups - paraDict['ctbNum'] = self.ctbNum - paraDict['rowsPerTbl'] = self.rowsPerTbl - - tmqCom.initConsumerTable() - tdLog.info("start create database....") - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) - tdLog.info("start create normal tables....") - tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) - tdLog.info("start insert data into normal tables....") - tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) - - tdLog.info("create topics from database") - topicFromDb = 'topic_dbt' - tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName'])) - - if self.snapshot == 0: - consumerId = 2 - elif self.snapshot == 1: - consumerId = 3 - - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) - topicList = topicFromDb - ifcheckdata = 1 - ifManualCommit = 1 - keyList = 'group.id:cgrp1,\ - enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ - auto.offset.reset:earliest' - tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - - tmqCom.getStartConsumeNotifyFromTmqsim() - tdLog.info("drop some ntables") - # drop 1/4 ctbls from half offset - paraDict["ctbStartIdx"] = paraDict["ctbStartIdx"] + int(paraDict["ctbNum"] * 1 / 2) - paraDict["ctbNum"] = int(paraDict["ctbNum"] / 4) - tmqCom.drop_ctable(tdSql, dbname=paraDict['dbName'], count=paraDict["ctbNum"], default_ctbname_prefix=paraDict["ctbPrefix"], ctbStartIdx=paraDict["ctbStartIdx"]) - - tdLog.info("start create some new normal tables....") - paraDict["ctbPrefix"] = 'newCtb' - paraDict["ctbNum"] = self.ctbNum - tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) - tdLog.info("start insert data into these new normal tables....") - tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) - - tdLog.info("start to check consume result") - expectRows = 1 - resultList = tmqCom.selectConsumeResult(expectRows) - totalConsumeRows = 0 - for i in range(expectRows): - totalConsumeRows += resultList[i] - - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) - - if not ((totalConsumeRows >= expectrowcnt / 2 * (1 + 3/4)) and (totalConsumeRows < expectrowcnt)): - tdLog.exit("tmq consume rows error with snapshot = 0!") - - tdLog.info("wait subscriptions exit ....") - tmqCom.waitSubscriptionExit(tdSql, topicFromDb) - - tdSql.query("drop topic %s"%topicFromDb) - tdLog.info("success dorp topic: %s"%topicFromDb) - tdLog.printNoPrefix("======== test case 2 end ...... ") - - def run(self): - tdLog.printNoPrefix("=============================================") - tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") - self.snapshot = 0 - # self.tmqCase1() - self.tmqCase2() - - tdLog.printNoPrefix("====================================================================") - tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") - self.snapshot = 1 - # self.tmqCase1() - self.tmqCase2() - - def stop(self): - tdSql.close() - tdLog.success(f"{__file__} successfully executed") - -event = threading.Event() - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) From 2fcfa22520826752e919ff2e17a64b66fd06db30 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 25 Jul 2022 14:10:46 +0800 Subject: [PATCH 17/27] fix: fix out of block in data sink issue --- source/libs/qworker/src/qworker.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ebccb7950c..d77e42388b 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -66,7 +66,7 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { return TSDB_CODE_SUCCESS; } -int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { +int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { int32_t code = 0; bool qcontinue = true; SSDataBlock *pRes = NULL; @@ -104,8 +104,8 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { QW_ERR_RET(qwHandleTaskComplete(QW_FPARAMS(), ctx)); - if (queryEnd) { - *queryEnd = true; + if (queryStop) { + *queryStop = true; } break; @@ -125,6 +125,10 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", rows, qcontinue); if (!qcontinue) { + if (queryStop) { + *queryStop = true; + } + break; } @@ -566,7 +570,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { SQWPhaseInput input = {0}; void *rsp = NULL; int32_t dataLen = 0; - bool queryEnd = false; + bool queryStop = false; do { QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, NULL)); @@ -576,7 +580,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { atomic_store_8((int8_t *)&ctx->queryInQueue, 0); atomic_store_8((int8_t *)&ctx->queryContinue, 0); - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryEnd)); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryStop)); if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { SOutputData sOutput = {0}; @@ -627,7 +631,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } QW_LOCK(QW_WRITE, &ctx->lock); - if (queryEnd || code || 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) { + if (queryStop || code || 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) { // Note: query is not running anymore QW_SET_PHASE(ctx, 0); QW_UNLOCK(QW_WRITE, &ctx->lock); From baca08d05dfc1735f3c5f980912e854c601a9f21 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Jul 2022 14:15:49 +0800 Subject: [PATCH 18/27] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 5 + source/libs/executor/src/executorimpl.c | 580 +------------------- source/libs/executor/src/projectoperator.c | 585 +++++++++++++++++++++ 3 files changed, 612 insertions(+), 558 deletions(-) create mode 100644 source/libs/executor/src/projectoperator.c diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f4d0eb3b5e..bde7a94c53 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -351,6 +351,11 @@ typedef enum EStreamScanMode { STREAM_SCAN_FROM_DATAREADER_RANGE, } EStreamScanMode; +enum { + PROJECT_RETRIEVE_CONTINUE = 0x1, + PROJECT_RETRIEVE_DONE = 0x2, +}; + typedef struct SCatchSupporter { SHashObj* pWindowHashTable; // quick locate the window object for each window SDiskbasedBuf* pDataBuf; // buffer based on blocked-wised disk file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 906952be9c..d9cc2dbeb2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -42,11 +42,6 @@ #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) -enum { - PROJECT_RETRIEVE_CONTINUE = 0x1, - PROJECT_RETRIEVE_DONE = 0x2, -}; - #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = taosRand(); @@ -575,6 +570,26 @@ static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, S int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t numOfOutput, SArray* pPseudoList) { setPseudoOutputColInfo(pResult, pCtx, pPseudoList); + + if (pSrcBlock == NULL) { + for (int32_t k = 0; k < numOfOutput; ++k) { + int32_t outputSlotId = pExpr[k].base.resSchema.slotId; + + ASSERT(pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE); + SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); + + int32_t type = pExpr[k].base.pParam[0].param.nType; + if (TSDB_DATA_TYPE_NULL == type) { + colDataAppendNNULL(pColInfoData, 0, 1); + } else { + colDataAppend(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false); + } + } + + pResult->info.rows = 1; + return TSDB_CODE_SUCCESS; + } + pResult->info.groupId = pSrcBlock->info.groupId; // if the source equals to the destination, it is to create a new column as the result of scalar @@ -1243,52 +1258,6 @@ void initResultRow(SResultRow* pResultRow) { // pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow)); } -/* - * The start of each column SResultRowEntryInfo is denote by RowCellInfoOffset. - * Note that in case of top/bottom query, the whole multiple rows of result is treated as only one row of results. - * +------------+-----------------result column 1------------+------------------result column 2-----------+ - * | SResultRow | SResultRowEntryInfo | intermediate buffer1 | SResultRowEntryInfo | intermediate buffer 2| - * +------------+--------------------------------------------+--------------------------------------------+ - * offset[0] offset[1] offset[2] - */ -// TODO refactor: some function move away -void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, - int32_t numOfExprs) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - int32_t* rowEntryInfoOffset = pOperator->exprSupp.rowEntryInfoOffset; - - SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; - initResultRowInfo(pResultRowInfo); - - int64_t tid = 0; - int64_t groupId = 0; - SResultRow* pRow = doSetResultOutBufByKey(pSup->pResultBuf, pResultRowInfo, (char*)&tid, sizeof(tid), true, groupId, - pTaskInfo, false, pSup); - - for (int32_t i = 0; i < numOfExprs; ++i) { - struct SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, rowEntryInfoOffset); - cleanupResultRowEntry(pEntry); - - pCtx[i].resultInfo = pEntry; - pCtx[i].scanFlag = stage; - } - - initCtxOutputBuffer(pCtx, numOfExprs); -} - -void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { - for (int32_t j = 0; j < size; ++j) { - struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); - if (isRowEntryInitialized(pResInfo) || fmIsPseudoColumnFunc(pCtx[j].functionId) || pCtx[j].functionId == -1 || - fmIsScalarFunc(pCtx[j].functionId)) { - continue; - } - - pCtx[j].fpSet.init(&pCtx[j], pCtx[j].resultInfo); - } -} - void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { if (status == TASK_NOT_COMPLETED) { pTaskInfo->status = status; @@ -2805,73 +2774,6 @@ static int32_t initGroupCol(SExprInfo* pExprInfo, int32_t numOfCols, SArray* pGr return TSDB_CODE_SUCCESS; } -SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, - int32_t num, SArray* pSortInfo, SArray* pGroupInfo, - SExecTaskInfo* pTaskInfo) { - SSortedMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortedMergeOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, num); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - initResultRowInfo(&pInfo->binfo.resultRowInfo); - - if (pOperator->exprSupp.pCtx == NULL || pInfo->binfo.pRes == NULL) { - goto _error; - } - - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - code = doInitAggInfoSup(&pInfo->aggSup, pOperator->exprSupp.pCtx, num, keyBufSize, pTaskInfo->id.str); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, num); - code = initGroupCol(pExprInfo, num, pGroupInfo, pInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - // pInfo->resultRowFactor = (int32_t)(getRowNumForMultioutput(pRuntimeEnv->pQueryAttr, - // pRuntimeEnv->pQueryAttr->topBotQuery, false)); - pInfo->sortBufSize = 1024 * 16; // 1MB - pInfo->bufPageSize = 1024; - pInfo->pSortInfo = pSortInfo; - - pOperator->resultInfo.capacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, pInfo->bufPageSize); - - pOperator->name = "SortedMerge"; - // pOperator->operatorType = OP_SortedMerge; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSortedMerge, NULL, NULL, destroySortedMergeOperatorInfo, - NULL, NULL, NULL); - code = appendDownstream(pOperator, downstream, numOfDownstream); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - if (pInfo != NULL) { - destroySortedMergeOperatorInfo(pInfo, num); - } - - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; -} - int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag) { // todo add more information about exchange operation int32_t type = pOperator->operatorType; @@ -3274,168 +3176,6 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa } } -static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { - SProjectOperatorInfo* pProjectInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pProjectInfo->binfo; - - SExprSupp* pSup = &pOperator->exprSupp; - SSDataBlock* pRes = pInfo->pRes; - SSDataBlock* pFinalRes = pProjectInfo->pFinalRes; - - blockDataCleanup(pFinalRes); - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pOperator->status == OP_EXEC_DONE) { - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { - pOperator->status = OP_OPENED; - return NULL; - } - return NULL; - } - - int64_t st = 0; - int32_t order = 0; - int32_t scanFlag = 0; - - if (pOperator->cost.openCost == 0) { - st = taosGetTimestampUs(); - } - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - SLimitInfo* pLimitInfo = &pProjectInfo->limitInfo; - - while(1) { - while (1) { - blockDataCleanup(pRes); - - // The downstream exec may change the value of the newgroup, so use a local variable instead. - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); - if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); - break; - } - - if (pBlock->info.type == STREAM_RETRIEVE) { - // for stream interval - return pBlock; - } - - if (pLimitInfo->remainGroupOffset > 0) { - if (pLimitInfo->currentGroupId == 0 || pLimitInfo->currentGroupId == pBlock->info.groupId) { // it is the first group - pLimitInfo->currentGroupId = pBlock->info.groupId; - continue; - } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { - // now it is the data from a new group - pLimitInfo->remainGroupOffset -= 1; - pLimitInfo->currentGroupId = pBlock->info.groupId; - - // ignore data block in current group - if (pLimitInfo->remainGroupOffset > 0) { - continue; - } - } - - // set current group id of the project operator - pLimitInfo->currentGroupId = pBlock->info.groupId; - } - - // remainGroupOffset == 0 - // here check for a new group data, we need to handle the data of the previous group. - if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { - pLimitInfo->numOfOutputGroups += 1; - if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - doSetOperatorCompleted(pOperator); - break; - } - - // reset the value for a new group data - // existing rows that belongs to previous group. - pLimitInfo->numOfOutputRows = 0; - pLimitInfo->remainOffset = pLimitInfo->limit.offset; - } - - // the pDataBlock are always the same one, no need to call this again - int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } - - setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, scanFlag, false); - blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); - - code = projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, - pProjectInfo->pPseudoColInfo); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } - - // set current group id - pLimitInfo->currentGroupId = pBlock->info.groupId; - - if (pLimitInfo->remainOffset >= pInfo->pRes->info.rows) { - pLimitInfo->remainOffset -= pInfo->pRes->info.rows; - blockDataCleanup(pInfo->pRes); - continue; - } else if (pLimitInfo->remainOffset < pInfo->pRes->info.rows && pLimitInfo->remainOffset > 0) { - blockDataTrimFirstNRows(pInfo->pRes, pLimitInfo->remainOffset); - pLimitInfo->remainOffset = 0; - } - - // check for the limitation in each group - if (pLimitInfo->limit.limit >= 0 && - pLimitInfo->numOfOutputRows + pInfo->pRes->info.rows >= pLimitInfo->limit.limit) { - int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); - blockDataKeepFirstNRows(pInfo->pRes, keepRows); - if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { - pOperator->status = OP_EXEC_DONE; - } - } - - pLimitInfo->numOfOutputRows += pInfo->pRes->info.rows; - break; - } - - if (pProjectInfo->mergeDataBlocks) { - if (pRes->info.rows > 0) { - pFinalRes->info.groupId = pRes->info.groupId; - pFinalRes->info.version = pRes->info.version; - - // continue merge data, ignore the group id - blockDataMerge(pFinalRes, pRes); - if (pFinalRes->info.rows + pRes->info.rows <= pOperator->resultInfo.threshold) { - continue; - } - } - - // do apply filter - doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL); - if (pFinalRes->info.rows > 0 || pRes->info.rows == 0) { - break; - } - } else { - // do apply filter - if (pRes->info.rows > 0) { - doFilter(pProjectInfo->pFilterNode, pRes, NULL); - if (pRes->info.rows == 0) { - continue; - } - } - - // no results generated - break; - } - } - - SSDataBlock* p = pProjectInfo->mergeDataBlocks ? pFinalRes : pRes; - pOperator->resultInfo.totalRows += p->info.rows; - - if (pOperator->cost.openCost == 0) { - pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - } - - return (p->info.rows > 0) ? p : NULL; -} - static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; @@ -3816,30 +3556,6 @@ void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(param); } -static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { - if (NULL == param) { - return; - } - SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*)param; - cleanupBasicInfo(&pInfo->binfo); - cleanupAggSup(&pInfo->aggSup); - taosArrayDestroy(pInfo->pPseudoColInfo); - - blockDataDestroy(pInfo->pFinalRes); - taosMemoryFreeClear(param); -} - -static void destroyIndefinitOperatorInfo(void* param, int32_t numOfOutput) { - SIndefOperatorInfo* pInfo = (SIndefOperatorInfo*)param; - cleanupBasicInfo(&pInfo->binfo); - - taosArrayDestroy(pInfo->pPseudoColInfo); - cleanupAggSup(&pInfo->aggSup); - cleanupExprSupp(&pInfo->scalarSup); - - taosMemoryFreeClear(param); -} - void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { SExchangeInfo* pExInfo = (SExchangeInfo*)param; taosRemoveRef(exchangeObjRefPool, pExInfo->self); @@ -3859,259 +3575,6 @@ void doDestroyExchangeOperatorInfo(void* param) { taosMemoryFreeClear(param); } -static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) { - SArray* pList = taosArrayInit(4, sizeof(int32_t)); - for (int32_t i = 0; i < numOfCols; ++i) { - if (fmIsPseudoColumnFunc(pCtx[i].functionId)) { - taosArrayPush(pList, &i); - } - } - - return pList; -} - -SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, - SExecTaskInfo* pTaskInfo) { - SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pProjPhyNode->pProjections, NULL, &numOfCols); - - SSDataBlock* pResBlock = createResDataBlock(pProjPhyNode->node.pOutputDataBlockDesc); - initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo); - - pInfo->binfo.pRes = pResBlock; - pInfo->pFinalRes = createOneDataBlock(pResBlock, false); - pInfo->pFilterNode = pProjPhyNode->node.pConditions; - pInfo->mergeDataBlocks = pProjPhyNode->mergeDataBlock; - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - // Make sure the size of SSDataBlock will never exceed the size of 2MB. - int32_t TWOMB = 2 * 1024 * 1024; - if (numOfRows * pResBlock->info.rowSize > TWOMB) { - numOfRows = TWOMB / pResBlock->info.rowSize; - } - initResultSizeInfo(&pOperator->resultInfo, numOfRows); - - initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); - initBasicInfo(&pInfo->binfo, pResBlock); - setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfCols); - - pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pOperator->exprSupp.pCtx, numOfCols); - pOperator->name = "ProjectOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, - destroyProjectOperatorInfo, NULL, NULL, NULL); - - int32_t code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - return NULL; -} - -static void doHandleDataBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, SOperatorInfo* downstream, - SExecTaskInfo* pTaskInfo) { - int32_t order = 0; - int32_t scanFlag = 0; - - SIndefOperatorInfo* pIndefInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pIndefInfo->binfo; - SExprSupp* pSup = &pOperator->exprSupp; - - // the pDataBlock are always the same one, no need to call this again - int32_t code = getTableScanInfo(downstream, &order, &scanFlag); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } - - // there is an scalar expression that needs to be calculated before apply the group aggregation. - SExprSupp* pScalarSup = &pIndefInfo->scalarSup; - if (pScalarSup->pExprInfo != NULL) { - code = projectApplyFunctions(pScalarSup->pExprInfo, pBlock, pBlock, pScalarSup->pCtx, pScalarSup->numOfExprs, - pIndefInfo->pPseudoColInfo); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } - } - - setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, scanFlag, false); - blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); - - code = projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, - pIndefInfo->pPseudoColInfo); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } -} - -static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { - SIndefOperatorInfo* pIndefInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pIndefInfo->binfo; - SExprSupp* pSup = &pOperator->exprSupp; - - SSDataBlock* pRes = pInfo->pRes; - blockDataCleanup(pRes); - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - int64_t st = 0; - - if (pOperator->cost.openCost == 0) { - st = taosGetTimestampUs(); - } - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - while (1) { - // here we need to handle the existsed group results - if (pIndefInfo->pNextGroupRes != NULL) { // todo extract method - for (int32_t k = 0; k < pSup->numOfExprs; ++k) { - SqlFunctionCtx* pCtx = &pSup->pCtx[k]; - - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - pResInfo->initialized = false; - pCtx->pOutput = NULL; - } - - doHandleDataBlock(pOperator, pIndefInfo->pNextGroupRes, downstream, pTaskInfo); - pIndefInfo->pNextGroupRes = NULL; - } - - if (pInfo->pRes->info.rows < pOperator->resultInfo.threshold) { - while (1) { - // The downstream exec may change the value of the newgroup, so use a local variable instead. - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); - if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); - break; - } - - if (pIndefInfo->groupId == 0 && pBlock->info.groupId != 0) { - pIndefInfo->groupId = pBlock->info.groupId; // this is the initial group result - } else { - if (pIndefInfo->groupId != pBlock->info.groupId) { // reset output buffer and computing status - pIndefInfo->groupId = pBlock->info.groupId; - pIndefInfo->pNextGroupRes = pBlock; - break; - } - } - - doHandleDataBlock(pOperator, pBlock, downstream, pTaskInfo); - if (pInfo->pRes->info.rows >= pOperator->resultInfo.threshold) { - break; - } - } - } - - doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL); - size_t rows = pInfo->pRes->info.rows; - if (rows > 0 || pOperator->status == OP_EXEC_DONE) { - break; - } else { - blockDataCleanup(pInfo->pRes); - } - } - - size_t rows = pInfo->pRes->info.rows; - pOperator->resultInfo.totalRows += rows; - - if (pOperator->cost.openCost == 0) { - pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - } - - return (rows > 0) ? pInfo->pRes : NULL; -} - -SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, - SExecTaskInfo* pTaskInfo) { - SIndefOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIndefOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - SExprSupp* pSup = &pOperator->exprSupp; - - SIndefRowsFuncPhysiNode* pPhyNode = (SIndefRowsFuncPhysiNode*)pNode; - - int32_t numOfExpr = 0; - SExprInfo* pExprInfo = createExprInfo(pPhyNode->pFuncs, NULL, &numOfExpr); - - if (pPhyNode->pExprs != NULL) { - int32_t num = 0; - SExprInfo* pSExpr = createExprInfo(pPhyNode->pExprs, NULL, &num); - int32_t code = initExprSupp(&pInfo->scalarSup, pSExpr, num); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } - - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->node.pOutputDataBlockDesc); - - int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - - // Make sure the size of SSDataBlock will never exceed the size of 2MB. - int32_t TWOMB = 2 * 1024 * 1024; - if (numOfRows * pResBlock->info.rowSize > TWOMB) { - numOfRows = TWOMB / pResBlock->info.rowSize; - } - - initResultSizeInfo(&pOperator->resultInfo, numOfRows); - - initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str); - initBasicInfo(&pInfo->binfo, pResBlock); - - setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr); - - pInfo->binfo.pRes = pResBlock; - pInfo->pCondition = pPhyNode->node.pConditions; - pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); - - pOperator->name = "IndefinitOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, NULL, - destroyIndefinitOperatorInfo, NULL, NULL, NULL); - - int32_t code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - taosMemoryFree(pInfo); - taosMemoryFree(pOperator); - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - return NULL; -} - static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SNodeListNode* pValNode, STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pValNode); @@ -4504,7 +3967,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; - int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = terrno; @@ -4555,6 +4017,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } return createLastrowScanOperator(pScanNode, pHandle, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { + return createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pPhyNode, pTaskInfo); } else { ASSERT(0); } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c new file mode 100644 index 0000000000..87ba0006e1 --- /dev/null +++ b/source/libs/executor/src/projectoperator.c @@ -0,0 +1,585 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "executorimpl.h" +#include "functionMgt.h" + +static SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator); +static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator); +static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator); +static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols); +static void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, + int32_t numOfExprs); + +static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { + if (NULL == param) { + return; + } + + SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*)param; + cleanupBasicInfo(&pInfo->binfo); + cleanupAggSup(&pInfo->aggSup); + taosArrayDestroy(pInfo->pPseudoColInfo); + + blockDataDestroy(pInfo->pFinalRes); + taosMemoryFreeClear(param); +} + +static void destroyIndefinitOperatorInfo(void* param, int32_t numOfOutput) { + SIndefOperatorInfo* pInfo = (SIndefOperatorInfo*)param; + cleanupBasicInfo(&pInfo->binfo); + + taosArrayDestroy(pInfo->pPseudoColInfo); + cleanupAggSup(&pInfo->aggSup); + cleanupExprSupp(&pInfo->scalarSup); + + taosMemoryFreeClear(param); +} + +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, + SExecTaskInfo* pTaskInfo) { + SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pProjPhyNode->pProjections, NULL, &numOfCols); + + SSDataBlock* pResBlock = createResDataBlock(pProjPhyNode->node.pOutputDataBlockDesc); + initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo); + + pInfo->binfo.pRes = pResBlock; + pInfo->pFinalRes = createOneDataBlock(pResBlock, false); + pInfo->pFilterNode = pProjPhyNode->node.pConditions; + pInfo->mergeDataBlocks = pProjPhyNode->mergeDataBlock; + + int32_t numOfRows = 4096; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + // Make sure the size of SSDataBlock will never exceed the size of 2MB. + int32_t TWOMB = 2 * 1024 * 1024; + if (numOfRows * pResBlock->info.rowSize > TWOMB) { + numOfRows = TWOMB / pResBlock->info.rowSize; + } + initResultSizeInfo(&pOperator->resultInfo, numOfRows); + + initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + initBasicInfo(&pInfo->binfo, pResBlock); + setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfCols); + + pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pOperator->exprSupp.pCtx, numOfCols); + pOperator->name = "ProjectOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, + destroyProjectOperatorInfo, NULL, NULL, NULL); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} + +static int32_t discardGroupDataBlock(SSDataBlock* pBlock, SLimitInfo* pLimitInfo) { + if (pLimitInfo->remainGroupOffset > 0) { + // it is the first group + if (pLimitInfo->currentGroupId == 0 || pLimitInfo->currentGroupId == pBlock->info.groupId) { + pLimitInfo->currentGroupId = pBlock->info.groupId; + return PROJECT_RETRIEVE_CONTINUE; + } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { + // now it is the data from a new group + pLimitInfo->remainGroupOffset -= 1; + pLimitInfo->currentGroupId = pBlock->info.groupId; + + // ignore data block in current group + if (pLimitInfo->remainGroupOffset > 0) { + return PROJECT_RETRIEVE_CONTINUE; + } + } + + // set current group id of the project operator + pLimitInfo->currentGroupId = pBlock->info.groupId; + } + + return PROJECT_RETRIEVE_DONE; +} + +static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, SOperatorInfo* pOperator) { + // remainGroupOffset == 0 + // here check for a new group data, we need to handle the data of the previous group. + ASSERT(pLimitInfo->remainGroupOffset == 0); + + if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { + pLimitInfo->numOfOutputGroups += 1; + if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { + doSetOperatorCompleted(pOperator); + return PROJECT_RETRIEVE_DONE; + } + + // reset the value for a new group data + // existing rows that belongs to previous group. + pLimitInfo->numOfOutputRows = 0; + pLimitInfo->remainOffset = pLimitInfo->limit.offset; + } + + return PROJECT_RETRIEVE_DONE; +} + +static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SSDataBlock* pBlock, SOperatorInfo* pOperator) { + // set current group id + pLimitInfo->currentGroupId = groupId; + + if (pLimitInfo->remainOffset >= pBlock->info.rows) { + pLimitInfo->remainOffset -= pBlock->info.rows; + blockDataCleanup(pBlock); + return PROJECT_RETRIEVE_CONTINUE; + } else if (pLimitInfo->remainOffset < pBlock->info.rows && pLimitInfo->remainOffset > 0) { + blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); + pLimitInfo->remainOffset = 0; + } + + // check for the limitation in each group + if (pLimitInfo->limit.limit >= 0 && + pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) { + int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); + blockDataKeepFirstNRows(pBlock, keepRows); + if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { + doSetOperatorCompleted(pOperator); + } + } + + pLimitInfo->numOfOutputRows += pBlock->info.rows; + return PROJECT_RETRIEVE_DONE; +} + +SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { + SProjectOperatorInfo* pProjectInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pProjectInfo->binfo; + + SExprSupp* pSup = &pOperator->exprSupp; + SSDataBlock* pRes = pInfo->pRes; + SSDataBlock* pFinalRes = pProjectInfo->pFinalRes; + + blockDataCleanup(pFinalRes); + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + if (pOperator->status == OP_EXEC_DONE) { + if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { + pOperator->status = OP_OPENED; + return NULL; + } + + return NULL; + } + + int64_t st = 0; + int32_t order = 0; + int32_t scanFlag = 0; + + if (pOperator->cost.openCost == 0) { + st = taosGetTimestampUs(); + } + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SLimitInfo* pLimitInfo = &pProjectInfo->limitInfo; + + if (downstream == NULL) { + return doGenerateSourceData(pOperator); + } + + while (1) { + while (1) { + blockDataCleanup(pRes); + + // The downstream exec may change the value of the newgroup, so use a local variable instead. + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + break; + } + + // for stream interval + if (pBlock->info.type == STREAM_RETRIEVE) { + return pBlock; + } + + int32_t status = discardGroupDataBlock(pBlock, pLimitInfo); + if (status == PROJECT_RETRIEVE_CONTINUE) { + continue; + } + + setInfoForNewGroup(pBlock, pLimitInfo, pOperator); + if (pOperator->status == OP_EXEC_DONE) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + int32_t code = getTableScanInfo(downstream, &order, &scanFlag); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, scanFlag, false); + blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); + + code = projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, + pProjectInfo->pPseudoColInfo); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + status = doIngroupLimitOffset(pLimitInfo, pBlock->info.groupId, pInfo->pRes, pOperator); + if (status == PROJECT_RETRIEVE_CONTINUE) { + continue; + } + + break; + } + + if (pProjectInfo->mergeDataBlocks) { + if (pRes->info.rows > 0) { + pFinalRes->info.groupId = pRes->info.groupId; + pFinalRes->info.version = pRes->info.version; + + // continue merge data, ignore the group id + blockDataMerge(pFinalRes, pRes); + if (pFinalRes->info.rows + pRes->info.rows <= pOperator->resultInfo.threshold) { + continue; + } + } + + // do apply filter + doFilter(pProjectInfo->pFilterNode, pFinalRes, NULL); + if (pFinalRes->info.rows > 0 || pRes->info.rows == 0) { + break; + } + } else { + // do apply filter + if (pRes->info.rows > 0) { + doFilter(pProjectInfo->pFilterNode, pRes, NULL); + if (pRes->info.rows == 0) { + continue; + } + } + + // no results generated + break; + } + } + + SSDataBlock* p = pProjectInfo->mergeDataBlocks ? pFinalRes : pRes; + pOperator->resultInfo.totalRows += p->info.rows; + + if (pOperator->cost.openCost == 0) { + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + } + + return (p->info.rows > 0) ? p : NULL; +} + +SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, + SExecTaskInfo* pTaskInfo) { + SIndefOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIndefOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SExprSupp* pSup = &pOperator->exprSupp; + + SIndefRowsFuncPhysiNode* pPhyNode = (SIndefRowsFuncPhysiNode*)pNode; + + int32_t numOfExpr = 0; + SExprInfo* pExprInfo = createExprInfo(pPhyNode->pFuncs, NULL, &numOfExpr); + + if (pPhyNode->pExprs != NULL) { + int32_t num = 0; + SExprInfo* pSExpr = createExprInfo(pPhyNode->pExprs, NULL, &num); + int32_t code = initExprSupp(&pInfo->scalarSup, pSExpr, num); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->node.pOutputDataBlockDesc); + + int32_t numOfRows = 4096; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + // Make sure the size of SSDataBlock will never exceed the size of 2MB. + int32_t TWOMB = 2 * 1024 * 1024; + if (numOfRows * pResBlock->info.rowSize > TWOMB) { + numOfRows = TWOMB / pResBlock->info.rowSize; + } + + initResultSizeInfo(&pOperator->resultInfo, numOfRows); + + initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str); + initBasicInfo(&pInfo->binfo, pResBlock); + + setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr); + + pInfo->binfo.pRes = pResBlock; + pInfo->pCondition = pPhyNode->node.pConditions; + pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); + + pOperator->name = "IndefinitOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, NULL, + destroyIndefinitOperatorInfo, NULL, NULL, NULL); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} + +static void doHandleDataBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, SOperatorInfo* downstream, + SExecTaskInfo* pTaskInfo) { + int32_t order = 0; + int32_t scanFlag = 0; + + SIndefOperatorInfo* pIndefInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pIndefInfo->binfo; + SExprSupp* pSup = &pOperator->exprSupp; + + // the pDataBlock are always the same one, no need to call this again + int32_t code = getTableScanInfo(downstream, &order, &scanFlag); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + // there is an scalar expression that needs to be calculated before apply the group aggregation. + SExprSupp* pScalarSup = &pIndefInfo->scalarSup; + if (pScalarSup->pExprInfo != NULL) { + code = projectApplyFunctions(pScalarSup->pExprInfo, pBlock, pBlock, pScalarSup->pCtx, pScalarSup->numOfExprs, + pIndefInfo->pPseudoColInfo); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + } + + setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, scanFlag, false); + blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); + + code = projectApplyFunctions(pSup->pExprInfo, pInfo->pRes, pBlock, pSup->pCtx, pSup->numOfExprs, + pIndefInfo->pPseudoColInfo); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } +} + +SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { + SIndefOperatorInfo* pIndefInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pIndefInfo->binfo; + SExprSupp* pSup = &pOperator->exprSupp; + + SSDataBlock* pRes = pInfo->pRes; + blockDataCleanup(pRes); + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + int64_t st = 0; + + if (pOperator->cost.openCost == 0) { + st = taosGetTimestampUs(); + } + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + while (1) { + // here we need to handle the existsed group results + if (pIndefInfo->pNextGroupRes != NULL) { // todo extract method + for (int32_t k = 0; k < pSup->numOfExprs; ++k) { + SqlFunctionCtx* pCtx = &pSup->pCtx[k]; + + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + pResInfo->initialized = false; + pCtx->pOutput = NULL; + } + + doHandleDataBlock(pOperator, pIndefInfo->pNextGroupRes, downstream, pTaskInfo); + pIndefInfo->pNextGroupRes = NULL; + } + + if (pInfo->pRes->info.rows < pOperator->resultInfo.threshold) { + while (1) { + // The downstream exec may change the value of the newgroup, so use a local variable instead. + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + break; + } + + if (pIndefInfo->groupId == 0 && pBlock->info.groupId != 0) { + pIndefInfo->groupId = pBlock->info.groupId; // this is the initial group result + } else { + if (pIndefInfo->groupId != pBlock->info.groupId) { // reset output buffer and computing status + pIndefInfo->groupId = pBlock->info.groupId; + pIndefInfo->pNextGroupRes = pBlock; + break; + } + } + + doHandleDataBlock(pOperator, pBlock, downstream, pTaskInfo); + if (pInfo->pRes->info.rows >= pOperator->resultInfo.threshold) { + break; + } + } + } + + doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL); + size_t rows = pInfo->pRes->info.rows; + if (rows > 0 || pOperator->status == OP_EXEC_DONE) { + break; + } else { + blockDataCleanup(pInfo->pRes); + } + } + + size_t rows = pInfo->pRes->info.rows; + pOperator->resultInfo.totalRows += rows; + + if (pOperator->cost.openCost == 0) { + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + } + + return (rows > 0) ? pInfo->pRes : NULL; +} + +void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { + for (int32_t j = 0; j < size; ++j) { + struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); + if (isRowEntryInitialized(pResInfo) || fmIsPseudoColumnFunc(pCtx[j].functionId) || pCtx[j].functionId == -1 || + fmIsScalarFunc(pCtx[j].functionId)) { + continue; + } + + pCtx[j].fpSet.init(&pCtx[j], pCtx[j].resultInfo); + } +} + +/* + * The start of each column SResultRowEntryInfo is denote by RowCellInfoOffset. + * Note that in case of top/bottom query, the whole multiple rows of result is treated as only one row of results. + * +------------+-----------------result column 1------------+------------------result column 2-----------+ + * | SResultRow | SResultRowEntryInfo | intermediate buffer1 | SResultRowEntryInfo | intermediate buffer 2| + * +------------+--------------------------------------------+--------------------------------------------+ + * offset[0] offset[1] offset[2] + */ +// TODO refactor: some function move away +void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, + int32_t numOfExprs) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; + int32_t* rowEntryInfoOffset = pOperator->exprSupp.rowEntryInfoOffset; + + SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; + initResultRowInfo(pResultRowInfo); + + int64_t tid = 0; + int64_t groupId = 0; + SResultRow* pRow = doSetResultOutBufByKey(pSup->pResultBuf, pResultRowInfo, (char*)&tid, sizeof(tid), true, groupId, + pTaskInfo, false, pSup); + + for (int32_t i = 0; i < numOfExprs; ++i) { + struct SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, rowEntryInfoOffset); + cleanupResultRowEntry(pEntry); + + pCtx[i].resultInfo = pEntry; + pCtx[i].scanFlag = stage; + } + + initCtxOutputBuffer(pCtx, numOfExprs); +} + +SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) { + SArray* pList = taosArrayInit(4, sizeof(int32_t)); + for (int32_t i = 0; i < numOfCols; ++i) { + if (fmIsPseudoColumnFunc(pCtx[i].functionId)) { + taosArrayPush(pList, &i); + } + } + + return pList; +} + +SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) { + SProjectOperatorInfo* pProjectInfo = pOperator->info; + + SExprSupp* pSup = &pOperator->exprSupp; + SSDataBlock* pRes = pProjectInfo->binfo.pRes; + + blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); + SExprInfo* pExpr = pSup->pExprInfo; + + int64_t st = taosGetTimestampUs(); + + for (int32_t k = 0; k < pSup->numOfExprs; ++k) { + int32_t outputSlotId = pExpr[k].base.resSchema.slotId; + + ASSERT(pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE); + SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, outputSlotId); + + int32_t type = pExpr[k].base.pParam[0].param.nType; + if (TSDB_DATA_TYPE_NULL == type) { + colDataAppendNNULL(pColInfoData, 0, 1); + } else { + colDataAppend(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false); + } + } + + pRes->info.rows = 1; + doFilter(pProjectInfo->pFilterNode, pRes, NULL); + + /*int32_t status = */doIngroupLimitOffset(&pProjectInfo->limitInfo, 0, pRes, pOperator); + + pOperator->resultInfo.totalRows += pRes->info.rows; + + doSetOperatorCompleted(pOperator); + if (pOperator->cost.openCost == 0) { + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + } + + return (pRes->info.rows > 0) ? pRes : NULL; +} \ No newline at end of file From a09a509df47cb7798c4ec6442ecb1561905616f2 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 25 Jul 2022 15:14:36 +0800 Subject: [PATCH 19/27] ci(stream): temporary disable basic1 --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0c19e4a2fe..a606311f3c 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -224,7 +224,7 @@ # ---- stream ./test.sh -f tsim/stream/basic0.sim -./test.sh -f tsim/stream/basic1.sim +#./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/drop_stream.sim ./test.sh -f tsim/stream/distributeInterval0.sim From 015193e81609fae5b75699f0dd9730644ac7ad89 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 25 Jul 2022 15:17:53 +0800 Subject: [PATCH 20/27] feat: super table order by primary key optimization --- include/libs/nodes/plannodes.h | 9 ++-- source/libs/command/src/explain.c | 15 +++--- source/libs/executor/inc/executorimpl.h | 45 +++++++++++++++++- source/libs/executor/src/executorimpl.c | 22 +++++---- source/libs/executor/src/joinoperator.c | 26 +++++----- source/libs/executor/src/scanoperator.c | 55 ++++------------------ source/libs/nodes/src/nodesCodeFuncs.c | 4 +- source/libs/nodes/src/nodesTraverseFuncs.c | 2 +- source/libs/nodes/src/nodesUtilFuncs.c | 4 +- source/libs/parser/src/parInsert.c | 14 +++--- source/libs/parser/src/parUtil.c | 2 +- source/libs/planner/src/planOptimizer.c | 35 ++++++++++---- source/libs/planner/src/planPhysiCreater.c | 4 +- source/libs/planner/src/planSpliter.c | 19 ++++---- source/util/src/terror.c | 2 +- 15 files changed, 147 insertions(+), 111 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index e382fa4efd..ba16acf7b0 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -104,6 +104,7 @@ typedef struct SJoinLogicNode { SNode* pMergeCondition; SNode* pOnConditions; bool isSingleTableJoin; + EOrder inputTsOrder; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -201,6 +202,7 @@ typedef struct SWindowLogicNode { int64_t watermark; int8_t igExpired; EWindowAlgorithm windowAlgo; + EOrder inputTsOrder; } SWindowLogicNode; typedef struct SFillLogicNode { @@ -356,15 +358,14 @@ typedef struct SInterpFuncPhysiNode { SNode* pTimeSeries; // SColumnNode } SInterpFuncPhysiNode; -typedef struct SJoinPhysiNode { +typedef struct SSortMergeJoinPhysiNode { SPhysiNode node; EJoinType joinType; SNode* pMergeCondition; SNode* pOnConditions; SNodeList* pTargets; -} SJoinPhysiNode; - -typedef SJoinPhysiNode SSortMergeJoinPhysiNode; + EOrder inputTsOrder; +} SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { SPhysiNode node; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 9ffdfc2289..266f96b41e 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -135,7 +135,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo break; } case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: { - SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; + SSortMergeJoinPhysiNode *pJoinNode = (SSortMergeJoinPhysiNode *)pNode; pPhysiChildren = pJoinNode->node.pChildren; break; } @@ -434,7 +434,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: { STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, - QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == pNode->type ? EXPLAIN_TBL_MERGE_SCAN_FORMAT : EXPLAIN_TBL_SCAN_FORMAT, + QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == pNode->type ? EXPLAIN_TBL_MERGE_SCAN_FORMAT + : EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { @@ -551,7 +552,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); if (pSTblScanNode->scan.pScanPseudoCols) { EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pSTblScanNode->scan.pScanPseudoCols->length); - EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); } EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); @@ -613,7 +614,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i break; } case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: { - SJoinPhysiNode *pJoinNode = (SJoinPhysiNode *)pNode; + SSortMergeJoinPhysiNode *pJoinNode = (SSortMergeJoinPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { @@ -1180,7 +1181,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); if (pDistScanNode->pScanPseudoCols) { EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pDistScanNode->pScanPseudoCols->length); - EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); } EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pDistScanNode->node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); @@ -1367,7 +1368,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pInterpNode->pFuncs->length); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); } - + EXPLAIN_ROW_APPEND(EXPLAIN_MODE_FORMAT, nodesGetFillModeString(pInterpNode->fillMode)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); @@ -1419,7 +1420,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } } break; - } + } default: qError("not supported physical node type %d", pNode->type); return TSDB_CODE_QRY_APP_ERROR; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1ad17bbc76..44394bb2f0 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -321,6 +321,49 @@ typedef struct STableScanInfo { int8_t noTable; } STableScanInfo; +typedef struct STableMergeScanInfo { + STableListInfo* tableListInfo; + int32_t tableStartIndex; + int32_t tableEndIndex; + bool hasGroupId; + uint64_t groupId; + SArray* dataReaders; // array of tsdbReaderT* + SReadHandle readHandle; + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + + SSDataBlock* pSortInputBlock; + int64_t startTs; // sort start time + SArray* sortSourceParams; + + SFileBlockLoadRecorder readRecorder; + int64_t numOfRows; + SScanInfo scanInfo; + int32_t scanTimes; + SNode* pFilterNode; // filter info, which is push down by optimizer + SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context + SResultRowInfo* pResultRowInfo; + int32_t* rowEntryInfoOffset; + SExprInfo* pExpr; + SSDataBlock* pResBlock; + SArray* pColMatchInfo; + int32_t numOfOutput; + + SExprInfo* pPseudoExpr; + int32_t numOfPseudoExpr; + SqlFunctionCtx* pPseudoCtx; + + SQueryTableDataCond cond; + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + int32_t dataBlockLoadFlag; + // if the upstream is an interval operator, the interval info is also kept here to get the time + // window to check if current data block needs to be loaded. + SInterval interval; + SSampleExecInfo sample; // sample execution info +} STableMergeScanInfo; + typedef struct STagScanInfo { SColumnInfo *pCols; SSDataBlock *pRes; @@ -881,7 +924,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SJoinPhysiNode* pJoinNode, +SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7bac828a53..7194b16a78 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1356,7 +1356,7 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); if (pColMatchInfo != NULL) { - for(int32_t i = 0; i < taosArrayGetSize(pColMatchInfo); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pColMatchInfo); ++i) { SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, i); if (pInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pInfo->targetSlotId); @@ -2885,11 +2885,16 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan *order = TSDB_ORDER_ASC; *scanFlag = MAIN_SCAN; return TSDB_CODE_SUCCESS; - } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN) { + } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = pOperator->info; *order = pTableScanInfo->cond.order; *scanFlag = pTableScanInfo->scanFlag; return TSDB_CODE_SUCCESS; + } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN) { + STableMergeScanInfo* pTableScanInfo = pOperator->info; + *order = pTableScanInfo->cond.order; + *scanFlag = pTableScanInfo->scanFlag; + return TSDB_CODE_SUCCESS; } else { if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { return TSDB_CODE_INVALID_PARA; @@ -3307,9 +3312,9 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } SOperatorInfo* downstream = pOperator->pDownstream[0]; - SLimitInfo* pLimitInfo = &pProjectInfo->limitInfo; + SLimitInfo* pLimitInfo = &pProjectInfo->limitInfo; - while(1) { + while (1) { while (1) { blockDataCleanup(pRes); @@ -3326,7 +3331,8 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } if (pLimitInfo->remainGroupOffset > 0) { - if (pLimitInfo->currentGroupId == 0 || pLimitInfo->currentGroupId == pBlock->info.groupId) { // it is the first group + if (pLimitInfo->currentGroupId == 0 || + pLimitInfo->currentGroupId == pBlock->info.groupId) { // it is the first group pLimitInfo->currentGroupId = pBlock->info.groupId; continue; } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { @@ -4265,7 +4271,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { } // this the tags and pseudo function columns, we only keep the tag columns - for(int32_t i = 0; i < numOfTags; ++i) { + for (int32_t i = 0; i < numOfTags; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanPseudoCols, i); int32_t type = nodeType(pNode->pExpr); @@ -4381,7 +4387,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, int32_t groupNum = 0; for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); - int32_t code = getGroupIdFromTagsVal(pHandle->meta, info->uid, group, keyBuf, &info->groupId); + int32_t code = getGroupIdFromTagsVal(pHandle->meta, info->uid, group, keyBuf, &info->groupId); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4701,7 +4707,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE == type) { pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) { - pOptr = createMergeJoinOperatorInfo(ops, size, (SJoinPhysiNode*)pPhyNode, pTaskInfo); + pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { pOptr = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) { diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 2e6c9bd351..f26b2f4f0a 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -28,30 +28,30 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator); static void destroyMergeJoinOperator(void* param, int32_t numOfOutput); static void extractTimeCondition(SJoinOperatorInfo* Info, SLogicConditionNode* pLogicConditionNode); -SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SJoinPhysiNode* pJoinNode, - SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, + SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pOperator == NULL || pInfo == NULL) { goto _error; } - SSDataBlock* pResBlock = createResDataBlock(pJoinNode->node.pOutputDataBlockDesc); + SSDataBlock* pResBlock = createResDataBlock(pJoinNode->node.pOutputDataBlockDesc); - int32_t numOfCols = 0; + int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &numOfCols); initResultSizeInfo(&pOperator->resultInfo, 4096); - pInfo->pRes = pResBlock; - pOperator->name = "MergeJoinOperator"; + pInfo->pRes = pResBlock; + pOperator->name = "MergeJoinOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->exprSupp.pExprInfo = pExprInfo; + pOperator->exprSupp.numOfExprs = numOfCols; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; SNode* pMergeCondition = pJoinNode->pMergeCondition; if (nodeType(pMergeCondition) == QUERY_NODE_OPERATOR) { @@ -104,7 +104,7 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { void destroyMergeJoinOperator(void* param, int32_t numOfOutput) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; nodesDestroyNode(pJoinOperator->pCondAfterMerge); - + taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a9d03aebbe..539ef18d87 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -274,7 +274,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); } else { - qDebug("%s data block filter out, elapsed time:%"PRId64, GET_TASKID(pTaskInfo), (et - st)); + qDebug("%s data block filter out, elapsed time:%" PRId64, GET_TASKID(pTaskInfo), (et - st)); } return TSDB_CODE_SUCCESS; @@ -1838,11 +1838,14 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int8_t tagType = smr.me.stbEntry.schemaTag.pSchema[i].type; pColInfoData = taosArrayGet(p->pDataBlock, 4); char tagTypeStr[VARSTR_HEADER_SIZE + 32]; - int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); + int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); if (tagType == TSDB_DATA_TYPE_VARCHAR) { - tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", (int32_t)(smr.me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); + tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", + (int32_t)(smr.me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); } else if (tagType == TSDB_DATA_TYPE_NCHAR) { - tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", (int32_t)((smr.me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + tagTypeLen += + sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", + (int32_t)((smr.me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(tagTypeStr, tagTypeLen); colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); @@ -2527,49 +2530,6 @@ _error: return NULL; } -typedef struct STableMergeScanInfo { - STableListInfo* tableListInfo; - int32_t tableStartIndex; - int32_t tableEndIndex; - bool hasGroupId; - uint64_t groupId; - SArray* dataReaders; // array of tsdbReaderT* - SReadHandle readHandle; - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - - SSDataBlock* pSortInputBlock; - int64_t startTs; // sort start time - SArray* sortSourceParams; - - SFileBlockLoadRecorder readRecorder; - int64_t numOfRows; - SScanInfo scanInfo; - int32_t scanTimes; - SNode* pFilterNode; // filter info, which is push down by optimizer - SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context - SResultRowInfo* pResultRowInfo; - int32_t* rowEntryInfoOffset; - SExprInfo* pExpr; - SSDataBlock* pResBlock; - SArray* pColMatchInfo; - int32_t numOfOutput; - - SExprInfo* pPseudoExpr; - int32_t numOfPseudoExpr; - SqlFunctionCtx* pPseudoCtx; - - SQueryTableDataCond cond; - int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan - int32_t dataBlockLoadFlag; - // if the upstream is an interval operator, the interval info is also kept here to get the time - // window to check if current data block needs to be loaded. - SInterval interval; - SSampleExecInfo sample; // sample execution info -} STableMergeScanInfo; - int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idStr) { @@ -2975,6 +2935,7 @@ void destroyTableMergeScanOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pTableScanInfo->pSortInfo); + taosMemoryFreeClear(pTableScanInfo->rowEntryInfoOffset); taosMemoryFreeClear(param); } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index eec4780293..186a51f000 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1717,7 +1717,7 @@ static const char* jkJoinPhysiPlanOnConditions = "OnConditions"; static const char* jkJoinPhysiPlanTargets = "Targets"; static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { - const SJoinPhysiNode* pNode = (const SJoinPhysiNode*)pObj; + const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { @@ -1737,7 +1737,7 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { } static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { - SJoinPhysiNode* pNode = (SJoinPhysiNode*)pObj; + SSortMergeJoinPhysiNode* pNode = (SSortMergeJoinPhysiNode*)pObj; int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index b12e3b14c7..77681af1bc 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -468,7 +468,7 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk break; } case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: { - SJoinPhysiNode* pJoin = (SJoinPhysiNode*)pNode; + SSortMergeJoinPhysiNode* pJoin = (SSortMergeJoinPhysiNode*)pNode; res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext); if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { res = walkPhysiPlan(pJoin->pMergeCondition, order, walker, pContext); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 23f0bb088d..3c6fbe409c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -287,7 +287,7 @@ SNode* nodesMakeNode(ENodeType type) { case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return makeNode(type, sizeof(SProjectPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: - return makeNode(type, sizeof(SJoinPhysiNode)); + return makeNode(type, sizeof(SSortMergeJoinPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: return makeNode(type, sizeof(SAggPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: @@ -883,7 +883,7 @@ void nodesDestroyNode(SNode* pNode) { break; } case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: { - SJoinPhysiNode* pPhyNode = (SJoinPhysiNode*)pNode; + SSortMergeJoinPhysiNode* pPhyNode = (SSortMergeJoinPhysiNode*)pNode; destroyPhysiNode((SPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 702422e022..d564d53633 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -739,12 +739,13 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* return TSDB_CODE_SUCCESS; } -static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, SArray* tagName, uint8_t tagNum) { +static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, + SArray* tagName, uint8_t tagNum) { pTbReq->type = TD_CHILD_TABLE; pTbReq->name = strdup(tname); pTbReq->ctb.suid = suid; pTbReq->ctb.tagNum = tagNum; - if(sname) pTbReq->ctb.name = strdup(sname); + if (sname) pTbReq->ctb.name = strdup(sname); pTbReq->ctb.pTag = (uint8_t*)pTag; pTbReq->ctb.tagName = taosArrayDup(tagName); pTbReq->commentLen = -1; @@ -969,7 +970,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // todo this can be optimize with parse column + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // todo this can be optimize with parse column code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -1012,7 +1013,8 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint goto end; } - buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid, pCxt->sTableName, tagName, pCxt->pTableMeta->tableInfo.numOfTags); + buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid, pCxt->sTableName, tagName, + pCxt->pTableMeta->tableInfo.numOfTags); end: for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { @@ -1650,7 +1652,6 @@ static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt) { static int32_t collectTableMetaKey(SInsertParseSyntaxCxt* pCxt, SToken* pTbToken) { SName name; CHECK_CODE(createSName(&name, pTbToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); - CHECK_CODE(reserveDbCfgInCache(pCxt->pComCxt->acctId, name.dbname, pCxt->pMetaCache)); CHECK_CODE(reserveUserAuthInCacheExt(pCxt->pComCxt->pUser, &name, AUTH_TYPE_WRITE, pCxt->pMetaCache)); CHECK_CODE(reserveTableMetaInCacheExt(&name, pCxt->pMetaCache)); CHECK_CODE(reserveTableVgroupInCacheExt(&name, pCxt->pMetaCache)); @@ -2332,7 +2333,8 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols return ret; } - buildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, pTableMeta->tableInfo.numOfTags); + buildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, + pTableMeta->tableInfo.numOfTags); taosArrayDestroy(tagName); smlHandle->tableExecHandle.createTblReq.ctb.name = taosMemoryMalloc(sTableNameLen + 1); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 74d5f03dc1..7c9a8b10dd 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -92,7 +92,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG: return "sliding value no larger than the interval value"; case TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL: - return "sliding value can not less than 1% of interval value"; + return "sliding value can not less than 1%% of interval value"; case TSDB_CODE_PAR_ONLY_ONE_JSON_TAG: return "Only one tag if there is a json tag"; case TSDB_CODE_PAR_INCORRECT_NUM_OF_COL: diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 3b545777a8..fcc395af62 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -993,22 +993,28 @@ static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) { } static int32_t sortPriKeyOptGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) { - int32_t code = TSDB_CODE_SUCCESS; - switch (nodeType(pNode)) { - case QUERY_NODE_LOGIC_PLAN_SCAN: + case QUERY_NODE_LOGIC_PLAN_SCAN: { + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + if (NULL != pScan->pGroupTags) { + *pNotOptimize = true; + return TSDB_CODE_SUCCESS; + } return nodesListMakeAppend(pScanNodes, (SNode*)pNode); - case QUERY_NODE_LOGIC_PLAN_JOIN: - code = + } + case QUERY_NODE_LOGIC_PLAN_JOIN: { + int32_t code = sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); if (TSDB_CODE_SUCCESS == code) { code = sortPriKeyOptGetScanNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes); } return code; + } case QUERY_NODE_LOGIC_PLAN_AGG: + case QUERY_NODE_LOGIC_PLAN_PARTITION: *pNotOptimize = true; - return code; + return TSDB_CODE_SUCCESS; default: break; } @@ -1034,6 +1040,18 @@ static EOrder sortPriKeyOptGetPriKeyOrder(SSortLogicNode* pSort) { return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order; } +static void sortPriKeyOptSetParentOrder(SLogicNode* pNode, EOrder order) { + if (NULL == pNode) { + return; + } + if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode)) { + ((SWindowLogicNode*)pNode)->inputTsOrder = order; + } else if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pNode)) { + ((SJoinLogicNode*)pNode)->inputTsOrder = order; + } + sortPriKeyOptSetParentOrder(pNode->pParent, order); +} + static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort, SNodeList* pScanNodes) { EOrder order = sortPriKeyOptGetPriKeyOrder(pSort); @@ -1048,6 +1066,7 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS pScan->node.resultDataOrder = DATA_ORDER_LEVEL_GLOBAL; pScan->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; } + sortPriKeyOptSetParentOrder(pScan->node.pParent, order); } SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0); @@ -1613,10 +1632,10 @@ static void alignProjectionWithTarget(SLogicNode* pNode) { } SProjectLogicNode* pProjectNode = (SProjectLogicNode*)pNode; - SNode* pProjection = NULL; + SNode* pProjection = NULL; FOREACH(pProjection, pProjectNode->pProjections) { SNode* pTarget = NULL; - bool keep = false; + bool keep = false; FOREACH(pTarget, pNode->pTargets) { if (0 == strcmp(((SColumnNode*)pProjection)->node.aliasName, ((SColumnNode*)pTarget)->colName)) { keep = true; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index fd359fce45..38f3c23925 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -621,8 +621,8 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode, SPhysiNode** pPhyNode) { - SJoinPhysiNode* pJoin = - (SJoinPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pJoinLogicNode, QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN); + SSortMergeJoinPhysiNode* pJoin = + (SSortMergeJoinPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pJoinLogicNode, QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN); if (NULL == pJoin) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 879c04743d..81e2bff179 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -469,7 +469,7 @@ static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent return code; } -static int32_t stbSplCreateMergeKeysByPrimaryKey(SNode* pPrimaryKey, SNodeList** pMergeKeys) { +static int32_t stbSplCreateMergeKeysByPrimaryKey(SNode* pPrimaryKey, EOrder order, SNodeList** pMergeKeys) { SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); if (NULL == pMergeKey) { return TSDB_CODE_OUT_OF_MEMORY; @@ -479,7 +479,7 @@ static int32_t stbSplCreateMergeKeysByPrimaryKey(SNode* pPrimaryKey, SNodeList** nodesDestroyNode((SNode*)pMergeKey); return TSDB_CODE_OUT_OF_MEMORY; } - pMergeKey->order = ORDER_ASC; + pMergeKey->order = order; pMergeKey->nullOrder = NULL_ORDER_FIRST; return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey); } @@ -491,7 +491,8 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo ((SWindowLogicNode*)pPartWindow)->windowAlgo = INTERVAL_ALGO_HASH; ((SWindowLogicNode*)pInfo->pSplitNode)->windowAlgo = INTERVAL_ALGO_MERGE; SNodeList* pMergeKeys = NULL; - code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, &pMergeKeys); + code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, + ((SWindowLogicNode*)pInfo->pSplitNode)->inputTsOrder, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, true); } @@ -579,7 +580,8 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pWindow->pChildren, 0); SNodeList* pMergeKeys = NULL; - int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys); + int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, + ((SWindowLogicNode*)pWindow)->inputTsOrder, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild, true); @@ -950,7 +952,8 @@ static int32_t stbSplCreateMergeScanNode(SScanLogicNode* pScan, SLogicNode** pOu pMergeScan->scanType = SCAN_TYPE_TABLE_MERGE; pMergeScan->node.pChildren = pChildren; splSetParent((SLogicNode*)pMergeScan); - code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pMergeScan), &pMergeKeys); + code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pMergeScan), + pMergeScan->scanSeq[0] > 0 ? ORDER_ASC : ORDER_DESC, &pMergeKeys); } if (TSDB_CODE_SUCCESS == code) { @@ -1020,14 +1023,14 @@ static int32_t stbSplSplitJoinNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) } static int32_t stbSplCreateMergeKeysForPartitionNode(SLogicNode* pPart, SNodeList** pMergeKeys) { - SNode* pPrimaryKey = - nodesCloneNode(stbSplFindPrimaryKeyFromScan((SScanLogicNode*)nodesListGetNode(pPart->pChildren, 0))); + SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pPart->pChildren, 0); + SNode* pPrimaryKey = nodesCloneNode(stbSplFindPrimaryKeyFromScan(pScan)); if (NULL == pPrimaryKey) { return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = nodesListAppend(pPart->pTargets, pPrimaryKey); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeKeysByPrimaryKey(pPrimaryKey, pMergeKeys); + code = stbSplCreateMergeKeysByPrimaryKey(pPrimaryKey, pScan->scanSeq[0] > 0 ? ORDER_ASC : ORDER_DESC, pMergeKeys); } return code; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 80ad480e43..ad6eff3c12 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -512,7 +512,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_OFFSET_UNIT, "Cannot use 'year' as TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG, "Interval offset should be shorter than interval") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_SLIDING_UNIT, "Does not support sliding when interval is natural month/year") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG, "sliding value no larger than the interval value") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL, "sliding value can not less than 1% of interval value") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL, "sliding value can not less than 1%% of interval value") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_ONLY_ONE_JSON_TAG, "Only one tag if there is a json tag") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INCORRECT_NUM_OF_COL, "Query block has incorrect number of result columns") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL, "Incorrect TIMESTAMP value") From b87bb4c08396fea4848e8d4ee072d63d54e82e40 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Jul 2022 15:23:13 +0800 Subject: [PATCH 21/27] fix(query): remove invalid assert. --- source/libs/executor/src/projectoperator.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 87ba0006e1..34149d7499 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -67,6 +67,11 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pInfo->pFilterNode = pProjPhyNode->node.pConditions; pInfo->mergeDataBlocks = pProjPhyNode->mergeDataBlock; + // todo remove it soon + if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + pInfo->mergeDataBlocks = true; + } + int32_t numOfRows = 4096; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; @@ -131,7 +136,7 @@ static int32_t discardGroupDataBlock(SSDataBlock* pBlock, SLimitInfo* pLimitInfo static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, SOperatorInfo* pOperator) { // remainGroupOffset == 0 // here check for a new group data, we need to handle the data of the previous group. - ASSERT(pLimitInfo->remainGroupOffset == 0); + ASSERT(pLimitInfo->remainGroupOffset == 0 || pLimitInfo->remainGroupOffset == -1); if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { pLimitInfo->numOfOutputGroups += 1; From 9e9a12cb094a15d702054b6144e44e10d0d042aa Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 25 Jul 2022 15:23:43 +0800 Subject: [PATCH 22/27] fix: fix memory leak of pseduo col computation in table merge scan --- source/libs/executor/src/scanoperator.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a9d03aebbe..8df22bbb11 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2557,9 +2557,7 @@ typedef struct STableMergeScanInfo { SArray* pColMatchInfo; int32_t numOfOutput; - SExprInfo* pPseudoExpr; - int32_t numOfPseudoExpr; - SqlFunctionCtx* pPseudoCtx; + SExprSupp pseudoSup; SQueryTableDataCond cond; int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan @@ -2974,6 +2972,7 @@ void destroyTableMergeScanOperatorInfo(void* param, int32_t numOfOutput) { pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); taosArrayDestroy(pTableScanInfo->pSortInfo); + cleanupExprSupp(&pTableScanInfo->pseudoSup); taosMemoryFreeClear(param); } @@ -3031,8 +3030,9 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN } if (pTableScanNode->scan.pScanPseudoCols != NULL) { - pInfo->pPseudoExpr = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pInfo->numOfPseudoExpr); - pInfo->pPseudoCtx = createSqlFunctionCtx(pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, &pInfo->rowEntryInfoOffset); + SExprSupp* pSup = &pInfo->pseudoSup; + pSup->pExprInfo = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pSup->numOfExprs); + pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset); } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; From 66bad33c9e8e3f986c43f519202ef9bc42e35baa Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 25 Jul 2022 15:34:11 +0800 Subject: [PATCH 23/27] fix: fix compilation error --- source/libs/executor/src/scanoperator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8df22bbb11..9d6c800ceb 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2698,9 +2698,9 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true); // currently only the tbname pseudo column - if (pTableScanInfo->numOfPseudoExpr > 0) { - int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pPseudoExpr, - pTableScanInfo->numOfPseudoExpr, pBlock, GET_TASKID(pTaskInfo)); + if (pTableScanInfo->pseudoSup.numOfExprs > 0) { + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo, + pTableScanInfo->pseudoSup.numOfExprs, pBlock, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } From 222e925644eeb77a977301e7b342f4bbdf6d17a4 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 25 Jul 2022 15:31:36 +0800 Subject: [PATCH 24/27] feat(wal): ref --- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/dnode/vnode/src/tq/tqMeta.c | 11 +++++++++-- source/libs/wal/src/walRead.c | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 740cfba9e3..118e3a5d43 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -385,8 +385,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN) { - int64_t fetchVer = fetchOffsetNew.version + 1; - SWalCkHead* pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); + int64_t fetchVer = fetchOffsetNew.version + 1; + pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { code = -1; goto OVER; diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 620417016f..290ffe5c8d 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -52,7 +52,7 @@ int32_t tqMetaOpen(STQ* pTq) { ASSERT(0); } - TXN txn; + TXN txn = {0}; if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { ASSERT(0); @@ -75,7 +75,13 @@ int32_t tqMetaOpen(STQ* pTq) { STqHandle handle; tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecodeSTqHandle(&decoder, &handle); - handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); + + handle.pRef = walOpenRef(pTq->pVnode->pWal); + if (handle.pRef == NULL) { + ASSERT(0); + } + walRefVer(handle.pRef, handle.snapshotVer); + if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { SReadHandle reader = { .meta = pTq->pVnode->pMeta, @@ -94,6 +100,7 @@ int32_t tqMetaOpen(STQ* pTq) { handle.execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); ASSERT(handle.execHandle.pExecReader); } else { + handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); handle.execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 37d7a8c3a9..ac62b7d98d 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -54,7 +54,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { } /*if (pReader->cond.enableRef) {*/ - /*taosHashPut(pWal->pRefHash, &pReader->readerId, sizeof(int64_t), &pReader, sizeof(void *));*/ + /* taosHashPut(pWal->pRefHash, &pReader->readerId, sizeof(int64_t), &pReader, sizeof(void *));*/ /*}*/ return pReader; From 0494f4a6e9716b31a935b7a8e127a8370deab369 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 25 Jul 2022 16:41:32 +0800 Subject: [PATCH 25/27] fix: fix table merge scan memory leak --- source/libs/executor/src/scanoperator.c | 34 ++++++++++++++----------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 89db5761c2..2dcb555834 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2829,29 +2829,31 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - tsortDestroySortHandle(pInfo->pSortHandle); + size_t numReaders = taosArrayGetSize(pInfo->dataReaders); + + for (int32_t i = 0; i < numReaders; ++i) { + STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); + blockDataDestroy(param->inputBlock); + } taosArrayClear(pInfo->sortSourceParams); - for (int32_t i = 0; i < taosArrayGetSize(pInfo->dataReaders); ++i) { + tsortDestroySortHandle(pInfo->pSortHandle); + + for (int32_t i = 0; i < numReaders; ++i) { STsdbReader* reader = taosArrayGetP(pInfo->dataReaders, i); tsdbReaderClose(reader); } - taosArrayDestroy(pInfo->dataReaders); pInfo->dataReaders = NULL; return TSDB_CODE_SUCCESS; } -SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, int32_t capacity, SOperatorInfo* pOperator) { +SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* pResBlock, int32_t capacity, SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSDataBlock* p = tsortGetSortedDataBlock(pHandle); - if (p == NULL) { - return NULL; - } - - blockDataEnsureCapacity(p, capacity); + blockDataCleanup(pResBlock); + blockDataEnsureCapacity(pResBlock, capacity); while (1) { STupleHandle* pTupleHandle = tsortNextTuple(pHandle); @@ -2859,14 +2861,15 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, int32_t capa break; } - appendOneRowToDataBlock(p, pTupleHandle); - if (p->info.rows >= capacity) { + appendOneRowToDataBlock(pResBlock, pTupleHandle); + if (pResBlock->info.rows >= capacity) { break; } } - qDebug("%s get sorted row blocks, rows:%d", GET_TASKID(pTaskInfo), p->info.rows); - return (p->info.rows > 0) ? p : NULL; + + qDebug("%s get sorted row blocks, rows:%d", GET_TASKID(pTaskInfo), pResBlock->info.rows); + return (pResBlock->info.rows > 0) ? pResBlock : NULL; } SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { @@ -2895,7 +2898,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { } SSDataBlock* pBlock = NULL; while (pInfo->tableStartIndex < tableListSize) { - pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); + pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pInfo->pResBlock, pOperator->resultInfo.capacity, pOperator); if (pBlock != NULL) { pBlock->info.groupId = pInfo->groupId; pOperator->resultInfo.totalRows += pBlock->info.rows; @@ -2919,6 +2922,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { void destroyTableMergeScanOperatorInfo(void* param, int32_t numOfOutput) { STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param; cleanupQueryTableDataCond(&pTableScanInfo->cond); + taosArrayDestroy(pTableScanInfo->sortSourceParams); for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->dataReaders); ++i) { STsdbReader* reader = taosArrayGetP(pTableScanInfo->dataReaders, i); From 8bb432c348e8e5bf8cc03fd6ddfc49978110ca2f Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 25 Jul 2022 16:45:23 +0800 Subject: [PATCH 26/27] feat: super table order by primary key optimization --- include/libs/nodes/querynodes.h | 1 + source/libs/nodes/src/nodesCloneFuncs.c | 2 ++ source/libs/parser/src/parAstCreater.c | 1 + source/libs/planner/src/planLogicCreater.c | 8 +++++--- source/libs/planner/src/planPhysiCreater.c | 3 +++ 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index f8c7024591..81ed5b5ecd 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -255,6 +255,7 @@ typedef struct SSelectStmt { int32_t selectFuncNum; bool isEmptyResult; bool isTimeLineResult; + bool isSubquery; bool hasAggFuncs; bool hasRepeatScanFuncs; bool hasIndefiniteRowsFunc; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 121e697630..5279d015b4 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -375,6 +375,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); COPY_SCALAR_FIELD(isSingleTableJoin); + COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; } @@ -440,6 +441,7 @@ static int32_t logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* p COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(igExpired); COPY_SCALAR_FIELD(windowAlgo); + COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index f52ee9af50..a54dae1ee9 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -527,6 +527,7 @@ SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const STok } if (QUERY_NODE_SELECT_STMT == nodeType(pSubquery)) { strcpy(((SSelectStmt*)pSubquery)->stmtName, tempTable->table.tableAlias); + ((SSelectStmt*)pSubquery)->isSubquery = true; } else if (QUERY_NODE_SET_OPERATOR == nodeType(pSubquery)) { strcpy(((SSetOperator*)pSubquery)->stmtName, tempTable->table.tableAlias); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 84e712b466..30e3b676df 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -339,6 +339,7 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pJoin->joinType = pJoinTable->joinType; pJoin->isSingleTableJoin = pJoinTable->table.singleTable; + pJoin->inputTsOrder = ORDER_ASC; pJoin->node.groupAction = GROUP_ACTION_CLEAR; pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; @@ -625,14 +626,14 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) { - int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs); - if (pCxt->pPlanCxt->streamQuery) { pWindow->triggerType = pCxt->pPlanCxt->triggerType; pWindow->watermark = pCxt->pPlanCxt->watermark; pWindow->igExpired = pCxt->pPlanCxt->igExpired; } + pWindow->inputTsOrder = ORDER_ASC; + int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) { code = rewriteExprsForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); } @@ -861,7 +862,8 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel TSWAP(pProject->node.pLimit, pSelect->pLimit); TSWAP(pProject->node.pSlimit, pSelect->pSlimit); - pProject->node.groupAction = GROUP_ACTION_CLEAR; + pProject->node.groupAction = + (!pSelect->isSubquery && pCxt->pPlanCxt->streamQuery) ? GROUP_ACTION_KEEP : GROUP_ACTION_CLEAR; pProject->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pProject->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 38f3c23925..587e566939 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -974,6 +974,9 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh } static bool projectCanMergeDataBlock(SProjectLogicNode* pProject) { + if (GROUP_ACTION_KEEP == pProject->node.groupAction) { + return false; + } if (DATA_ORDER_LEVEL_NONE == pProject->node.resultDataOrder) { return true; } From 3614db50c61b878aa983768bccde409fc65ccc83 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 25 Jul 2022 17:50:36 +0800 Subject: [PATCH 27/27] test: add test case for fix --- tests/system-test/7-tmq/TD-17803.py | 198 ++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 tests/system-test/7-tmq/TD-17803.py diff --git a/tests/system-test/7-tmq/TD-17803.py b/tests/system-test/7-tmq/TD-17803.py new file mode 100644 index 0000000000..771ff83a29 --- /dev/null +++ b/tests/system-test/7-tmq/TD-17803.py @@ -0,0 +1,198 @@ +from distutils.log import error +import taos +import sys +import time +import socket +import os +import threading +import subprocess +import platform + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + + + +class TDTestCase: + def __init__(self): + self.snapshot = 0 + self.replica = 3 + self.vgroups = 3 + self.ctbNum = 2 + self.rowsPerTbl = 2 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) + else: + break + return + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 2, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replica) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + # tmqCom.asyncInsertDataByInterlace(paraDict) + tdLog.printNoPrefix("11111111111111111111111") + tmqCom.create_ntable(tdSql, dbname=paraDict["dbName"], tbname_prefix="ntb", tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=1) + tdLog.printNoPrefix("222222222222222") + tmqCom.insert_rows_into_ntbl(tdSql, dbname=paraDict["dbName"], tbname_prefix="ntb", tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"], startTs=paraDict["startTs"], tblNum=1, rows=2) # tdLog.info("restart taosd to ensure that the data falls into the disk") + + tdLog.printNoPrefix("333333333333333333333") + tdSql.query("drop database %s"%paraDict["dbName"]) + tdLog.printNoPrefix("44444444444444444") + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + + # create and start thread + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha' "%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicFromStb1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 100 + showMsg = 1 + showRow = 1 + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsInserted = tdSql.getRows() + + tdLog.info("act consume rows: %d, act insert rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsInserted, expectrowcnt)) + + if totalConsumeRows != expectrowcnt: + tdLog.exit("tmq consume rows error!") + + # tmqCom.checkFileContent(consumerId, queryString) + + tmqCom.waitSubscriptionExit(tdSql, topicFromStb1) + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + # self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase())