From 9a022bca606879b13e734c944e080b31af5bc377 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 19 May 2022 23:54:44 +0800 Subject: [PATCH 01/23] feat: add case for update without multi-version --- source/dnode/vnode/src/tsdb/tsdbRead.c | 11 +- tests/script/tsim/insert/update0.sim | 155 +++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 tests/script/tsim/insert/update0.sim diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 90adba6f4d..6640090085 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2030,8 +2030,14 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } #endif if (TD_SUPPORT_UPDATE(pCfg->update)) { + if (lastKeyAppend != key) { + lastKeyAppend = key; + if (lastKeyAppend != TSKEY_INITIAL_VAL) { + ++curRow; + } + } + // load data from file firstly numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos); - lastKeyAppend = key; if (rv1 != TD_ROW_SVER(row1)) { rv1 = TD_ROW_SVER(row1); @@ -2041,7 +2047,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf } // still assign data into current row - mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, + numOfRows += mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, &curRow, row1, row2, numOfCols, pCheckInfo->tableId, pSchema1, pSchema2, pCfg->update, &lastKeyAppend); if (cur->win.skey == TSKEY_INITIAL_VAL) { @@ -2053,7 +2059,6 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf cur->mixBlock = true; moveToNextRowInMem(pCheckInfo); - ++curRow; pos += step; } else { diff --git a/tests/script/tsim/insert/update0.sim b/tests/script/tsim/insert/update0.sim new file mode 100644 index 0000000000..c34a08c79d --- /dev/null +++ b/tests/script/tsim/insert/update0.sim @@ -0,0 +1,155 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database d0 keep 365000d,365000d,365000d +sql use d0 + +print =============== create super table and register rsma +sql create table if not exists stb (ts timestamp, c1 int) tags (city binary(20),district binary(20)) rollup(min) file_factor 0.1 delay 2; + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags("BeiJing", "ChaoYang") +sql create table ct2 using stb tags("BeiJing", "HaiDian") + +sql show tables +if $rows != 2 then + return -1 +endi + +print =============== step3-1 insert records into ct1 +sql insert into ct1 values('2022-05-03 16:59:00.010', 10); +sql insert into ct1 values('2022-05-03 16:59:00.011', 11); +sql insert into ct1 values('2022-05-03 16:59:00.016', 16); +sql insert into ct1 values('2022-05-03 16:59:00.016', 17); +sql insert into ct1 values('2022-05-03 16:59:00.020', 20); +sql insert into ct1 values('2022-05-03 16:59:00.016', 18); +sql insert into ct1 values('2022-05-03 16:59:00.021', 21); +sql insert into ct1 values('2022-05-03 16:59:00.022', 22); + +print =============== step3-1 query records from ct1 from memory +sql select * from ct1; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 +print $data40 $data41 +print $data50 $data51 + +if $rows != 6 then + print rows $rows != 6 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data21 != 18 then + print data21 $data21 != 18 + return -1 +endi + +if $data51 != 22 then + print data51 $data51 != 22 + return -1 +endi + +print =============== step3-1 insert records into ct2 +sql insert into ct2 values('2022-03-02 16:59:00.010', 1),('2022-03-02 16:59:00.010',11),('2022-04-01 16:59:00.011',2),('2022-04-01 16:59:00.011',5),('2022-03-06 16:59:00.013',7); +sql insert into ct2 values('2022-03-02 16:59:00.010', 3),('2022-03-02 16:59:00.010',33),('2022-04-01 16:59:00.011',4),('2022-04-01 16:59:00.011',6),('2022-03-06 16:59:00.013',8); +sql insert into ct2 values('2022-03-02 16:59:00.010', 103),('2022-03-02 16:59:00.010',303),('2022-04-01 16:59:00.011',40),('2022-04-01 16:59:00.011',60),('2022-03-06 16:59:00.013',80); + +print =============== step3-1 query records from ct2 from memory +sql select * from ct2; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 + +if $rows != 3 then + print rows $rows != 3 + return -1 +endi + +if $data01 != 103 then + print data01 $data01 != 103 + return -1 +endi + +if $data11 != 80 then + print data11 $data11 != 80 + return -1 +endi + +if $data21 != 40 then + print data21 $data21 != 40 + return -1 +endi + +#==================== reboot to trigger commit data to file +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + +print =============== step3-2 query records from ct1 from file +sql select * from ct1; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 +print $data40 $data41 +print $data50 $data51 + +if $rows != 6 then + print rows $rows != 6 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data21 != 18 then + print data21 $data21 != 18 + return -1 +endi + +if $data51 != 22 then + print data51 $data51 != 22 + return -1 +endi + +print =============== step3-2 query records from ct2 from file +sql select * from ct2; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 + +if $rows != 3 then + print rows $rows != 3 + return -1 +endi + +if $data01 != 103 then + print data01 $data01 != 103 + return -1 +endi + +if $data11 != 80 then + print data11 $data11 != 80 + return -1 +endi + +if $data21 != 40 then + print data21 $data21 != 40 + return -1 +endi \ No newline at end of file From 4d50a1da76179fa17d0418702ff036d3cd734008 Mon Sep 17 00:00:00 2001 From: tomchon Date: Fri, 20 May 2022 09:49:47 +0800 Subject: [PATCH 02/23] test:modify testcase of muti-inserting data --- .../sync/threeReplica1VgElectWihtInsert.sim | 9 +- .../1-insert/insertWithMoreVgroup.py | 105 ++++++++---------- 2 files changed, 49 insertions(+), 65 deletions(-) diff --git a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim index f6996f1291..f568008a82 100644 --- a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim +++ b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim @@ -71,7 +71,7 @@ sql create database db replica $replica vgroups $vgroups $loop_cnt = 0 check_db_ready: $loop_cnt = $loop_cnt + 1 -sleep 200 +sleep 20 if $loop_cnt == 10 then print ====> db not ready! return -1 @@ -83,7 +83,7 @@ print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $dat if $rows != 3 then return -1 endi -if $data(db)[19] != nostrict then +if $data(db)[19] != ready then goto check_db_ready endi @@ -93,7 +93,7 @@ $loop_cnt = 0 check_vg_ready: $loop_cnt = $loop_cnt + 1 sleep 200 -if $loop_cnt == 40 then +if $loop_cnt == 300 then print ====> vgroups not ready! return -1 endi @@ -175,6 +175,7 @@ while $i < $tbNum endw $totalTblNum = $tbNum * 2 +print ====>totalTblNum:$totalTblNum sql show tables if $rows != $totalTblNum then return -1 @@ -226,7 +227,7 @@ $loop_cnt = 0 check_vg_ready_2: $loop_cnt = $loop_cnt + 1 sleep 200 -if $loop_cnt == 10 then +if $loop_cnt == 300 then print ====> vgroups switch fail!!! return -1 endi diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index d8720e8045..ede33e7aa7 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -30,9 +30,12 @@ class TDTestCase: # # --------------- main frame ------------------- # - clientCfgDict = {'queryproxy': '1'} + clientCfgDict = {'queryproxy': '1','debugFlag': 135} clientCfgDict["queryproxy"] = '2' + clientCfgDict["debugFlag"] = 143 + updatecfgDict = {'clientCfg': {}} + updatecfgDict = {'debugFlag': 143} updatecfgDict["clientCfg"] = clientCfgDict def caseDescription(self): ''' @@ -146,41 +149,39 @@ class TDTestCase: # insert data - def insert_data(self, host, dbname, stbname, ts_start,rowCount): + def insert_data(self, host, dbname, stbname, chilCount, ts_start, rowCount): buildPath = self.getBuildPath() config = buildPath+ "../sim/dnode1/cfg/" - tsql=self.newcur(host,config) tdLog.debug("ready to inser data") - tsql.execute("use %s" %dbname) pre_insert = "insert into " sql = pre_insert - tcount=int(tcount) - allRows=tcount*rowCount + chilCount=int(chilCount) + allRows=chilCount*rowCount tdLog.debug("doing insert data into stable-index:%s rows:%d ..."%(stbname, allRows)) exeStartTime=time.time() - for i in range(0,tcount): + for i in range(0,chilCount): sql += " %s_%d values "%(stbname,i) for j in range(rowCount): sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j) - if j >0 and j%5000 == 0: + if j >0 and j%4000 == 0: # print(sql) - tdSql.execute(sql) + tsql.execute(sql) sql = "insert into %s_%d values " %(stbname,i) # end sql if sql != pre_insert: - # print(sql) - tdSql.execute(sql) + print(sql) + print(len(sql)) + tsql.execute(sql) exeEndTime=time.time() spendTime=exeEndTime-exeStartTime speedInsert=allRows/spendTime - # tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert)) - - tdLog.debug("INSERT TABLE DATA ............ [OK]") + tdLog.debug("spent %.2fs to INSERT %d rows into %s , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,stbname,speedInsert)) + # tdLog.debug("INSERT TABLE DATA ............ [OK]") return - def mutiThread_insert_data(self, host, dbname, stbname, threadNumbers, ts_start, tcountStart,tcountStop,rowCount): + def mutiThread_insert_data(self, host, dbname, stbname, threadNumbers, chilCount, ts_start, childrowcount): buildPath = self.getBuildPath() config = buildPath+ "../sim/dnode1/cfg/" @@ -188,42 +189,11 @@ class TDTestCase: tdLog.debug("ready to inser data") tsql.execute("use %s" %dbname) - pre_insert = "insert into " - sql = pre_insert - tcount=tcountStop-tcountStart - allRows=tcount*rowCount - tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbname, allRows)) - exeStartTime=time.time() - for i in range(tcountStart,tcountStop): - sql += " %s_%d values "%(stbname,i) - for j in range(rowCount): - sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j) - if j >0 and j%5000 == 0: - # print(sql) - tdSql.execute(sql) - sql = "insert into %s_%d values " %(stbname,i) - # end sql - if sql != pre_insert: - # print(sql) - tdSql.execute(sql) - exeEndTime=time.time() - spendTime=exeEndTime-exeStartTime - speedInsert=allRows/spendTime - # tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert)) - - tdLog.debug("INSERT TABLE DATA ............ [OK]") - - - buildPath = self.getBuildPath() - config = buildPath+ "../sim/dnode1/cfg/" - - tsql=self.newcur(host,config) - tsql.execute("use %s" %dbname) - count=int(count) + chilCount=int(chilCount) threads = [] for i in range(threadNumbers): - tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i)) - threads.append(thd.Thread(target=self.create_tables, args=(host, dbname, stbname+"%d"%i, count,))) + # tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i)) + threads.append(thd.Thread(target=self.insert_data, args=(host, dbname, stbname+"%d"%i, chilCount, ts_start, childrowcount,))) start_time = time.time() for tr in threads: tr.start() @@ -231,8 +201,18 @@ class TDTestCase: tr.join() end_time = time.time() spendTime=end_time-start_time - speedCreate=count/spendTime - tdLog.debug("spent %.2fs to create %d stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,threadNumbers,threadNumbers*count,speedCreate)) + tableCounts=threadNumbers*chilCount + stableRows=chilCount*childrowcount + allRows=stableRows*threadNumbers + speedInsert=allRows/spendTime + + for i in range(threadNumbers): + tdSql.execute("use %s" %dbname) + tdSql.query("select count(*) from %s%d"%(stbname,i)) + tdSql.checkData(0,0,stableRows) + tdLog.debug("spent %.2fs to insert %d rows into %d stable and %d table, speed is %.2f table/s... [OK]"% (spendTime,allRows,threadNumbers,tableCounts,speedInsert)) + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return @@ -288,7 +268,10 @@ class TDTestCase: def test_case1(self): tdLog.debug("-----create database and muti-thread create tables test------- ") #host,dbname,stbname,vgroups,threadNumbers,tcountStart,tcountStop - self.mutiThread_create_tables(host="localhost",dbname="db2",stbname="stb2", vgroups=1, threadNumbers=5, count=10000) + #host, dbname, stbname, threadNumbers, chilCount, ts_start, childrowcount + self.mutiThread_create_tables(host="localhost",dbname="db",stbname="stb", vgroups=1, threadNumbers=5, count=50) + self.mutiThread_insert_data(host="localhost",dbname="db",stbname="stb", threadNumbers=5,chilCount=50,ts_start=self.ts,childrowcount=10) + return # test case2 base:insert data @@ -366,17 +349,17 @@ class TDTestCase: # run case def run(self): - # # test base case - # self.test_case1() - # tdLog.debug(" LIMIT test_case1 ............ [OK]") + # create database and tables。 + self.test_case1() + tdLog.debug(" LIMIT test_case1 ............ [OK]") - # test case - # self.test_case2() - # tdLog.debug(" LIMIT test_case2 ............ [OK]") + # # taosBenchmark ： create database and table + # self.test_case2() + # tdLog.debug(" LIMIT test_case2 ............ [OK]") - # test case - self.test_case3() - tdLog.debug(" LIMIT test_case3 ............ [OK]") + # # taosBenchmark：create database/table and insert data + # self.test_case3() + # tdLog.debug(" LIMIT test_case3 ............ [OK]") # # test qnode From 18eb037706ea039476905b77a64b1b3965c0407f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 20 May 2022 11:47:08 +0800 Subject: [PATCH 03/23] fix: ref count --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 18 +++++++++--------- source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 12 +++++++----- source/util/src/tqueue.c | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index eec6bb3fb4..c3f5a2cd93 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -308,22 +308,23 @@ int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SSingleWorker *pWorker = &pMgmt->monitorWorker; - dTrace("msg:%p, put into vnode-monitor worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); return 0; } static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType qtype) { - SMsgHead *pHead = pRpc->pCont; - + SMsgHead *pHead = pRpc->pCont; SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) return -1; SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - int32_t code = 0; + int32_t code = -1; - if (pMsg != NULL) { + if (pMsg == NULL) { + rpcFreeCont(pRpc->pCont); + pRpc->pCont = NULL; + } else { memcpy(pMsg, pRpc, sizeof(SRpcMsg)); switch (qtype) { case WRITE_QUEUE: @@ -351,7 +352,6 @@ static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType q taosWriteQitem(pVnode->pSyncQ, pMsg); break; default: - code = -1; terrno = TSDB_CODE_INVALID_PARA; break; } @@ -428,7 +428,7 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { return -1; } - dDebug("vgId:%d, vnode queue is alloced", pVnode->vgId); + dDebug("vgId:%d, queue is alloced", pVnode->vgId); return 0; } @@ -445,7 +445,7 @@ void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { pVnode->pQueryQ = NULL; pVnode->pFetchQ = NULL; pVnode->pMergeQ = NULL; - dDebug("vgId:%d, vnode queue is freed", pVnode->vgId); + dDebug("vgId:%d, queue is freed", pVnode->vgId); } int32_t vmStartWorker(SVnodeMgmt *pMgmt) { @@ -496,7 +496,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { - dError("failed to start mnode vnode-monitor worker since %s", terrstr()); + dError("failed to start vnode-monitor worker since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index 2497da13ec..0b74d865fd 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -161,10 +161,12 @@ void dmGetVnodeLoads(SMonVloadInfo *pInfo) { void dmGetMnodeLoads(SMonMloadInfo *pInfo) { SDnode *pDnode = dmInstance(); SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE]; - if (tsMultiProcess) { - dmSendLocalRecv(pDnode, TDMT_MON_MM_LOAD, tDeserializeSMonMloadInfo, pInfo); - } else if (pWrapper->pMgmt != NULL) { - mmGetMnodeLoads(pWrapper->pMgmt, pInfo); + if (dmMarkWrapper(pWrapper) == 0) { + if (tsMultiProcess) { + dmSendLocalRecv(pDnode, TDMT_MON_MM_LOAD, tDeserializeSMonMloadInfo, pInfo); + } else if (pWrapper->pMgmt != NULL) { + mmGetMnodeLoads(pWrapper->pMgmt, pInfo); + } + dmReleaseWrapper(pWrapper); } - dmReleaseWrapper(pWrapper); } diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 5e206f3e6e..6a10794ea1 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -162,7 +162,7 @@ void *taosAllocateQitem(int32_t size, EQItype itype) { uTrace("item:%p, node:%p is allocated", pNode->item, pNode); } - return (void *)pNode->item; + return pNode->item; } void taosFreeQitem(void *pItem) { From eddd5b390cb4eb313094a5992e26c457a4d39e15 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 20 May 2022 12:48:15 +0800 Subject: [PATCH 04/23] refactor: adjust vm worker --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 44 ++++++++------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index c3f5a2cd93..50e2256145 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -22,21 +22,19 @@ static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, - .info = pMsg->info, .pCont = pMsg->info.rsp, .contLen = pMsg->info.rspLen, + .info = pMsg->info, }; tmsgSendRsp(&rsp); } static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SVnodeMgmt *pMgmt = pInfo->ahandle; + int32_t code = -1; + dTrace("msg:%p, get from vnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - int32_t code = -1; - tmsg_t msgType = pMsg->msgType; - dTrace("msg:%p, get from vnode queue, type:%s", pMsg, TMSG_INFO(msgType)); - - switch (msgType) { + switch (pMsg->msgType) { case TDMT_MON_VM_INFO: code = vmProcessGetMonitorInfoReq(pMgmt, pMsg); break; @@ -54,7 +52,7 @@ static void vmProcessMgmtMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { dError("msg:%p, not processed in vnode queue", pMsg); } - if (msgType & 1u) { + if (IsReq(pMsg)) { if (code != 0 && terrno != 0) code = terrno; vmSendRsp(pMsg, code); } @@ -96,7 +94,6 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); if (pArray == NULL) { dError("failed to process %d msgs in write-queue since %s", numOfMsgs, terrstr()); @@ -116,7 +113,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO for (int i = 0; i < taosArrayGetSize(pArray); i++) { SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - SRpcMsg rsp = {.info = pMsg->info, .pCont = NULL, .contLen = 0}; + SRpcMsg rsp = {.info = pMsg->info}; int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pMsg, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { @@ -130,7 +127,6 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR; tmsgSendRsp(&rsp); } else if (ret == TAOS_SYNC_PROPOSE_SUCCESS) { - // ok // send response in applyQ } else { assert(0); @@ -149,16 +145,13 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg *pMsg = NULL; - SRpcMsg rsp; for (int32_t i = 0; i < numOfMsgs; ++i) { + SRpcMsg *pMsg = NULL; taosGetQitem(qall, (void **)&pMsg); // init response rpc msg - rsp.code = 0; - rsp.pCont = NULL; - rsp.contLen = 0; + SRpcMsg rsp = {0}; // get original rpc msg assert(pMsg->msgType == TDMT_VND_SYNC_APPLY_MSG); @@ -177,7 +170,6 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO rpcFreeCont(originalRpcMsg.pCont); // if leader, send response - // if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) { if (pMsg->info.handle != NULL) { rsp.info = pMsg->info; tmsgSendRsp(&rsp); @@ -190,21 +182,19 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { + SRpcMsg *pMsg = NULL; taosGetQitem(qall, (void **)&pMsg); - // todo - SRpcMsg *pRsp = NULL; - int32_t ret = vnodeProcessSyncReq(pVnode->pImpl, pMsg, &pRsp); - if (ret != 0) { - // if leader, send response + int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); + if (code != 0) { if (pMsg->info.handle != NULL) { - SRpcMsg rsp = {0}; - rsp.code = terrno; - rsp.info = pMsg->info; - dTrace("msg:%p, process sync queue error since code:%s", pMsg, terrstr()); + SRpcMsg rsp = { + .code = (terrno < 0) ? terrno : code, + .info = pMsg->info, + }; + dTrace("msg:%p, failed to process sync queue since %s", pMsg, terrstr()); tmsgSendRsp(&rsp); } } @@ -216,9 +206,9 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { + SRpcMsg *pMsg = NULL; taosGetQitem(qall, (void **)&pMsg); dTrace("msg:%p, get from vnode-merge queue", pMsg); From a67e46efc813b58a98c904163cef7af7c916ec89 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 20 May 2022 13:42:03 +0800 Subject: [PATCH 05/23] fix: invalid code --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 50e2256145..9b60260cc1 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -309,11 +309,12 @@ static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType q if (pVnode == NULL) return -1; SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - int32_t code = -1; + int32_t code = 0; if (pMsg == NULL) { rpcFreeCont(pRpc->pCont); pRpc->pCont = NULL; + code = -1; } else { memcpy(pMsg, pRpc, sizeof(SRpcMsg)); switch (qtype) { @@ -342,6 +343,7 @@ static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType q taosWriteQitem(pVnode->pSyncQ, pMsg); break; default: + code = -1; terrno = TSDB_CODE_INVALID_PARA; break; } From 09fa500d049b965e09e631dcb364cb88b9529eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Fri, 20 May 2022 14:05:38 +0800 Subject: [PATCH 06/23] feat(query): add nested query function --- tests/system-test/2-query/nestedQuery.py | 617 +++++++++++++++-------- 1 file changed, 407 insertions(+), 210 deletions(-) diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py index 11f156c7a4..8c61451756 100755 --- a/tests/system-test/2-query/nestedQuery.py +++ b/tests/system-test/2-query/nestedQuery.py @@ -244,7 +244,7 @@ class TDTestCase: q_u_or_where = ['(t1.q_binary like \'binary%\' or t1.q_binary = \'0\' or t2.q_binary like \'binary%\' or t2.q_binary = \'0\' )' , '(t1.q_nchar like \'nchar%\' or t1.q_nchar = \'0\' or t2.q_nchar like \'nchar%\' or t2.q_nchar = \'0\' )' , '(t1.q_bool = true or t1.q_bool = false or t2.q_bool = true or t2.q_bool = false)' , - '(t1.q_bool in (0 , 1) or t2.q_bool in (0 , 1)' , 't1.q_bool in ( true , false) or t2.q_bool in ( true , false))' , '(t1.q_bool = 0 or t1.q_bool = 1 or t2.q_bool = 0 or t2.q_bool = 1)' , + '(t1.q_bool in (0 , 1) or t2.q_bool in (0 , 1))' , '(t1.q_bool in ( true , false) or t2.q_bool in ( true , false))' , '(t1.q_bool = 0 or t1.q_bool = 1 or t2.q_bool = 0 or t2.q_bool = 1)' , '(t1.q_bigint between -9223372036854775807 and 9223372036854775807 or t2.q_bigint between -9223372036854775807 and 9223372036854775807)', '(t1.q_int between -2147483647 and 2147483647 or t2.q_int between -2147483647 and 2147483647)', '(t1.q_smallint between -32767 and 32767 or t2.q_smallint between -32767 and 32767)', @@ -315,7 +315,63 @@ class TDTestCase: group_where = ['group by tbname , loc' , 'group by tbname', 'group by tbname, t_bigint', 'group by tbname,t_int', 'group by tbname, t_smallint', 'group by tbname,t_tinyint', 'group by tbname,t_float', 'group by tbname,t_double' , 'group by tbname,t_binary', 'group by tbname,t_nchar', 'group by tbname,t_bool' ,'group by tbname ,loc ,t_bigint', - 'group by tbname,t_binary ,t_nchar ,t_bool' , 'group by tbname,t_int ,t_smallint ,t_tinyint' , 'group by tbname,t_float ,t_double ' ] + 'group by tbname,t_binary ,t_nchar ,t_bool' , 'group by tbname,t_int ,t_smallint ,t_tinyint' , 'group by tbname,t_float ,t_double ' , + 'PARTITION BY tbname , loc' , 'PARTITION BY tbname', 'PARTITION BY tbname, t_bigint', 'PARTITION BY tbname,t_int', 'PARTITION BY tbname, t_smallint', 'PARTITION BY tbname,t_tinyint', + 'PARTITION BY tbname,t_float', 'PARTITION BY tbname,t_double' , 'PARTITION BY tbname,t_binary', 'PARTITION BY tbname,t_nchar', 'PARTITION BY tbname,t_bool' ,'PARTITION BY tbname ,loc ,t_bigint', + 'PARTITION BY tbname,t_binary ,t_nchar ,t_bool' , 'PARTITION BY tbname,t_int ,t_smallint ,t_tinyint' , 'PARTITION BY tbname,t_float ,t_double '] + group_where_j = ['group by t1.loc' , 'group by t1.t_bigint', 'group by t1.t_int', 'group by t1.t_smallint', 'group by t1.t_tinyint', + 'group by t1.t_float', 'group by t1.t_double' , 'group by t1.t_binary', 'group by t1.t_nchar', 'group by t1.t_bool' ,'group by t1.loc ,t1.t_bigint', + 'group by t1.t_binary ,t1.t_nchar ,t1.t_bool' , 'group by t1.t_int ,t1.t_smallint ,t1.t_tinyint' , 'group by t1.t_float ,t1.t_double ' , + 'PARTITION BY t1.loc' , 'PARTITION by t1.t_bigint', 'PARTITION by t1.t_int', 'PARTITION by t1.t_smallint', 'PARTITION by t1.t_tinyint', + 'PARTITION by t1.t_float', 'PARTITION by t1.t_double' , 'PARTITION by t1.t_binary', 'PARTITION by t1.t_nchar', 'PARTITION by t1.t_bool' ,'PARTITION BY t1.loc ,t1.t_bigint', + 'PARTITION by t1.t_binary ,t1.t_nchar ,t1.t_bool' , 'PARTITION by t1.t_int ,t1.t_smallint ,t1.t_tinyint' , 'PARTITION by t1.t_float ,t1.t_double ', + 'group by t2.loc' , 'group by t2.t_bigint', 'group by t2.t_int', 'group by t2.t_smallint', 'group by t2.t_tinyint', + 'group by t2.t_float', 'group by t2.t_double' , 'group by t2.t_binary', 'group by t2.t_nchar', 'group by t2.t_bool' ,'group by t2.loc ,t2.t_bigint', + 'group by t2.t_binary ,t2.t_nchar ,t2.t_bool' , 'group by t2.t_int ,t2.t_smallint ,t2.t_tinyint' , 'group by t2.t_float ,t2.t_double ' , + 'PARTITION BY t2.loc' , 'PARTITION by t2.t_bigint', 'PARTITION by t2.t_int', 'PARTITION by t2.t_smallint', 'PARTITION by t2.t_tinyint', + 'PARTITION by t2.t_float', 'PARTITION by t2.t_double' , 'PARTITION by t2.t_binary', 'PARTITION by t2.t_nchar', 'PARTITION by t2.t_bool' ,'PARTITION BY t2.loc ,t2.t_bigint', + 'PARTITION by t2.t_binary ,t2.t_nchar ,t2.t_bool' , 'PARTITION by t2.t_int ,t2.t_smallint ,t2.t_tinyint' , 'PARTITION by t2.t_float ,t2.t_double '] + + partiton_where = ['PARTITION BY tbname , loc' , 'PARTITION BY tbname', 'PARTITION BY tbname, t_bigint', 'PARTITION BY tbname,t_int', 'PARTITION BY tbname, t_smallint', 'PARTITION BY tbname,t_tinyint', + 'PARTITION BY tbname,t_float', 'PARTITION BY tbname,t_double' , 'PARTITION BY tbname,t_binary', 'PARTITION BY tbname,t_nchar', 'PARTITION BY tbname,t_bool' ,'PARTITION BY tbname ,loc ,t_bigint', + 'PARTITION BY tbname,t_binary ,t_nchar ,t_bool' , 'PARTITION BY tbname,t_int ,t_smallint ,t_tinyint' , 'PARTITION BY tbname,t_float ,t_double '] + partiton_where_j = ['PARTITION BY t1.loc' , 'PARTITION by t1.t_bigint', 'PARTITION by t1.t_int', 'PARTITION by t1.t_smallint', 'PARTITION by t1.t_tinyint', + 'PARTITION by t1.t_float', 'PARTITION by t1.t_double' , 'PARTITION by t1.t_binary', 'PARTITION by t1.t_nchar', 'PARTITION by t1.t_bool' ,'PARTITION BY t1.loc ,t1.t_bigint', + 'PARTITION by t1.t_binary ,t1.t_nchar ,t1.t_bool' , 'PARTITION by t1.t_int ,t1.t_smallint ,t1.t_tinyint' , 'PARTITION by t1.t_float ,t1.t_double ', + 'PARTITION BY t2.loc' , 'PARTITION by t2.t_bigint', 'PARTITION by t2.t_int', 'PARTITION by t2.t_smallint', 'PARTITION by t2.t_tinyint', + 'PARTITION by t2.t_float', 'PARTITION by t2.t_double' , 'PARTITION by t2.t_binary', 'PARTITION by t2.t_nchar', 'PARTITION by t2.t_bool' ,'PARTITION BY t2.loc ,t2.t_bigint', + 'PARTITION by t2.t_binary ,t2.t_nchar ,t2.t_bool' , 'PARTITION by t2.t_int ,t2.t_smallint ,t2.t_tinyint' , 'PARTITION by t2.t_float ,t2.t_double '] + + + group_where_regular = ['group by tbname ' , 'group by tbname', 'group by tbname, q_bigint', 'group by tbname,q_int', 'group by tbname, q_smallint', 'group by tbname,q_tinyint', + 'group by tbname,q_float', 'group by tbname,q_double' , 'group by tbname,q_binary', 'group by tbname,q_nchar', 'group by tbname,q_bool' ,'group by tbname ,q_bigint', + 'group by tbname,q_binary ,q_nchar ,q_bool' , 'group by tbname,q_int ,q_smallint ,q_tinyint' , 'group by tbname,q_float ,q_double ' , + 'PARTITION BY tbname ' , 'PARTITION BY tbname', 'PARTITION BY tbname, q_bigint', 'PARTITION BY tbname,q_int', 'PARTITION BY tbname, q_smallint', 'PARTITION BY tbname,q_tinyint', + 'PARTITION BY tbname,q_float', 'PARTITION BY tbname,q_double' , 'PARTITION BY tbname,q_binary', 'PARTITION BY tbname,q_nchar', 'PARTITION BY tbname,q_bool' ,'PARTITION BY tbname ,q_bigint', + 'PARTITION BY tbname,q_binary ,q_nchar ,q_bool' , 'PARTITION BY tbname,q_int ,q_smallint ,q_tinyint' , 'PARTITION BY tbname,q_float ,q_double '] + group_where_regular_j = ['group by t1.q_bigint', 'group by t1.q_int', 'group by t1.q_smallint', 'group by t1.q_tinyint', + 'group by t1.q_float', 'group by t1.q_double' , 'group by t1.q_binary', 'group by t1.q_nchar', 'group by t1.q_bool' ,'group by t1.q_bigint', + 'group by t1.q_binary ,t1.q_nchar ,t1.q_bool' , 'group by t1.q_int ,t1.q_smallint ,t1.q_tinyint' , 'group by t1.q_float ,t1.q_double ' , + 'PARTITION by t1.q_bigint', 'PARTITION by t1.q_int', 'PARTITION by t1.q_smallint', 'PARTITION by t1.q_tinyint', + 'PARTITION by t1.q_float', 'PARTITION by t1.q_double' , 'PARTITION by t1.q_binary', 'PARTITION by t1.q_nchar', 'PARTITION by t1.q_bool' ,'PARTITION BY t1.q_bigint', + 'PARTITION by t1.q_binary ,t1.q_nchar ,t1.q_bool' , 'PARTITION by t1.q_int ,t1.q_smallint ,t1.q_tinyint' , 'PARTITION by t1.q_float ,t1.q_double ', + 'group by t2.q_bigint', 'group by t2.q_int', 'group by t2.q_smallint', 'group by t2.q_tinyint', + 'group by t2.q_float', 'group by t2.q_double' , 'group by t2.q_binary', 'group by t2.q_nchar', 'group by t2.q_bool' ,'group by t2.q_bigint', + 'group by t2.q_binary ,t2.q_nchar ,t2.q_bool' , 'group by t2.q_int ,t2.q_smallint ,t2.q_tinyint' , 'group by t2.q_float ,t2.q_double ' , + 'PARTITION by t2.q_bigint', 'PARTITION by t2.q_int', 'PARTITION by t2.q_smallint', 'PARTITION by t2.q_tinyint', + 'PARTITION by t2.q_float', 'PARTITION by t2.q_double' , 'PARTITION by t2.q_binary', 'PARTITION by t2.q_nchar', 'PARTITION by t2.q_bool' ,'PARTITION BY t2.q_bigint', + 'PARTITION by t2.q_binary ,t2.q_nchar ,t2.q_bool' , 'PARTITION by t2.q_int ,t2.q_smallint ,t2.q_tinyint' , 'PARTITION by t2.q_float ,t2.q_double '] + + partiton_where_regular = ['PARTITION BY tbname ' , 'PARTITION BY tbname', 'PARTITION BY tbname, q_bigint', 'PARTITION BY tbname,q_int', 'PARTITION BY tbname, q_smallint', 'PARTITION BY tbname,q_tinyint', + 'PARTITION BY tbname,q_float', 'PARTITION BY tbname,q_double' , 'PARTITION BY tbname,q_binary', 'PARTITION BY tbname,q_nchar', 'PARTITION BY tbname,q_bool' ,'PARTITION BY tbname ,q_bigint', + 'PARTITION BY tbname,q_binary ,q_nchar ,q_bool' , 'PARTITION BY tbname,q_int ,q_smallint ,q_tinyint' , 'PARTITION BY tbname,q_float ,q_double '] + partiton_where_regular_j = ['PARTITION by t1.q_bigint', 'PARTITION by t1.q_int', 'PARTITION by t1.q_smallint', 'PARTITION by t1.q_tinyint', + 'PARTITION by t1.q_float', 'PARTITION by t1.q_double' , 'PARTITION by t1.q_binary', 'PARTITION by t1.q_nchar', 'PARTITION by t1.q_bool' ,'PARTITION BY t1.q_bigint', + 'PARTITION by t1.q_binary ,t1.q_nchar ,t1.q_bool' , 'PARTITION by t1.q_int ,t1.q_smallint ,t1.q_tinyint' , 'PARTITION by t1.q_float ,t1.q_double ', + 'PARTITION by t2.q_bigint', 'PARTITION by t2.q_int', 'PARTITION by t2.q_smallint', 'PARTITION by t2.q_tinyint', + 'PARTITION by t2.q_float', 'PARTITION by t2.q_double' , 'PARTITION by t2.q_binary', 'PARTITION by t2.q_nchar', 'PARTITION by t2.q_bool' ,'PARTITION BY t2.q_bigint', + 'PARTITION by t2.q_binary ,t2.q_nchar ,t2.q_bool' , 'PARTITION by t2.q_int ,t2.q_smallint ,t2.q_tinyint' , 'PARTITION by t2.q_float ,t2.q_double '] + having_support = ['having count(q_int) > 0','having count(q_bigint) > 0','having count(q_smallint) > 0','having count(q_tinyint) > 0','having count(q_float) > 0','having count(q_double) > 0','having count(q_bool) > 0', 'having avg(q_int) > 0','having avg(q_bigint) > 0','having avg(q_smallint) > 0','having avg(q_tinyint) > 0','having avg(q_float) > 0','having avg(q_double) > 0', 'having sum(q_int) > 0','having sum(q_bigint) > 0','having sum(q_smallint) > 0','having sum(q_tinyint) > 0','having sum(q_float) > 0','having sum(q_double) > 0', @@ -334,6 +390,18 @@ class TDTestCase: 'having PERCENTILE(q_int,10) > 0','having PERCENTILE(q_bigint,10) > 0','having PERCENTILE(q_smallint,10) > 0','having PERCENTILE(q_tinyint,10) > 0','having PERCENTILE(q_float,10) > 0','having PERCENTILE(q_double,10) > 0'] having_tagnot_support = ['having LAST_ROW(q_int) > 0','having LAST_ROW(q_bigint) > 0','having LAST_ROW(q_smallint) > 0','having LAST_ROW(q_tinyint) > 0','having LAST_ROW(q_float) > 0','having LAST_ROW(q_double) > 0'] + having_support_j = ['having count(t1.q_int) > 0','having count(t1.q_bigint) > 0','having count(t1.q_smallint) > 0','having count(t1.q_tinyint) > 0','having count(t1.q_float) > 0','having count(t1.q_double) > 0','having count(t1.q_bool) > 0', + 'having avg(t1.q_int) > 0','having avg(t1.q_bigint) > 0','having avg(t1.q_smallint) > 0','having avg(t1.q_tinyint) > 0','having avg(t1.q_float) > 0','having avg(t1.q_double) > 0', + 'having sum(t1.q_int) > 0','having sum(t1.q_bigint) > 0','having sum(t1.q_smallint) > 0','having sum(t1.q_tinyint) > 0','having sum(t1.q_float) > 0','having sum(t1.q_double) > 0', + 'having STDDEV(t1.q_int) > 0','having STDDEV(t1.q_bigint) > 0','having STDDEV(t1.q_smallint) > 0','having STDDEV(t1.q_tinyint) > 0','having STDDEV(t1.q_float) > 0','having STDDEV(t1.q_double) > 0', + 'having TWA(t1.q_int) > 0','having TWA(t1.q_bigint) > 0','having TWA(t1.q_smallint) > 0','having TWA(t1.q_tinyint) > 0','having TWA(t1.q_float) > 0','having TWA(t1.q_double) > 0', + 'having IRATE(t1.q_int) > 0','having IRATE(t1.q_bigint) > 0','having IRATE(t1.q_smallint) > 0','having IRATE(t1.q_tinyint) > 0','having IRATE(t1.q_float) > 0','having IRATE(t1.q_double) > 0', + 'having MIN(t1.q_int) > 0','having MIN(t1.q_bigint) > 0','having MIN(t1.q_smallint) > 0','having MIN(t1.q_tinyint) > 0','having MIN(t1.q_float) > 0','having MIN(t1.q_double) > 0', + 'having MAX(t1.q_int) > 0','having MAX(t1.q_bigint) > 0','having MAX(t1.q_smallint) > 0','having MAX(t1.q_tinyint) > 0','having MAX(t1.q_float) > 0','having MAX(t1.q_double) > 0', + 'having FIRST(t1.q_int) > 0','having FIRST(t1.q_bigint) > 0','having FIRST(t1.q_smallint) > 0','having FIRST(t1.q_tinyint) > 0','having FIRST(t1.q_float) > 0','having FIRST(t1.q_double) > 0', + 'having LAST(t1.q_int) > 0','having LAST(t1.q_bigint) > 0','having LAST(t1.q_smallint) > 0','having LAST(t1.q_tinyint) > 0','having LAST(t1.q_float) > 0','having LAST(t1.q_double) > 0', + 'having APERCENTILE(t1.q_int,10) > 0','having APERCENTILE(t1.q_bigint,10) > 0','having APERCENTILE(t1.q_smallint,10) > 0','having APERCENTILE(t1.q_tinyint,10) > 0','having APERCENTILE(t1.q_float,10) > 0','having APERCENTILE(t1.q_double,10) > 0'] + # limit offset where limit_where = ['limit 1 offset 1' , 'limit 1' , 'limit 2 offset 1' , 'limit 2', 'limit 12 offset 1' , 'limit 20', 'limit 20 offset 10' , 'limit 200'] limit1_where = ['limit 1 offset 1' , 'limit 1' ] @@ -375,7 +443,17 @@ class TDTestCase: 'last_row(q_int)' , 'last_row(q_bigint)' , 'last_row(q_smallint)' , 'last_row(q_tinyint)' , 'last_row(q_float)' , 'last_row(q_double)' , 'last_row(q_bool)' ,'last_row(q_binary)' ,'last_row(q_nchar)' ,'last_row(q_ts)'] - + calc_select_not_support_ts = ['first(q_int)' , 'first(q_bigint)' , 'first(q_smallint)' , 'first(q_tinyint)' , 'first(q_float)' ,'first(q_double)' ,'first(q_binary)' ,'first(q_nchar)' ,'first(q_bool)' ,'first(q_ts)' , + 'last(q_int)' , 'last(q_bigint)' , 'last(q_smallint)' , 'last(q_tinyint)' , 'last(q_float)' ,'last(q_double)' , 'last(q_binary)' ,'last(q_nchar)' ,'last(q_bool)' ,'last(q_ts)' , + 'last_row(q_int)' , 'last_row(q_bigint)' , 'last_row(q_smallint)' , 'last_row(q_tinyint)' , 'last_row(q_float)' , + 'last_row(q_double)' , 'last_row(q_bool)' ,'last_row(q_binary)' ,'last_row(q_nchar)' ,'last_row(q_ts)', + 'apercentile(q_int,20)' , 'apercentile(q_bigint,20)' ,'apercentile(q_smallint,20)' ,'apercentile(q_tinyint,20)' ,'apercentile(q_float,20)' ,'apercentile(q_double,20)'] + + calc_select_support_ts = ['bottom(q_int,20)' , 'bottom(q_bigint,20)' , 'bottom(q_smallint,20)' , 'bottom(q_tinyint,20)' ,'bottom(q_float,20)' , 'bottom(q_double,20)' , + 'top(q_int,20)' , 'top(q_bigint,20)' , 'top(q_smallint,20)' ,'top(q_tinyint,20)' ,'top(q_float,20)' ,'top(q_double,20)' , + 'min(q_int)' , 'min(q_bigint)' , 'min(q_smallint)' , 'min(q_tinyint)' , 'min(q_float)' ,'min(q_double)' , + 'max(q_int)' , 'max(q_bigint)' , 'max(q_smallint)' , 'max(q_tinyint)' ,'max(q_float)' ,'max(q_double)' ] + calc_select_regular = [ 'PERCENTILE(q_int,10)' ,'PERCENTILE(q_bigint,20)' , 'PERCENTILE(q_smallint,30)' ,'PERCENTILE(q_tinyint,40)' ,'PERCENTILE(q_float,50)' ,'PERCENTILE(q_double,60)'] @@ -392,6 +470,23 @@ class TDTestCase: 'first(t2.q_int)' , 'first(t2.q_bigint)' , 'first(t2.q_smallint)' , 'first(t2.q_tinyint)' , 'first(t2.q_float)' ,'first(t2.q_double)' ,'first(t2.q_binary)' ,'first(t2.q_nchar)' ,'first(t2.q_bool)' ,'first(t2.q_ts)' , 'last(t2.q_int)' , 'last(t2.q_bigint)' , 'last(t2.q_smallint)' , 'last(t2.q_tinyint)' , 'last(t2.q_float)' ,'last(t2.q_double)' , 'last(t2.q_binary)' ,'last(t2.q_nchar)' ,'last(t2.q_bool)' ,'last(t2.q_ts)'] + calc_select_in_support_ts_j = ['bottom(t1.q_int,20)' , 'bottom(t1.q_bigint,20)' , 'bottom(t1.q_smallint,20)' , 'bottom(t1.q_tinyint,20)' ,'bottom(t1.q_float,20)' , 'bottom(t1.q_double,20)' , + 'top(t1.q_int,20)' , 'top(t1.q_bigint,20)' , 'top(t1.q_smallint,20)' ,'top(t1.q_tinyint,20)' ,'top(t1.q_float,20)' ,'top(t1.q_double,20)' , + 'min(t1.q_int)' , 'min(t1.q_bigint)' , 'min(t1.q_smallint)' , 'min(t1.q_tinyint)' , 'min(t1.q_float)' ,'min(t1.q_double)' , + 'max(t1.q_int)' , 'max(t1.q_bigint)' , 'max(t1.q_smallint)' , 'max(t1.q_tinyint)' ,'max(t1.q_float)' ,'max(t1.q_double)' , + 'bottom(t2.q_int,20)' , 'bottom(t2.q_bigint,20)' , 'bottom(t2.q_smallint,20)' , 'bottom(t2.q_tinyint,20)' ,'bottom(t2.q_float,20)' , 'bottom(t2.q_double,20)' , + 'top(t2.q_int,20)' , 'top(t2.q_bigint,20)' , 'top(t2.q_smallint,20)' ,'top(t2.q_tinyint,20)' ,'top(t2.q_float,20)' ,'top(t2.q_double,20)' , + 'min(t2.q_int)' , 'min(t2.q_bigint)' , 'min(t2.q_smallint)' , 'min(t2.q_tinyint)' , 'min(t2.q_float)' ,'min(t2.q_double)' , + 'max(t2.q_int)' , 'max(t2.q_bigint)' , 'max(t2.q_smallint)' , 'max(t2.q_tinyint)' ,'max(t2.q_float)' ,'max(t2.q_double)' , + ] + + calc_select_in_not_support_ts_j = ['apercentile(t1.q_int,20)' , 'apercentile(t1.q_bigint,20)' ,'apercentile(t1.q_smallint,20)' ,'apercentile(t1.q_tinyint,20)' ,'apercentile(t1.q_float,20)' ,'apercentile(t1.q_double,20)' , + 'last_row(t1.q_int)' , 'last_row(t1.q_bigint)' , 'last_row(t1.q_smallint)' , 'last_row(t1.q_tinyint)' , 'last_row(t1.q_float)' , + 'last_row(t1.q_double)' , 'last_row(t1.q_bool)' ,'last_row(t1.q_binary)' ,'last_row(t1.q_nchar)' ,'last_row(t1.q_ts)' , + 'apercentile(t2.q_int,20)' , 'apercentile(t2.q_bigint,20)' ,'apercentile(t2.q_smallint,20)' ,'apercentile(t2.q_tinyint,20)' ,'apercentile(t2.q_float,20)' ,'apercentile(t2.q_double,20)' , + 'last_row(t2.q_int)' , 'last_row(t2.q_bigint)' , 'last_row(t2.q_smallint)' , 'last_row(t2.q_tinyint)' , 'last_row(t2.q_float)' , + 'last_row(t2.q_double)' , 'last_row(t2.q_bool)' ,'last_row(t2.q_binary)' ,'last_row(t2.q_nchar)' ,'last_row(t2.q_ts)'] + calc_select_in_j = ['min(t1.q_int)' , 'min(t1.q_bigint)' , 'min(t1.q_smallint)' , 'min(t1.q_tinyint)' , 'min(t1.q_float)' ,'min(t1.q_double)' , 'max(t1.q_int)' , 'max(t1.q_bigint)' , 'max(t1.q_smallint)' , 'max(t1.q_tinyint)' ,'max(t1.q_float)' ,'max(t1.q_double)' , 'apercentile(t1.q_int,20)' , 'apercentile(t1.q_bigint,20)' ,'apercentile(t1.q_smallint,20)' ,'apercentile(t1.q_tinyint,20)' ,'apercentile(t1.q_float,20)' ,'apercentile(t1.q_double,20)' , @@ -401,8 +496,7 @@ class TDTestCase: 'max(t2.q_int)' , 'max(t2.q_bigint)' , 'max(t2.q_smallint)' , 'max(t2.q_tinyint)' ,'max(t2.q_float)' ,'max(t2.q_double)' , 'apercentile(t2.q_int,20)' , 'apercentile(t2.q_bigint,20)' ,'apercentile(t2.q_smallint,20)' ,'apercentile(t2.q_tinyint,20)' ,'apercentile(t2.q_float,20)' ,'apercentile(t2.q_double,20)' , 'last_row(t2.q_int)' , 'last_row(t2.q_bigint)' , 'last_row(t2.q_smallint)' , 'last_row(t2.q_tinyint)' , 'last_row(t2.q_float)' , - 'last_row(t2.q_double)' , 'last_row(t2.q_bool)' ,'last_row(t2.q_binary)' ,'last_row(t2.q_nchar)' ,'last_row(t2.q_ts)'] - + 'last_row(t2.q_double)' , 'last_row(t2.q_bool)' ,'last_row(t2.q_binary)' ,'last_row(t2.q_nchar)' ,'last_row(t2.q_ts)'] calc_select_all_j = calc_select_in_ts_j + calc_select_in_j calc_select_regular_j = [ 'PERCENTILE(t1.q_int,10)' ,'PERCENTILE(t1.q_bigint,20)' , 'PERCENTILE(t1.q_smallint,30)' ,'PERCENTILE(t1.q_tinyint,40)' ,'PERCENTILE(t1.q_float,50)' ,'PERCENTILE(t1.q_double,60)' , @@ -465,6 +559,8 @@ class TDTestCase: calc_calculate_all = ['SPREAD(ts)' , 'SPREAD(q_ts)' , 'SPREAD(q_int)' ,'SPREAD(q_bigint)' , 'SPREAD(q_smallint)' ,'SPREAD(q_tinyint)' ,'SPREAD(q_float)' ,'SPREAD(q_double)' , '(SPREAD(q_int) + SPREAD(q_bigint))' , '(SPREAD(q_smallint) - SPREAD(q_float))', '(SPREAD(q_double) * SPREAD(q_tinyint))' , '(SPREAD(q_double) / SPREAD(q_float))'] calc_calculate_regular = ['DIFF(q_int)' ,'DIFF(q_bigint)' , 'DIFF(q_smallint)' ,'DIFF(q_tinyint)' ,'DIFF(q_float)' ,'DIFF(q_double)' , + 'DIFF(q_int,0)' ,'DIFF(q_bigint,0)' , 'DIFF(q_smallint,0)' ,'DIFF(q_tinyint,0)' ,'DIFF(q_float,0)' ,'DIFF(q_double,0)' , + 'DIFF(q_int,1)' ,'DIFF(q_bigint,1)' , 'DIFF(q_smallint,1)' ,'DIFF(q_tinyint,1)' ,'DIFF(q_float,1)' ,'DIFF(q_double,1)' , 'DERIVATIVE(q_int,15s,0)' , 'DERIVATIVE(q_bigint,10s,1)' , 'DERIVATIVE(q_smallint,20s,0)' ,'DERIVATIVE(q_tinyint,10s,1)' ,'DERIVATIVE(q_float,6s,0)' ,'DERIVATIVE(q_double,3s,1)' ] calc_calculate_groupbytbname = calc_calculate_regular @@ -475,8 +571,12 @@ class TDTestCase: '(SPREAD(t2.q_int) + SPREAD(t2.q_bigint))' , '(SPREAD(t2.q_smallint) - SPREAD(t2.q_float))', '(SPREAD(t2.q_double) * SPREAD(t2.q_tinyint))' , '(SPREAD(t2.q_double) / SPREAD(t2.q_tinyint))', '(SPREAD(t1.q_int) + SPREAD(t1.q_smallint))' , '(SPREAD(t2.q_smallint) - SPREAD(t2.q_float))', '(SPREAD(t1.q_double) * SPREAD(t1.q_tinyint))' , '(SPREAD(t1.q_double) / SPREAD(t1.q_float))'] calc_calculate_regular_j = ['DIFF(t1.q_int)' ,'DIFF(t1.q_bigint)' , 'DIFF(t1.q_smallint)' ,'DIFF(t1.q_tinyint)' ,'DIFF(t1.q_float)' ,'DIFF(t1.q_double)' , + 'DIFF(t1.q_int,0)' ,'DIFF(t1.q_bigint,0)' , 'DIFF(t1.q_smallint,0)' ,'DIFF(t1.q_tinyint,0)' ,'DIFF(t1.q_float,0)' ,'DIFF(t1.q_double,0)' , + 'DIFF(t1.q_int,1)' ,'DIFF(t1.q_bigint,1)' , 'DIFF(t1.q_smallint,1)' ,'DIFF(t1.q_tinyint,1)' ,'DIFF(t1.q_float,1)' ,'DIFF(t1.q_double,1)' , 'DERIVATIVE(t1.q_int,15s,0)' , 'DERIVATIVE(t1.q_bigint,10s,1)' , 'DERIVATIVE(t1.q_smallint,20s,0)' ,'DERIVATIVE(t1.q_tinyint,10s,1)' ,'DERIVATIVE(t1.q_float,6s,0)' ,'DERIVATIVE(t1.q_double,3s,1)' , 'DIFF(t2.q_int)' ,'DIFF(t2.q_bigint)' , 'DIFF(t2.q_smallint)' ,'DIFF(t2.q_tinyint)' ,'DIFF(t2.q_float)' ,'DIFF(t2.q_double)' , + 'DIFF(t2.q_int,0)' ,'DIFF(t2.q_bigint,0)' , 'DIFF(t2.q_smallint,0)' ,'DIFF(t2.q_tinyint,0)' ,'DIFF(t2.q_float,0)' ,'DIFF(t2.q_double,0)' , + 'DIFF(t2.q_int,1)' ,'DIFF(t2.q_bigint,1)' , 'DIFF(t2.q_smallint,1)' ,'DIFF(t2.q_tinyint,1)' ,'DIFF(t2.q_float,1)' ,'DIFF(t2.q_double,1)' , 'DERIVATIVE(t2.q_int,15s,0)' , 'DERIVATIVE(t2.q_bigint,10s,1)' , 'DERIVATIVE(t2.q_smallint,20s,0)' ,'DERIVATIVE(t2.q_tinyint,10s,1)' ,'DERIVATIVE(t2.q_float,6s,0)' ,'DERIVATIVE(t2.q_double,3s,1)' ] calc_calculate_groupbytbname_j = calc_calculate_regular_j @@ -503,7 +603,7 @@ class TDTestCase: tdSql.checkRows(100) #1 outer union not support - self.dropandcreateDB_random("%s" %db, 1) + #self.dropandcreateDB_random("%s" %db, 1) tdSql.query("select 1-2 from stable_1;") for i in range(self.fornum): #sql = "select ts , * from ( select " @@ -528,7 +628,7 @@ class TDTestCase: tdSql.query(sql) tdSql.checkRows(100) - self.dropandcreateDB_random("%s" %db, 1) + #self.dropandcreateDB_random("%s" %db, 1) tdSql.query("select 1-2 from stable_1;") for i in range(self.fornum): #sql = "select ts , * from ( select " @@ -725,8 +825,7 @@ class TDTestCase: sql += "ts from stable_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(order_where) - sql += ")" - sql += " %s " % random.choice(unionall_or_union) + sql += ") union " sql += "select ts from ( select " sql += "%s, " % random.choice(s_r_select) sql += "%s, " % random.choice(q_select) @@ -736,10 +835,29 @@ class TDTestCase: sql += ")" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.error(sql) - #TD-15610 tdSql.query(sql) - # tdSql.checkRows(100) - + tdSql.query(sql) + tdSql.checkRows(200) + for i in range(self.fornum): + #sql = "select ts , * from ( select " + sql = "select ts from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ") union all " + sql += "select ts from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from stable_2 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(order_where) + sql += ")" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + tdSql.checkRows(400) + #3 inter union not support tdSql.query("select 3-4 from stable_1;") for i in range(self.fornum): @@ -761,7 +879,7 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.error(sql) - #join:TD-6020\TD-6155 select * from (select column form stable1，stable2 where t1.ts=t2.ts and <\>\in\and\or order by) + #join:select * from (select column form stable1，stable2 where t1.ts=t2.ts and <\>\in\and\or order by) tdSql.query("select 3-5 from stable_1;") for i in range(self.fornum): sql = "select * from ( select t1.ts ," @@ -874,7 +992,20 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - tdSql.error(sql) + tdSql.error(sql) #distinct 和 order by 不能混合使用 + tdSql.query("select 7-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(dq_select) + sql += " from stable_1 where " + sql += "%s " % random.choice(qt_where) + #sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice([limit_where[0] , limit_where[1]] ) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + tdSql.checkRows(1) #calc_select,TWA/Diff/Derivative/Irate are not allowed to apply to super table directly #8 select * from (select ts,calc form ragular_table where <\>\in\and\or order by ) @@ -883,7 +1014,7 @@ class TDTestCase: tdSql.query("select 8-1 from stable_1;") for i in range(self.fornum): sql = "select * from ( select ts ," - sql += "%s " % random.choice(calc_select_in_ts) + sql += "%s " % random.choice(calc_select_support_ts) sql += "from regular_table_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(order_where) @@ -891,12 +1022,36 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + tdSql.query(sql) # 聚合函数不在可以和ts一起使用了 DB error: Not a single-group group function + tdSql.query("select 8-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_select_not_support_ts) + sql += "from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #TD-15651 tdSql.query(sql) # 聚合函数不在可以和ts一起使用了 DB error: Not a single-group group function + + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_select_in_ts) + sql += "from regular_table_1 where " + sql += "%s " % random.choice(q_where) + #sql += "%s " % random.choice(order_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + ##top返回结果有问题nest.sql tdSql.checkRows(1) tdSql.query("select 8-2 from stable_1;") for i in range(self.fornum): - sql = "select * from ( select " - sql += "%s " % random.choice(calc_select_in_ts_j) + sql = "select * from ( select t1.ts, " + sql += "%s " % random.choice(calc_select_in_support_ts_j) sql += "from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) sql += "%s " % random.choice(order_u_where) @@ -904,28 +1059,36 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) - - tdSql.query("select 8-3 from stable_1;") + tdSql.query(sql)# 聚合函数不在可以和ts一起使用了 DB error: Not a single-group group function for i in range(self.fornum): sql = "select * from ( select " - sql += "%s " % random.choice(calc_select_in_ts_j) + sql += "%s " % random.choice(calc_select_in_not_support_ts_j) sql += "from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " - sql += "%s " % random.choice(q_u_or_where) - sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(q_u_where) sql += "%s " % random.choice(limit1_where) sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.error(sql) - #tdSql.query(sql) + #TD-15651 tdSql.query(sql) + ##top返回结果有问题 tdSql.checkRows(1) #9 select * from (select ts,calc form stable where <\>\in\and\or order by ) # self.dropandcreateDB_random("%s" %db, 1) tdSql.query("select 9-1 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_select_not_support_ts) + sql += "from stable_1 where " + sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #TD-15651 tdSql.query(sql) + tdSql.query("select 9-2 from stable_1;") for i in range(self.fornum): sql = "select * from ( select ts ," - sql += "%s " % random.choice(calc_select_in_ts) + sql += "%s " % random.choice(calc_select_support_ts) sql += "from stable_1 where " sql += "%s " % random.choice(qt_where) sql += "%s " % random.choice(order_where) @@ -933,35 +1096,33 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) - - tdSql.query("select 9-2 from stable_1;") - for i in range(self.fornum): - sql = "select * from ( select " - sql += "%s " % random.choice(calc_select_in_ts_j) - sql += "from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " - sql += "%s " % random.choice(t_join_where) - sql += "%s " % random.choice(order_u_where) - sql += "%s " % random.choice(limit1_where) - sql += ") ;" - tdLog.info(sql) - tdLog.info(len(sql)) - #tdSql.query(sql) - + tdSql.query(sql) + tdSql.query("select 9-3 from stable_1;") for i in range(self.fornum): sql = "select * from ( select " - sql += "%s " % random.choice(calc_select_in_ts_j) + sql += "%s " % random.choice(calc_select_in_not_support_ts_j) sql += "from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " - sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(t_join_where) + sql += " and %s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + #TD-15651 tdSql.query(sql) + tdSql.query("select 9-4 from stable_1;") + for i in range(self.fornum): + sql = "select * from ( select t1.ts," + sql += "%s " % random.choice(calc_select_in_support_ts_j) + sql += "from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_join_where) + sql += " and %s " % random.choice(qt_u_or_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit1_where) sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.error(sql) - - #functions or others can not be mixed up ,calc out select not use with ts + tdSql.query(sql) #10 select calc from (select * form regualr_table where <\>\in\and\or order by ) tdSql.query("select 10-1 from stable_1;") @@ -975,8 +1136,8 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - # TD-15503 tdSql.query(sql) - # tdSql.checkRows(1) + tdSql.query(sql) + tdSql.checkRows(1) #10-1 select calc from (select * form regualr_table where <\>\in\and\or order by ) # rsDn = self.restartDnodes() @@ -986,44 +1147,47 @@ class TDTestCase: for i in range(self.fornum): sql = "select " sql += "%s " % random.choice(calc_select_all) - sql += "as calc10_1 from ( select * from regular_table_1 where " + sql += "as calc10_2 from ( select * from regular_table_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(order_desc_where) sql += "%s " % random.choice(limit1_where) sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.query(sql) + #TD-15651 tdSql.query(sql) # tdSql.checkRows(1) #10-2 select calc from (select * form regualr_tables where <\>\in\and\or order by ) tdSql.query("select 10-3 from stable_1;") for i in range(self.fornum): sql = "select " - sql += "%s as calc10_1 " % random.choice(calc_select_all) + sql += "%s as calc10_3 " % random.choice(calc_select_all) sql += " from ( select * from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) + sql += " and %s " % random.choice(q_u_or_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s ;" % random.choice(limit_u_where) + sql += "%s ;" % random.choice(limit1_where) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #TD-15651 tdSql.query(sql) tdSql.query("select 10-4 from stable_1;") for i in range(self.fornum): sql = "select " - sql += "%s as calc10_1 " % random.choice(calc_select_all) + sql += "%s as calc10_4 " % random.choice(calc_select_all) sql += " from ( select * from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_or_where) + sql += " and %s " % random.choice(q_u_or_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s ;" % random.choice(limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.error(sql) + #TD-15651 tdSql.query(sql) + # tdSql.checkRows(1) #11 select calc from (select * form stable where <\>\in\and\or order by limit ) tdSql.query("select 11-1 from stable_1;") @@ -1037,8 +1201,8 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.query(sql) - # tdSql.checkRows(1) + tdSql.query(sql) + tdSql.checkRows(1) #11-1 select calc from (select * form stable where <\>\in\and\or order by limit ) tdSql.query("select 11-2 from stable_1;") @@ -1052,8 +1216,8 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.query(sql) - # tdSql.checkRows(1) + #TD-15651 tdSql.query(sql) + #不好计算结果 tdSql.checkRows(1) #11-2 select calc from (select * form stables where <\>\in\and\or order by limit ) tdSql.query("select 11-3 from stable_1;") @@ -1068,7 +1232,7 @@ class TDTestCase: sql += "%s ;" % random.choice(limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15493 tdSql.query(sql) + #TD-15651 tdSql.query(sql) tdSql.query("select 11-4 from stable_1;") for i in range(self.fornum): @@ -1082,7 +1246,8 @@ class TDTestCase: sql += "%s ;" % random.choice(limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.error(sql) + tdLog.info(len(sql)) + #TD-15651 tdSql.query(sql) #12 select calc-diff from (select * form regualr_table where <\>\in\and\or order by limit ) ##self.dropandcreateDB_random("%s" %db, 1) @@ -1097,7 +1262,7 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.query(sql) + ##目前derivative不支持 tdSql.query(sql) # tdSql.checkRows(1) tdSql.query("select 12-2 from stable_1;") @@ -1111,8 +1276,8 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) - #tdSql.checkRows(1) + #目前derivative不支持 tdSql.query(sql) + # tdSql.checkRows(1) tdSql.query("select 12-2.2 from stable_1;") for i in range(self.fornum): @@ -1125,7 +1290,7 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.error(sql) + #目前derivative不支持 tdSql.query(sql) #12-1 select calc-diff from (select * form stable where <\>\in\and\or order by limit ) tdSql.query("select 12-3 from stable_1;") @@ -1142,7 +1307,7 @@ class TDTestCase: sql += " ;" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #目前derivative不支持 tdSql.query(sql) tdSql.query("select 12-4 from stable_1;") #join query does not support group by @@ -1151,14 +1316,14 @@ class TDTestCase: sql += "%s " % random.choice(calc_calculate_regular_j) sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(t_join_where) - sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(group_where_j) sql += ") " - sql += "%s " % random.choice(order_desc_where) + #sql += "%s " % random.choice(order_desc_where) sql += "%s " % random.choice([limit_where[2] , limit_where[3]] ) sql += " ;" tdLog.info(sql) tdLog.info(len(sql)) - tdSql.error(sql) + #tdSql.query(sql) 目前de函数不支持，另外看看需要不需要将group by和pari by分开 tdSql.query("select 12-5 from stable_1;") #join query does not support group by @@ -1167,14 +1332,14 @@ class TDTestCase: sql += "%s " % random.choice(calc_calculate_regular_j) sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(qt_u_or_where) - sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(group_where_j) sql += ") " sql += "%s " % random.choice(order_desc_where) sql += "%s " % random.choice([limit_where[2] , limit_where[3]] ) sql += " ;" tdLog.info(sql) tdLog.info(len(sql)) - tdSql.error(sql) + #derivative not support tdSql.query(sql) #13 select calc-diff as diffns from (select * form stable where <\>\in\and\or order by limit ) @@ -1189,7 +1354,7 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.error(sql) + #derivative not support tdSql.query(sql) #14 select * from (select calc_aggregate_alls as agg from stable where <\>\in\and\or group by order by slimit soffset ) # TD-5955 select * from ( select count (q_double) from stable_1 where t_bool = true or t_bool = false group by loc order by ts asc slimit 1 ) ; @@ -1207,8 +1372,8 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) - #tdSql.checkRows(1) + #TD-15678 tdSql.query(sql) + # tdSql.checkRows(1) # error group by in out query tdSql.query("select 14-2 from stable_1;") @@ -1226,10 +1391,11 @@ class TDTestCase: sql += ") " sql += "%s " % random.choice(group_where) tdLog.info(sql) - tdLog.info(len(sql)) - # tdSql.error(sql) + tdLog.info(len(sql)) + #TD-15678 tdSql.query(sql) + # tdSql.checkRows(1) - #14-2 TD-6426 select * from (select calc_aggregate_all_js as agg from stables where <\>\in\and\or group by order by slimit soffset ) + #14-2 select * from (select calc_aggregate_all_js as agg from stables where <\>\in\and\or group by order by slimit soffset ) tdSql.query("select 14-3 from stable_1;") for i in range(self.fornum): sql = "select * from ( select " @@ -1238,13 +1404,13 @@ class TDTestCase: sql += "%s " % random.choice(calc_aggregate_all_j) sql += " as calc14_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(t_join_where) - sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(partiton_where_j) sql += "%s " % random.choice(slimit1_where) sql += ") " sql += "%s ;" % random.choice(limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 14-4 from stable_1;") for i in range(self.fornum): @@ -1254,13 +1420,13 @@ class TDTestCase: sql += "%s " % random.choice(calc_aggregate_all_j) sql += " as calc14_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(qt_u_or_where) - sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(partiton_where_j) sql += "%s " % random.choice(slimit1_where) sql += ") " sql += "%s ;" % random.choice(limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + tdSql.query(sql) #15 TD-6320 select * from (select calc_aggregate_regulars as agg from regular_table where <\>\in\and\or order by slimit soffset ) tdSql.query("select 15-1 from stable_1;") @@ -1271,11 +1437,11 @@ class TDTestCase: sql += "%s " % random.choice(calc_aggregate_regular) sql += " as calc15_3 from regular_table_1 where " sql += "%s " % random.choice(q_where) - sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice(group_where_regular) sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.query(sql) + #tdSql.query(sql) #Invalid function name: twa' # tdSql.checkRows(1) tdSql.query("select 15-2 from stable_1;") @@ -1286,13 +1452,13 @@ class TDTestCase: sql += "%s " % random.choice(calc_aggregate_regular_j) sql += " as calc15_3 from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) - sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(group_where_regular_j) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s ;" % random.choice(limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #tdSql.query(sql) #Invalid function name: twa' tdSql.query("select 15-2.2 from stable_1;") for i in range(self.fornum): @@ -1302,13 +1468,13 @@ class TDTestCase: sql += "%s " % random.choice(calc_aggregate_regular_j) sql += " as calc15_3 from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_or_where) - sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(group_where_regular_j) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s ;" % random.choice(limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.error(sql) + #tdSql.query(sql) #Invalid function name: twa' rsDn = self.restartDnodes() tdSql.query("select 15-3 from stable_1;") @@ -1327,7 +1493,7 @@ class TDTestCase: sql += "%s " % random.choice(limit_where) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #tdSql.query(sql) #Invalid function name: twa',可能还的去掉order by tdSql.query("select 15-4 from stable_1;") for i in range(self.fornum): @@ -1337,15 +1503,15 @@ class TDTestCase: sql += "%s " % random.choice(calc_aggregate_groupbytbname_j) sql += " as calc15_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(t_join_where) - sql += "%s " % random.choice(group_where) - sql += "%s " % random.choice(having_support) - sql += "%s " % random.choice(orders_desc_where) + sql += "%s " % random.choice(group_where_j) + sql += "%s " % random.choice(having_support_j) + #sql += "%s " % random.choice(orders_desc_where) sql += ") " sql += "order by calc15_1 " sql += "%s " % random.choice(limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #tdSql.query(sql) #'Invalid function name: irate' tdSql.query("select 15-4.2 from stable_1;") for i in range(self.fornum): @@ -1355,15 +1521,15 @@ class TDTestCase: sql += "%s " % random.choice(calc_aggregate_groupbytbname_j) sql += " as calc15_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(qt_u_or_where) - sql += "%s " % random.choice(group_where) - sql += "%s " % random.choice(having_support) + sql += "%s " % random.choice(group_where_j) + sql += "%s " % random.choice(having_support_j) sql += "%s " % random.choice(orders_desc_where) sql += ") " sql += "order by calc15_1 " sql += "%s " % random.choice(limit_u_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15678 #tdSql.query(sql) tdSql.query("select 15-5 from stable_1;") for i in range(self.fornum): @@ -1373,13 +1539,13 @@ class TDTestCase: sql += "%s " % random.choice(calc_aggregate_groupbytbname) sql += " as calc15_3 from stable_1 where " sql += "%s " % random.choice(q_where) - sql += "%s " % random.choice(orders_desc_where) + sql += "%s " % random.choice(group_where) sql += ") " sql += "order by calc15_1 " sql += "%s " % random.choice(limit_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #tdSql.query(sql) #'Invalid function name: irate' #16 select * from (select calc_aggregate_regulars as agg from regular_table where <\>\in\and\or order by limit offset ) #self.dropandcreateDB_random("%s" %db, 1) @@ -1392,14 +1558,13 @@ class TDTestCase: sql += " from stable_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(group_where) - sql += "%s " % random.choice(having_support) + #sql += "%s " % random.choice(having_support)having和 partition不能混合使用 sql += ") " sql += "order by calc16_0 " sql += "%s " % random.choice(limit1_where) tdLog.info(sql) tdLog.info(len(sql)) - # TD-15497 tdSql.query(sql) - # tdSql.checkRows(1) + #TD-15651 tdSql.query(sql) tdSql.query("select 16-2 from stable_1;") for i in range(self.fornum): @@ -1414,7 +1579,7 @@ class TDTestCase: sql += "%s " % random.choice(limit1_where) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15493 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 16-2.2 from stable_1;") for i in range(self.fornum): @@ -1428,7 +1593,7 @@ class TDTestCase: sql += "%s " % random.choice(limit1_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + tdSql.query(sql) tdSql.query("select 16-3 from stable_1;") for i in range(self.fornum): @@ -1440,8 +1605,7 @@ class TDTestCase: sql += "%s " % random.choice(limit1_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.query(sql) - # tdSql.checkRows(1) + #tdSql.query(sql)#Invalid function name: derivative' tdSql.query("select 16-4 from stable_1;") for i in range(self.fornum): @@ -1453,8 +1617,7 @@ class TDTestCase: sql += "%s " % random.choice(limit1_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.query(sql) - # tdSql.checkRows(1) + #tdSql.query(sql)#Invalid function name: derivative' tdSql.query("select 16-4.2 from stable_1;") for i in range(self.fornum): @@ -1466,7 +1629,7 @@ class TDTestCase: sql += "%s " % random.choice(limit1_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #tdSql.query(sql)#Invalid function name: derivative' tdSql.query("select 16-5 from stable_1;") for i in range(self.fornum): @@ -1477,13 +1640,13 @@ class TDTestCase: sql += " from stable_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(group_where) - sql += "%s " % random.choice(having_support) + #sql += "%s " % random.choice(having_support) sql += ") " sql += "order by calc16_1 " sql += "%s " % random.choice(limit1_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + # tdSql.query(sql) tdSql.query("select 16-6 from stable_1;") for i in range(self.fornum): @@ -1496,8 +1659,7 @@ class TDTestCase: sql += "%s " % random.choice(limit1_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.query(sql) - # tdSql.checkRows(1) + #Invalid function name: derivative' tdSql.query(sql) tdSql.query("select 16-7 from stable_1;") for i in range(self.fornum): @@ -1509,7 +1671,7 @@ class TDTestCase: sql += "%s " % random.choice(limit1_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #Invalid function name: derivative' tdSql.query(sql) tdSql.query("select 16-8 from stable_1;") for i in range(self.fornum): @@ -1521,7 +1683,7 @@ class TDTestCase: sql += "%s " % random.choice(limit1_where) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #Invalid function name: derivative' tdSql.query(sql) #17 select apercentile from (select calc_aggregate_alls form regualr_table or stable where <\>\in\and\or interval_sliding group by having order by limit offset )interval_sliding #self.dropandcreateDB_random("%s" %db, 1) @@ -1534,16 +1696,16 @@ class TDTestCase: sql += "%s as cal17_2 " % random.choice(calc_aggregate_all) sql += " from stable_1 where " sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(partiton_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(group_where) - sql += "%s " % random.choice(having_support) - sql += "%s " % random.choice(order_where) + #sql += "%s " % random.choice(having_support) + #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 17-2 from stable_1;") for i in range(self.fornum): @@ -1555,13 +1717,13 @@ class TDTestCase: sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(t_join_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #Column ambiguously defined: ts tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 17-2.2 from stable_1;") for i in range(self.fornum): @@ -1573,15 +1735,15 @@ class TDTestCase: sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(qt_u_or_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15721 tdSql.query(sql) - rsDn = self.restartDnodes() + self.restartDnodes() tdSql.query("select 17-3 from stable_1;") for i in range(self.fornum): #this is having_tagnot_support , because tag-select cannot mix with last_row... @@ -1590,16 +1752,16 @@ class TDTestCase: sql += "%s as cal17_2 " % random.choice(calc_aggregate_all) sql += " from stable_1 where " sql += "%s " % random.choice(q_where) + sql += "%s " % random.choice(partiton_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(group_where) - sql += "%s " % random.choice(having_tagnot_support) - sql += "%s " % random.choice(order_where) + #sql += "%s " % random.choice(having_tagnot_support) + #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 17-4 from stable_1;") for i in range(self.fornum): @@ -1610,13 +1772,13 @@ class TDTestCase: sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(t_join_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 17-4.2 from stable_1;") for i in range(self.fornum): @@ -1627,13 +1789,13 @@ class TDTestCase: sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(qt_u_or_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 17-5 from stable_1;") for i in range(self.fornum): @@ -1643,16 +1805,16 @@ class TDTestCase: sql += "%s as cal17_2 " % random.choice(calc_aggregate_all) sql += " from stable_1 where " sql += "%s " % random.choice(qt_where) + sql += "%s " % random.choice(partiton_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(group_where) - sql += "%s " % random.choice(having_not_support) - sql += "%s " % random.choice(order_where) + # sql += "%s " % random.choice(having_not_support) + # sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 17-6 from stable_1;") for i in range(self.fornum): @@ -1662,13 +1824,13 @@ class TDTestCase: sql += " from stable_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(order_where) + #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 17-7 from stable_1;") for i in range(self.fornum): @@ -1678,13 +1840,13 @@ class TDTestCase: sql += " from stable_1_1 t1, stable_1_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit1_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 17-7.2 from stable_1;") for i in range(self.fornum): @@ -1694,13 +1856,13 @@ class TDTestCase: sql += " from stable_1_1 t1, stable_1_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_or_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit1_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15721 tdSql.query(sql) self.restartDnodes() tdSql.query("select 17-8 from stable_1;") @@ -1711,13 +1873,13 @@ class TDTestCase: sql += " from regular_table_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(order_where) + #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 17-9 from stable_1;") for i in range(self.fornum): @@ -1727,13 +1889,13 @@ class TDTestCase: sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 17-10 from stable_1;") for i in range(self.fornum): @@ -1743,13 +1905,13 @@ class TDTestCase: sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_or_where) sql += "%s " % random.choice(interval_sliding) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15721 tdSql.query(sql) #18 select apercentile from (select calc_aggregate_alls form regualr_table or stable where <\>\in\and\or session order by limit )interval_sliding tdSql.query("select 18-1 from stable_1;") @@ -1760,14 +1922,14 @@ class TDTestCase: sql += " from regular_table_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(session_where) - sql += "%s " % random.choice(fill_where) - sql += "%s " % random.choice(order_where) + #sql += "%s " % random.choice(fill_where) + #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 18-2 from stable_1;") for i in range(self.fornum): @@ -1777,13 +1939,13 @@ class TDTestCase: sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 18-2.2 from stable_1;") for i in range(self.fornum): @@ -1793,13 +1955,13 @@ class TDTestCase: sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_or_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15721 tdSql.query(sql) self.restartDnodes() tdSql.query("select 18-3 from stable_1;") @@ -1810,14 +1972,14 @@ class TDTestCase: sql += " from stable_1_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(session_where) - sql += "%s " % random.choice(fill_where) - sql += "%s " % random.choice(order_where) + #sql += "%s " % random.choice(fill_where) + #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 18-4 from stable_1;") for i in range(self.fornum): @@ -1827,13 +1989,13 @@ class TDTestCase: sql += " from stable_1_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.query(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 18-4.2 from stable_1;") for i in range(self.fornum): @@ -1843,13 +2005,13 @@ class TDTestCase: sql += " from stable_1_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_or_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - tdSql.error(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 18-5 from stable_1;") for i in range(self.fornum): @@ -1859,14 +2021,14 @@ class TDTestCase: sql += " from stable_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(session_where) - sql += "%s " % random.choice(fill_where) - sql += "%s " % random.choice(order_where) + #sql += "%s " % random.choice(fill_where) + #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 18-6 from stable_1;") for i in range(self.fornum): @@ -1876,13 +2038,13 @@ class TDTestCase: sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(t_join_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 18-7 from stable_1;") for i in range(self.fornum): @@ -1892,13 +2054,13 @@ class TDTestCase: sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(qt_u_or_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15721 tdSql.query(sql) #19 select apercentile from (select calc_aggregate_alls form regualr_table or stable where <\>\in\and\or session order by limit )interval_sliding #self.dropandcreateDB_random("%s" %db, 1) @@ -1910,15 +2072,14 @@ class TDTestCase: sql += " from regular_table_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(state_window) - sql += "%s " % random.choice(order_where) + #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - #TD-15545 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 19-2 from stable_1;") - #TD-6435 state_window not support join for i in range(self.fornum): sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all_j) @@ -1926,12 +2087,12 @@ class TDTestCase: sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) sql += "%s " % random.choice(state_u_window) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + tdSql.query(sql) tdSql.query("select 19-2.2 from stable_1;") for i in range(self.fornum): @@ -1941,12 +2102,12 @@ class TDTestCase: sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_or_where) sql += "%s " % random.choice(state_u_window) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + tdSql.query(sql) tdSql.query("select 19-3 from stable_1;") for i in range(self.fornum): @@ -1956,12 +2117,12 @@ class TDTestCase: sql += " from stable_1_1 where " sql += "%s " % random.choice(q_where) sql += "%s " % random.choice(state_window) - sql += "%s " % random.choice(order_where) + #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - #不支持order by tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 19-4 from stable_1;") for i in range(self.fornum): @@ -1971,12 +2132,12 @@ class TDTestCase: sql += " from stable_1_1 t1, stable_1_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) #sql += "%s " % random.choice(state_window) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 19-4.2 from stable_1;") for i in range(self.fornum): @@ -1985,12 +2146,12 @@ class TDTestCase: sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) sql += " from stable_1_1 t1, stable_1_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_or_where) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + tdSql.query(sql) tdSql.query("select 19-5 from stable_1;") for i in range(self.fornum): @@ -2006,7 +2167,7 @@ class TDTestCase: sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + tdSql.error(sql) #'STATE_WINDOW not support for super table query' tdSql.query("select 19-6 from stable_1;") for i in range(self.fornum): @@ -2016,13 +2177,13 @@ class TDTestCase: sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) #sql += "%s " % random.choice(state_window) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15721 tdSql.query(sql) tdSql.query("select 19-7 from stable_1;") for i in range(self.fornum): @@ -2031,13 +2192,13 @@ class TDTestCase: sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(qt_u_or_where) - sql += "%s " % random.choice(order_u_where) + #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #TD-15721 tdSql.query(sql) #20 select * from (select calc_select_fills form regualr_table or stable where <\>\in\and\or fill_where group by order by limit offset ) #self.dropandcreateDB_random("%s" %db, 1) @@ -2056,11 +2217,10 @@ class TDTestCase: sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #interp不支持 tdSql.query(sql) rsDn = self.restartDnodes() tdSql.query("select 20-2 from stable_1;") - #TD-6438 for i in range(self.fornum): sql = "select * from ( select " sql += "%s , " % random.choice(calc_select_fill_j) @@ -2074,7 +2234,7 @@ class TDTestCase: sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #interp不支持 tdSql.query(sql) tdSql.query("select 20-2.2 from stable_1;") for i in range(self.fornum): @@ -2090,7 +2250,7 @@ class TDTestCase: sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #interp不支持 tdSql.query(sql) tdSql.query("select 20-3 from stable_1;") for i in range(self.fornum): @@ -2122,7 +2282,7 @@ class TDTestCase: sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + #interp不支持 tdSql.query(sql) tdSql.query("select 20-4.2 from stable_1;") for i in range(self.fornum): @@ -2139,7 +2299,7 @@ class TDTestCase: sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + ##interp不支持 tdSql.error(sql) tdSql.query("select 20-5 from stable_1;") for i in range(self.fornum): @@ -2155,7 +2315,7 @@ class TDTestCase: sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + ##interp不支持 tdSql.query(sql) tdSql.query("select 20-6 from stable_1;") for i in range(self.fornum): @@ -2171,40 +2331,77 @@ class TDTestCase: sql += ") " tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + ##interp不支持 tdSql.query(sql) - # error #1 select * from (select * from (select * form regular_table where <\>\in\and\or order by limit )) tdSql.query("select 1-1 from stable_1;") - for i in range(self.fornum): - sql = "select * , ts from ( select * from ( select " + for i in range(self.fornum): + # sql_start = "select * from ( " + # sql_end = ")" + for_num = random.randint(1, 15); + sql = "select * from (" * for_num + sql += "select * from ( select * from ( select " sql += "%s, " % random.choice(s_r_select) sql += "%s, " % random.choice(q_select) sql += "ts from regular_table_1 where " sql += "%s " % random.choice(q_where) - sql += ")) ;" + sql += ")) " + sql += ")" * for_num tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) - + tdSql.query(sql) + + for i in range(self.fornum): + # sql_start = "select * from ( " + # sql_end = ")" + for_num = random.randint(1, 10); + sql = "select ts from (" * for_num + sql += "select * from ( select * from ( select " + sql += "%s, " % random.choice(s_r_select) + sql += "%s, " % random.choice(q_select) + sql += "ts from regular_table_1 where " + sql += "%s " % random.choice(q_where) + sql += ")) " + sql += ")" * for_num + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + #2 select * from (select * from (select * form stable where <\>\in\and\or order by limit )) tdSql.query("select 2-1 from stable_1;") for i in range(self.fornum): - sql = "select * , ts from ( select * from ( select " + for_num = random.randint(1, 15); + sql = "select * from (" * for_num + sql += "select * from ( select * from ( select " sql += "%s, " % random.choice(s_s_select) sql += "%s, " % random.choice(qt_select) sql += "ts from stable_1 where " sql += "%s " % random.choice(q_where) - sql += ")) ;" + sql += ")) " + sql += ")" * for_num tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) - + tdSql.query(sql) + tdSql.query("select 2-1 from stable_1;") + for i in range(self.fornum): + for_num = random.randint(1, 10); + sql = "select ts from (" * for_num + sql += "select * from ( select * from ( select " + sql += "%s, " % random.choice(s_s_select) + sql += "%s, " % random.choice(qt_select) + sql += "ts from stable_1 where " + sql += "%s " % random.choice(q_where) + sql += ")) " + sql += ")" * for_num + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + #3 select ts ,calc from (select * form stable where <\>\in\and\or order by limit ) #self.dropandcreateDB_random("%s" %db, 1) tdSql.query("select 3-1 from stable_1;") for i in range(self.fornum): - sql = "select ts , " + sql = "select " sql += "%s " % random.choice(calc_calculate_regular) sql += " from ( select * from stable_1 where " sql += "%s " % random.choice(qt_where) @@ -2213,7 +2410,7 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + #'Invalid function name: derivative' tdSql.query(sql) #4 select * from (select calc form stable where <\>\in\and\or order by limit ) tdSql.query("select 4-1 from stable_1;") @@ -2222,12 +2419,12 @@ class TDTestCase: sql += "%s " % random.choice(calc_select_in_ts) sql += "from stable_1 where " sql += "%s " % random.choice(qt_where) - sql += "%s " % random.choice(order_desc_where) + #sql += "%s " % random.choice(order_desc_where) sql += "%s " % random.choice(limit_where) sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - #tdSql.query(sql) + tdSql.query(sql) #5 select ts ,tbname from (select * form stable where <\>\in\and\or order by limit ) tdSql.query("select 5-1 from stable_1;") @@ -2243,7 +2440,7 @@ class TDTestCase: sql += ") ;" tdLog.info(sql) tdLog.info(len(sql)) - # tdSql.error(sql) + tdSql.error(sql) #special sql tdSql.query("select 6-1 from stable_1;") From 3b4551cc269005934fd483d0c7b630084b10bf7a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 20 May 2022 07:07:51 +0000 Subject: [PATCH 07/23] fix: tdb concurrency --- source/libs/tdb/src/db/tdbPCache.c | 58 ++++++++++++++++++------------ source/libs/tdb/src/db/tdbPager.c | 13 ++++++- source/libs/tdb/src/inc/tdbInt.h | 14 ++++---- source/libs/tdb/test/tdbTest.cpp | 10 +++--- 4 files changed, 60 insertions(+), 35 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 22d7e8e5a4..5ba5d65815 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -14,6 +14,9 @@ */ #include "tdbInt.h" +#include +#include + struct SPCache { int szPage; int nPages; @@ -32,7 +35,6 @@ static inline uint32_t tdbPCachePageHash(const SPgid *pPgid) { uint32_t *t = (uint32_t *)((pPgid)->fileid); return (uint32_t)(t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno); } -#define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL) static int tdbPCacheOpenImpl(SPCache *pCache); static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn); @@ -80,16 +82,22 @@ int tdbPCacheClose(SPCache *pCache) { SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { SPage *pPage; + i32 nRef; tdbPCacheLock(pCache); pPage = tdbPCacheFetchImpl(pCache, pPgid, pTxn); if (pPage) { - tdbRefPage(pPage); + nRef = tdbRefPage(pPage); } + ASSERT(pPage); + tdbPCacheUnlock(pCache); + // printf("thread %" PRId64 " fetch page %d pgno %d pPage %p nRef %d\n", taosGetSelfPthreadId(), pPage->id, + // TDB_PAGE_PGNO(pPage), pPage, nRef); + return pPage; } @@ -98,30 +106,31 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { ASSERT(pTxn); + // nRef = tdbUnrefPage(pPage); + // ASSERT(nRef >= 0); + + tdbPCacheLock(pCache); nRef = tdbUnrefPage(pPage); - ASSERT(nRef >= 0); - if (nRef == 0) { - tdbPCacheLock(pCache); - // test the nRef again to make sure // it is safe th handle the page - nRef = tdbGetPageRef(pPage); - if (nRef == 0) { - if (pPage->isLocal) { - tdbPCacheUnpinPage(pCache, pPage); - } else { - if (TDB_TXN_IS_WRITE(pTxn)) { - // remove from hash - tdbPCacheRemovePageFromHash(pCache, pPage); - } - - tdbPageDestroy(pPage, pTxn->xFree, pTxn->xArg); + // nRef = tdbGetPageRef(pPage); + // if (nRef == 0) { + if (pPage->isLocal) { + tdbPCacheUnpinPage(pCache, pPage); + } else { + if (TDB_TXN_IS_WRITE(pTxn)) { + // remove from hash + tdbPCacheRemovePageFromHash(pCache, pPage); } - } - tdbPCacheUnlock(pCache); + tdbPageDestroy(pPage, pTxn->xFree, pTxn->xArg); + } + // } } + tdbPCacheUnlock(pCache); + // printf("thread %" PRId64 " relas page %d pgno %d pPage %p nRef %d\n", taosGetSelfPthreadId(), pPage->id, + // TDB_PAGE_PGNO(pPage), pPage, nRef); } int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->szPage; } @@ -223,6 +232,7 @@ static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { pCache->nRecyclable--; + // printf("pin page %d pgno %d pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); tdbTrace("pin page %d", pPage->id); } } @@ -243,6 +253,7 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { pCache->nRecyclable++; + // printf("unpin page %d pgno %d pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); tdbTrace("unpin page %d", pPage->id); } @@ -253,10 +264,12 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { h = tdbPCachePageHash(&(pPage->pgid)); for (ppPage = &(pCache->pgHash[h % pCache->nHash]); (*ppPage) && *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) ; - ASSERT(*ppPage == pPage); - *ppPage = pPage->pHashNext; - pCache->nPage--; + if (*ppPage) { + *ppPage = pPage->pHashNext; + pCache->nPage--; + // printf("rmv page %d to hash, pgno %d, pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); + } tdbTrace("remove page %d to hash", pPage->id); } @@ -271,6 +284,7 @@ static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { pCache->nPage++; + // printf("add page %d to hash, pgno %d, pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); tdbTrace("add page %d to hash", pPage->id); } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 4024cfe745..a74bb54883 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -265,6 +265,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa pgid.pgno = pgno; pPage = tdbPCacheFetch(pPager->pCache, &pgid, pTxn); if (pPage == NULL) { + ASSERT(0); return -1; } @@ -272,10 +273,14 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa if (!TDB_PAGE_INITIALIZED(pPage)) { ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage); if (ret < 0) { + ASSERT(0); return -1; } } + // printf("thread %" PRId64 " pager fetch page %d pgno %d ppage %p\n", taosGetSelfPthreadId(), pPage->id, + // TDB_PAGE_PGNO(pPage), pPage); + ASSERT(TDB_PAGE_INITIALIZED(pPage)); ASSERT(pPage->pPager == pPager); @@ -284,7 +289,11 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa return 0; } -void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) { tdbPCacheRelease(pPager->pCache, pPage, pTxn); } +void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) { + tdbPCacheRelease(pPager->pCache, pPage, pTxn); + // printf("thread %" PRId64 " pager retun page %d pgno %d ppage %p\n", taosGetSelfPthreadId(), pPage->id, + // TDB_PAGE_PGNO(pPage), pPage); +} static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { // TODO: Allocate a page from the free list @@ -352,6 +361,7 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage ret = (*initPage)(pPage, arg, init); if (ret < 0) { + ASSERT(0); TDB_UNLOCK_PAGE(pPage); return -1; } @@ -370,6 +380,7 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage } } } else { + ASSERT(0); return -1; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 2c79f39bc0..9f0267da93 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -275,15 +275,15 @@ static inline i32 tdbUnrefPage(SPage *pPage) { #define P_LOCK_FAIL -1 static inline int tdbTryLockPage(tdb_spinlock_t *pLock) { - int ret; - if (tdbSpinlockTrylock(pLock) == 0) { - ret = P_LOCK_SUCC; - } else if (errno == EBUSY) { - ret = P_LOCK_BUSY; + int ret = tdbSpinlockTrylock(pLock); + if (ret == 0) { + return P_LOCK_SUCC; + } else if (ret == EBUSY) { + return P_LOCK_BUSY; } else { - ret = P_LOCK_FAIL; + ASSERT(0); + return P_LOCK_FAIL; } - return ret; } #define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 5a3b45cca5..6070052127 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -486,18 +486,18 @@ TEST(tdb_test, DISABLED_simple_upsert1) { tdbClose(pEnv); } -TEST(tdb_test, DISABLED_multi_thread_query) { +TEST(tdb_test, multi_thread_query) { int ret; TDB *pEnv; TTB *pDb; tdb_cmpr_fn_t compFunc; - int nData = 100000; + int nData = 1000000; TXN txn; taosRemoveDir("tdb"); // Open Env - ret = tdbOpen("tdb", 512, 1, &pEnv); + ret = tdbOpen("tdb", 4096, 10, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -507,7 +507,7 @@ TEST(tdb_test, DISABLED_multi_thread_query) { char key[64]; char val[64]; - int64_t poolLimit = 4096; // 1M pool limit + int64_t poolLimit = 4096 * 20; // 1M pool limit int64_t txnid = 0; SPoolMem *pPool; @@ -600,7 +600,7 @@ TEST(tdb_test, DISABLED_multi_thread_query) { GTEST_ASSERT_EQ(ret, 0); } -TEST(tdb_test, multi_thread1) { +TEST(tdb_test, DISABLED_multi_thread1) { #if 0 int ret; TDB *pDb; From 9025abe617c30ba71b2c553e0574f8bdb9303baa Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 20 May 2022 15:14:02 +0800 Subject: [PATCH 08/23] fix(os): disable win compile warning output --- cmake/cmake.define | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 4e27ff5f47..55412fd26f 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -46,11 +46,17 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(COMMON_FLAGS "/W3 /D_WIN32") + SET(COMMON_FLAGS "/w /D_WIN32") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") # ENDIF () + IF (CMAKE_DEPFILE_FLAGS_C) + SET(CMAKE_DEPFILE_FLAGS_C "") + ENDIF () + IF (CMAKE_DEPFILE_FLAGS_CXX) + SET(CMAKE_DEPFILE_FLAGS_CXX "") + ENDIF () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}") From b563902bb494f7e62ecf8f8257502dc3e8f3f66c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 20 May 2022 07:40:21 +0000 Subject: [PATCH 09/23] make it compile --- source/libs/tdb/src/db/tdbPCache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 5ba5d65815..cdae73bfb9 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -14,8 +14,8 @@ */ #include "tdbInt.h" -#include -#include +// #include +// #include struct SPCache { int szPage; From afda7c637bdc82f48ec089b383125e49b8dc8212 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 May 2022 15:58:25 +0800 Subject: [PATCH 10/23] fix: tq mem leak --- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 4 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 22 +++++------ source/dnode/vnode/src/tq/tq.c | 43 +++++++++------------ source/libs/qworker/src/qworkerMsg.c | 26 ++++++------- source/libs/transport/src/trans.c | 2 + source/libs/transport/src/transComm.c | 1 + source/libs/transport/src/transSrv.c | 2 + 7 files changed, 50 insertions(+), 50 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index cf76d3a167..c4314a57b1 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -126,7 +126,9 @@ int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutRpcMsgToWorker(&pMgmt->readWorker, pMsg); } -int32_t mmPutRpcMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pMsg); } +int32_t mmPutRpcMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutRpcMsgToWorker(&pMgmt->syncWorker, pMsg); +} int32_t mmStartWorker(SMnodeMgmt *pMgmt) { SSingleWorkerCfg qCfg = { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index eec6bb3fb4..6632480aae 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -72,11 +72,10 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { if (code != 0) { if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); - - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); } + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); } static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { @@ -87,11 +86,10 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { if (code != 0) { if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); - - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); } + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); } static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { @@ -149,7 +147,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg *pMsg = NULL; + SRpcMsg * pMsg = NULL; SRpcMsg rsp; for (int32_t i = 0; i < numOfMsgs; ++i) { @@ -190,7 +188,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg *pMsg = NULL; + SRpcMsg * pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); @@ -216,7 +214,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg *pMsg = NULL; + SRpcMsg * pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); @@ -235,7 +233,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { - SRpcMsg *pRpc = pMsg; + SRpcMsg * pRpc = pMsg; SMsgHead *pHead = pRpc->pCont; int32_t code = 0; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9361c0e6d2..4a3b47c0d1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -129,7 +129,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { return 0; } -int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { +int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver, SRpcHandleInfo handleInfo) { if (msgType != TDMT_VND_SUBMIT) return 0; void* pIter = NULL; STqExec* pExec = NULL; @@ -239,10 +239,9 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqDataBlkRsp(&abuf, &rsp); - pMsg->pCont = buf; - pMsg->contLen = tlen; - pMsg->code = 0; - tmsgSendRsp(pMsg); + + SRpcMsg resp = {.info = handleInfo, .pCont = buf, .contLen = tlen, .code = 0}; + tmsgSendRsp(&resp); atomic_store_ptr(&pExec->pushHandle.handle, NULL); taosWUnLockLatch(&pExec->pushHandle.lock); @@ -407,9 +406,9 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu pTopic->buffer.output[j].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { - .reader = pReadHandle, - .meta = pTq->pVnode->pMeta, - .pMsgCb = &pTq->pVnode->msgCb, + .reader = pReadHandle, + .meta = pTq->pVnode->pMeta, + .pMsgCb = &pTq->pVnode->msgCb, }; pTopic->buffer.output[j].pReadHandle = pReadHandle; pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); @@ -663,10 +662,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqDataBlkRsp(&abuf, &rsp); - pMsg->pCont = buf; - pMsg->contLen = tlen; - pMsg->code = 0; - tmsgSendRsp(pMsg); + + SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0}; + tmsgSendRsp(&resp); tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset); @@ -845,12 +843,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { /*rsp.pBlockData = pRes;*/ /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/ - pMsg->pCont = buf; - pMsg->contLen = msgLen; - pMsg->code = 0; + SRpcMsg resp = {.info = pMsg->info, pCont = buf, .contLen = msgLen, .code = 0}; tqDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset, pHead->msgType, consumerId, pReq->epoch); - tmsgSendRsp(pMsg); + tmsgSendRsp(&resp); taosMemoryFree(pHead); return 0; } else { @@ -878,10 +874,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqPollRspV2(&abuf, &rspV2); - pMsg->pCont = buf; - pMsg->contLen = tlen; - pMsg->code = 0; - tmsgSendRsp(pMsg); + + SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0}; + tmsgSendRsp(&resp); tqDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch); /*}*/ @@ -990,10 +985,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { for (int32_t i = 0; i < parallel; i++) { STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { - .reader = pStreamReader, - .meta = pTq->pVnode->pMeta, - .pMsgCb = &pTq->pVnode->msgCb, - .vnode = pTq->pVnode, + .reader = pStreamReader, + .meta = pTq->pVnode->pMeta, + .pMsgCb = &pTq->pVnode->msgCb, + .vnode = pTq->pVnode, }; pTask->exec.runners[i].inputHandle = pStreamReader; pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 60270d3e06..562e550bdc 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -47,7 +47,7 @@ int32_t qwBuildAndSendQueryRsp(SRpcHandleInfo *pConn, int32_t code) { SQueryTableRsp rsp = {.code = code}; int32_t contLen = tSerializeSQueryTableRsp(NULL, 0, &rsp); - void *msg = rpcMallocCont(contLen); + void * msg = rpcMallocCont(contLen); tSerializeSQueryTableRsp(msg, contLen, &rsp); SRpcMsg rpcRsp = { @@ -85,7 +85,7 @@ int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SExplainExecInfo *execIn SExplainRsp rsp = {.numOfPlans = num, .subplanInfo = execInfo}; int32_t contLen = tSerializeSExplainRsp(NULL, 0, &rsp); - void *pRsp = rpcMallocCont(contLen); + void * pRsp = rpcMallocCont(contLen); tSerializeSExplainRsp(pRsp, contLen, &rsp); SRpcMsg rpcRsp = { @@ -104,7 +104,7 @@ int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SExplainExecInfo *execIn int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) { int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus); - void *pRsp = rpcMallocCont(contLen); + void * pRsp = rpcMallocCont(contLen); tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus); SRpcMsg rpcRsp = { @@ -212,7 +212,7 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { showRsp.tableMeta.numOfColumns = cols; int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp); - void *pBuf = rpcMallocCont(bufLen); + void * pBuf = rpcMallocCont(bufLen); tSerializeSShowRsp(pBuf, bufLen, &showRsp); SRpcMsg rpcMsg = { @@ -341,7 +341,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; SSubQueryMsg *msg = pMsg->pCont; - SQWorker *mgmt = (SQWorker *)qWorkerMgmt; + SQWorker * mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -361,7 +361,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int64_t rId = msg->refId; SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connInfo = pMsg->info}; - char *sql = strndup(msg->msg, msg->sqlLen); + char * sql = strndup(msg->msg, msg->sqlLen); QW_SCH_TASK_DLOG("processQuery start, node:%p, handle:%p, sql:%s", node, pMsg->info.handle, sql); taosMemoryFreeClear(sql); @@ -378,8 +378,8 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { bool queryDone = false; SQueryContinueReq *msg = (SQueryContinueReq *)pMsg->pCont; bool needStop = false; - SQWTaskCtx *handles = NULL; - SQWorker *mgmt = (SQWorker *)qWorkerMgmt; + SQWTaskCtx * handles = NULL; + SQWorker * mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -407,7 +407,7 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - SQWorker *mgmt = (SQWorker *)qWorkerMgmt; + SQWorker * mgmt = (SQWorker *)qWorkerMgmt; SResReadyReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid task ready msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -467,7 +467,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } SResFetchReq *msg = pMsg->pCont; - SQWorker *mgmt = (SQWorker *)qWorkerMgmt; + SQWorker * mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -505,7 +505,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - SQWorker *mgmt = (SQWorker *)qWorkerMgmt; + SQWorker * mgmt = (SQWorker *)qWorkerMgmt; int32_t code = 0; STaskCancelReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { @@ -542,7 +542,7 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; STaskDropReq *msg = pMsg->pCont; - SQWorker *mgmt = (SQWorker *)qWorkerMgmt; + SQWorker * mgmt = (SQWorker *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid task drop msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -581,7 +581,7 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; SSchedulerHbReq req = {0}; - SQWorker *mgmt = (SQWorker *)qWorkerMgmt; + SQWorker * mgmt = (SQWorker *)qWorkerMgmt; if (NULL == pMsg->pCont) { QW_ELOG("invalid hb msg, msg:%p, msgLen:%d", pMsg->pCont, pMsg->contLen); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 5c01034f35..5627dbfbf5 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -94,7 +94,9 @@ void rpcFreeCont(void* cont) { if (cont == NULL) { return; } + taosMemoryFree((char*)cont - TRANS_MSG_OVERHEAD); + tTrace("free mem: %p", (char*)cont - TRANS_MSG_OVERHEAD); } void* rpcReallocCont(void* ptr, int contLen) { if (ptr == NULL) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 98e9e67ede..d5c76ccbf2 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -133,6 +133,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { } else { p->cap = p->total; p->buf = taosMemoryRealloc(p->buf, p->cap); + tTrace("internal malloc mem: %p", p->buf); uvBuf->base = p->buf + p->len; uvBuf->len = p->cap - p->len; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 84adeb4bf6..da83a6f37f 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -469,6 +469,8 @@ static void uvStartSendResp(SSrvMsg* smsg) { if (pConn->broken == true) { // persist by + transFreeMsg(smsg->msg.pCont); + taosMemoryFree(smsg); transUnrefSrvHandle(pConn); return; } From bda392bbe6e9374eabda4ee162febf307a7208b9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 20 May 2022 16:19:32 +0800 Subject: [PATCH 11/23] fix: deal with error in schemaless --- source/client/src/clientSml.c | 48 ++++++++++------------------------ source/client/test/smlTest.cpp | 19 +++++--------- 2 files changed, 20 insertions(+), 47 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5b5071f79e..04dae5bfe1 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -58,13 +58,9 @@ for (int i = 1; i < keyLen; ++i) { \ #define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) -#define OTD_MAX_FIELDS_NUM 2 #define OTD_JSON_SUB_FIELDS_NUM 2 #define OTD_JSON_FIELDS_NUM 4 -#define OTD_TIMESTAMP_COLUMN_NAME "ts" -#define OTD_METRIC_VALUE_COLUMN_NAME "value" - #define TS "_ts" #define TS_LEN 3 #define TAG "_tag" @@ -731,24 +727,24 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { double ts = tsInt64; switch (type) { case TSDB_TIME_PRECISION_HOURS: - ts *= (3600 * 1e9); - tsInt64 *= (3600 * 1e9); + ts *= NANOSECOND_PER_HOUR; + tsInt64 *= NANOSECOND_PER_HOUR; break; case TSDB_TIME_PRECISION_MINUTES: - ts *= (60 * 1e9); - tsInt64 *= (60 * 1e9); + ts *= NANOSECOND_PER_MINUTE; + tsInt64 *= NANOSECOND_PER_MINUTE; break; case TSDB_TIME_PRECISION_SECONDS: - ts *= (1e9); - tsInt64 *= (1e9); + ts *= NANOSECOND_PER_SEC; + tsInt64 *= NANOSECOND_PER_SEC; break; case TSDB_TIME_PRECISION_MILLI: - ts *= (1e6); - tsInt64 *= (1e6); + ts *= NANOSECOND_PER_MSEC; + tsInt64 *= NANOSECOND_PER_MSEC; break; case TSDB_TIME_PRECISION_MICRO: - ts *= (1e3); - tsInt64 *= (1e3); + ts *= NANOSECOND_PER_USEC; + tsInt64 *= NANOSECOND_PER_USEC; break; case TSDB_TIME_PRECISION_NANO: break; @@ -762,23 +758,6 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { return tsInt64; } -static int64_t smlGetTimeNow(int8_t precision) { - switch (precision) { - case TSDB_TIME_PRECISION_HOURS: - return taosGetTimestampMs()/1000/3600; - case TSDB_TIME_PRECISION_MINUTES: - return taosGetTimestampMs()/1000/60; - case TSDB_TIME_PRECISION_SECONDS: - return taosGetTimestampMs()/1000; - case TSDB_TIME_PRECISION_MILLI: - case TSDB_TIME_PRECISION_MICRO: - case TSDB_TIME_PRECISION_NANO: - return taosGetTimestamp(precision); - default: - ASSERT(0); - } -} - static int8_t smlGetTsTypeByLen(int32_t len) { if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { return TSDB_TIME_PRECISION_SECONDS; @@ -810,14 +789,15 @@ static int8_t smlGetTsTypeByPrecision(int8_t precision) { } static int64_t smlParseInfluxTime(SSmlHandle* info, const char* data, int32_t len){ + if(len == 0){ + return taosGetTimestamp(TSDB_TIME_PRECISION_NANO); + } + int8_t tsType = smlGetTsTypeByPrecision(info->precision); if (tsType == -1) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL); return -1; } - if(len == 0){ - return smlGetTimeNow(tsType); - } int64_t ts = smlGetTimeValue(data, len, tsType); if(ts == -1){ diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index eeed9dc952..c7935b351c 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1203,24 +1203,17 @@ TEST(testCase, sml_TD15662_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); ASSERT_NE(info, nullptr); const char *sql[] = { - "iyyyje,id=iyyyje_41943_1303,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "hetrey,id=sub_table_0123456,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64", }; int ret = smlProcess(info, (char **)sql, sizeof(sql) / sizeof(sql[0])); ASSERT_EQ(ret, 0); - // case 1 - TAOS_RES *res = taos_query(taos, "select * from t_a5615048edae55218a22a149edebdc82"); - ASSERT_NE(res, nullptr); - - TAOS_ROW row = taos_fetch_row(res); - int64_t ts = *(int64_t*)row[0]; - ASSERT_EQ(ts, 1626006833639000000); - - taos_free_result(res); + destroyRequest(request); + smlDestroyInfo(info); } TEST(testCase, sml_TD15735_Test) { @@ -1262,11 +1255,11 @@ TEST(testCase, sml_TD15742_Test) { SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); - SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); ASSERT_NE(info, nullptr); const char *sql[] = { - "zgzbix 1626006833641 False id=zgzbix_992_38861 t0=t t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=11.12345f32 t6=22.123456789f64 t7=\"binaryTagValue\" t8=L\"ncharTagValue\"", + "test_ms,t0=t c0=f 1626006833641", }; int ret = smlProcess(info, (char**)sql, sizeof(sql)/sizeof(sql[0])); ASSERT_EQ(ret, 0); From 28b7f9599ed10d5a119b2f35c285f6b3a38b72a7 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 20 May 2022 16:20:15 +0800 Subject: [PATCH 12/23] fix(os): disable win compile tests --- cmake/cmake.options | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index d83ab49fd5..c77b580c17 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -46,6 +46,18 @@ IF(${TD_WINDOWS}) ON ) + option( + BUILD_TEST + "If build unit tests using googletest" + OFF + ) +ELSE () + + option( + BUILD_TEST + "If build unit tests using googletest" + ON + ) ENDIF () option( @@ -54,12 +66,6 @@ option( OFF ) -option( - BUILD_TEST - "If build unit tests using googletest" - ON -) - option( BUILD_WITH_LEVELDB "If build with leveldb" From 3c0a1bc7eea1407b56fab95b1cea21c4cc4cbd35 Mon Sep 17 00:00:00 2001 From: tomchon Date: Fri, 20 May 2022 16:20:27 +0800 Subject: [PATCH 13/23] add valgrind.log to dnodes.py --- tests/pytest/util/dnodes.py | 4 ++-- tests/system-test/1-insert/insertWithMoreVgroup.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index c78efb4e8d..952aca70cf 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -264,7 +264,7 @@ class TDDnode: cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( binPath, self.cfgDir) else: - valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" + valgrindCmdline = "valgrind --log-file=\"valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" cmd = "nohup %s %s -c %s 2>&1 & " % ( valgrindCmdline, binPath, self.cfgDir) @@ -325,7 +325,7 @@ class TDDnode: cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( binPath, self.cfgDir) else: - valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" + valgrindCmdline = "valgrind --log-file=\"valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" cmd = "nohup %s %s -c %s 2>&1 & " % ( valgrindCmdline, binPath, self.cfgDir) diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index ede33e7aa7..f0f35831db 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -119,7 +119,7 @@ class TDTestCase: # tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) return - def mutiThread_create_tables(self,host,dbname,stbname,vgroups,threadNumbers,count): + def mutiThread_create_tables(self,host,dbname,stbname,vgroups,threadNumbers,childrowcount): buildPath = self.getBuildPath() config = buildPath+ "../sim/dnode1/cfg/" @@ -128,7 +128,7 @@ class TDTestCase: tsql.execute("drop database if exists %s"%dbname) tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) tsql.execute("use %s" %dbname) - count=int(count) + count=int(childrowcount) threads = [] for i in range(threadNumbers): tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i)) @@ -140,7 +140,7 @@ class TDTestCase: tr.join() end_time = time.time() spendTime=end_time-start_time - speedCreate=count/spendTime + speedCreate=threadNumbers*count/spendTime tdLog.debug("spent %.2fs to create %d stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,threadNumbers,threadNumbers*count,speedCreate)) return @@ -171,7 +171,7 @@ class TDTestCase: sql = "insert into %s_%d values " %(stbname,i) # end sql if sql != pre_insert: - print(sql) + # print(sql) print(len(sql)) tsql.execute(sql) exeEndTime=time.time() @@ -269,7 +269,7 @@ class TDTestCase: tdLog.debug("-----create database and muti-thread create tables test------- ") #host,dbname,stbname,vgroups,threadNumbers,tcountStart,tcountStop #host, dbname, stbname, threadNumbers, chilCount, ts_start, childrowcount - self.mutiThread_create_tables(host="localhost",dbname="db",stbname="stb", vgroups=1, threadNumbers=5, count=50) + self.mutiThread_create_tables(host="localhost",dbname="db",stbname="stb", vgroups=1, threadNumbers=5, childrowcount=50) self.mutiThread_insert_data(host="localhost",dbname="db",stbname="stb", threadNumbers=5,chilCount=50,ts_start=self.ts,childrowcount=10) return From 7913887204e21f903bfa1087edf62af709833bd6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 20 May 2022 16:33:04 +0800 Subject: [PATCH 14/23] fix: deal with error in schemaless --- source/client/src/clientSml.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 04dae5bfe1..e884748fff 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -724,6 +724,9 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { if(value + len != endPtr){ return -1; } + if(tsInt64 == 0){ + return taosGetTimestampNs(); + } double ts = tsInt64; switch (type) { case TSDB_TIME_PRECISION_HOURS: @@ -751,7 +754,7 @@ static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { default: ASSERT(0); } - if(ts >= (double)INT64_MAX || ts <= 0){ + if(ts >= (double)INT64_MAX || ts < 0){ return -1; } @@ -1599,7 +1602,8 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - if(timeDouble <= 0){ + + if(timeDouble < 0){ return TSDB_CODE_TSC_INVALID_TIME_STAMP; } uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); @@ -1617,7 +1621,9 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { return TSDB_CODE_TSC_INVALID_TIME_STAMP; } tsVal = timeDouble; - } else { + } else if(timeDouble == 0){ + tsVal = taosGetTimestampNs(); + }else { return TSDB_CODE_TSC_INVALID_TIME_STAMP; } } else if (cJSON_IsObject(timestamp)) { From 4a786f1ad0bb009b706362ce6ede82818d34a9cf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 May 2022 16:48:56 +0800 Subject: [PATCH 15/23] fix: tq mem leak --- source/dnode/vnode/src/vnd/vnodeSvr.c | 30 +++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index dfd78d9dca..466c300d97 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -162,7 +162,7 @@ int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("message in fetch queue is processing"); - char *msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + char * msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); switch (pMsg->msgType) { case TDMT_VND_FETCH: @@ -184,8 +184,12 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { case TDMT_VND_TASK_PIPE_EXEC: case TDMT_VND_TASK_MERGE_EXEC: return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, 0); - case TDMT_VND_STREAM_TRIGGER: - return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, 0); + case TDMT_VND_STREAM_TRIGGER: { + // refactor, avoid double free + int code = tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, 0); + pMsg->pCont = NULL; + return code; + } case TDMT_VND_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg); default: @@ -328,12 +332,12 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, SDecoder decoder = {0}; int rcode = 0; SVCreateTbBatchReq req = {0}; - SVCreateTbReq *pCreateReq; + SVCreateTbReq * pCreateReq; SVCreateTbBatchRsp rsp = {0}; SVCreateTbRsp cRsp = {0}; char tbName[TSDB_TABLE_FNAME_LEN]; - STbUidStore *pStore = NULL; - SArray *tbUids = NULL; + STbUidStore * pStore = NULL; + SArray * tbUids = NULL; pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; pRsp->code = TSDB_CODE_SUCCESS; @@ -517,7 +521,7 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in SDecoder decoder = {0}; SEncoder encoder = {0}; int ret; - SArray *tbUids = NULL; + SArray * tbUids = NULL; pRsp->msgType = TDMT_VND_DROP_TABLE_RSP; pRsp->pCont = NULL; @@ -572,9 +576,9 @@ _exit: static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSubmitMsgIter *msgIter, const char *tags) { SSubmitBlkIter blkIter = {0}; - STSchema *pSchema = NULL; + STSchema * pSchema = NULL; tb_uid_t suid = 0; - STSRow *row = NULL; + STSRow * row = NULL; tInitSubmitBlkIter(msgIter, pBlock, &blkIter); if (blkIter.row == NULL) return 0; @@ -605,8 +609,8 @@ static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSub static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { ASSERT(pMsg != NULL); SSubmitMsgIter msgIter = {0}; - SMeta *pMeta = pVnode->pMeta; - SSubmitBlk *pBlock = NULL; + SMeta * pMeta = pVnode->pMeta; + SSubmitBlk * pBlock = NULL; if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; while (true) { @@ -620,10 +624,10 @@ static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char } static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + SSubmitReq * pSubmitReq = (SSubmitReq *)pReq; SSubmitRsp submitRsp = {0}; SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock; + SSubmitBlk * pBlock; SSubmitRsp rsp = {0}; SVCreateTbReq createTbReq = {0}; SDecoder decoder = {0}; From 5e819fbdb3c00a5704c1b84cf24c91c8fea4288d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 20 May 2022 16:50:39 +0800 Subject: [PATCH 16/23] fix(query): assign the group id in the ssdatablock when generating grouped results. --- source/libs/executor/src/executorimpl.c | 32 ++++++++++++------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 37aeb367f5..750554e828 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2062,15 +2062,7 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p pAggInfo->groupId = groupId; } -/** - * For interval query of both super table and table, copy the data in ascending order, since the output results are - * ordered in SWindowResutl already. While handling the group by query for both table and super table, - * all group result are completed already. - * - * @param pQInfo - * @param result - */ -int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, +int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t* rowCellOffset, SqlFunctionCtx* pCtx, int32_t numOfExprs) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t start = pGroupResInfo->index; @@ -2087,6 +2079,15 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn continue; } + if (pBlock->info.groupId == 0) { + pBlock->info.groupId = pPos->groupId; + } else { + // current value belongs to different group, it can't be packed into one datablock + if (pBlock->info.groupId != pPos->groupId) { + break; + } + } + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { break; } @@ -2100,9 +2101,8 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn if (pCtx[j].fpSet.finalize) { int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); if (TAOS_FAILED(code)) { - qError("%s build result data block error, code %s", GET_TASKID(taskInfo), tstrerror(code)); - taskInfo->code = code; - longjmp(taskInfo->env, code); + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); + longjmp(pTaskInfo->env, code); } } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { // do nothing, todo refactor @@ -2124,7 +2124,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn } } - // qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_TASKID(pRuntimeEnv)); + qDebug("%s result generated, rows:%d, groupId:%"PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.groupId); blockDataUpdateTsWindow(pBlock); return 0; } @@ -2145,10 +2145,9 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG return; } + // clear the existed group id + pBlock->info.groupId = 0; doCopyToSDataBlock(pTaskInfo, pBlock, pExprInfo, pBuf, pGroupResInfo, rowCellOffset, pCtx, numOfExprs); - - // add condition (pBlock->info.rows >= 1) just to runtime happy - blockDataUpdateTsWindow(pBlock); } static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, @@ -3656,7 +3655,6 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); } - doSetOperatorCompleted(pOperator); return (blockDataGetNumOfRows(pInfo->pRes) != 0) ? pInfo->pRes : NULL; } From fa84d0585bf5bb01a7d6d6f14f710dfe39d79827 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 20 May 2022 17:07:53 +0800 Subject: [PATCH 17/23] enh(query): enable remove the candidate table ids in the stream scanner. --- source/dnode/vnode/inc/vnode.h | 2 + source/dnode/vnode/src/tq/tqRead.c | 11 +++++ source/libs/executor/src/executor.c | 70 ++++++++++++++++------------- 3 files changed, 53 insertions(+), 30 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index ecb022426b..b870c406ec 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -126,6 +126,8 @@ STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList); int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); +int tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList); + int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 8fbd1e24e1..6ab2b0d917 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -235,3 +235,14 @@ int tqReadHandleAddTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { return 0; } + +int tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { + ASSERT(pHandle->tbIdHash != NULL); + + for(int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { + int64_t* pKey = (int64_t*) taosArrayGet(tbUidList, i); + taosHashRemove(pHandle->tbIdHash, pKey, sizeof(int64_t)); + } + + return 0; +} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index fa840e1cd6..6d308d7221 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -125,6 +125,33 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { return pTaskInfo; } +static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo, const SArray* tableIdList) { + SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); + + // let's discard the tables those are not created according to the queried super table. + SMetaReader mr = {0}; + metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); + for (int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { + int64_t* id = (int64_t*)taosArrayGet(tableIdList, i); + + int32_t code = metaGetTableEntryByUid(&mr, *id); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); + continue; + } + + ASSERT(mr.me.type == TSDB_CHILD_TABLE); + if (mr.me.ctbEntry.suid != pScanInfo->tableUid) { + continue; + } + + taosArrayPush(qa, id); + } + + metaReaderClear(&mr); + return qa; +} + int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; @@ -134,41 +161,24 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo pInfo = pInfo->pDownstream[0]; } + int32_t code = 0; SStreamBlockScanInfo* pScanInfo = pInfo->info; - if (isAdd) { - SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); - - SMetaReader mr = {0}; - metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); - for (int32_t i = 0; i < taosArrayGetSize(tableIdList); ++i) { - int64_t* id = (int64_t*)taosArrayGet(tableIdList, i); - - int32_t code = metaGetTableEntryByUid(&mr, *id); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); - continue; - } - - ASSERT(mr.me.type == TSDB_CHILD_TABLE); - if (mr.me.ctbEntry.suid != pScanInfo->tableUid) { - continue; - } - - taosArrayPush(qa, id); - } - - metaReaderClear(&mr); + if (isAdd) { // add new table id + SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList); qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); - int32_t code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - assert(0); + code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, qa); + taosArrayDestroy(qa); + + } else { // remove the table id in current list + SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList); + + qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList)); + code = tqReadHandleAddTbUidList(pScanInfo->streamBlockReader, tableIdList); + taosArrayDestroy(qa); } - return TSDB_CODE_SUCCESS; + return code; } int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, int32_t* tversion) { From e9607a9e25c7dfdf906b6a8203360038aa3c6f14 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 20 May 2022 17:24:10 +0800 Subject: [PATCH 18/23] feat: add cases to merge dup data in file and mem --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- tests/script/tsim/insert/update0.sim | 83 ++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b9ef72edc5..f0aa4d2ac6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2077,10 +2077,10 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf #endif if (TD_SUPPORT_UPDATE(pCfg->update)) { if (lastKeyAppend != key) { - lastKeyAppend = key; if (lastKeyAppend != TSKEY_INITIAL_VAL) { ++curRow; } + lastKeyAppend = key; } // load data from file firstly numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, curRow, pos, pos); diff --git a/tests/script/tsim/insert/update0.sim b/tests/script/tsim/insert/update0.sim index c34a08c79d..89eecaf860 100644 --- a/tests/script/tsim/insert/update0.sim +++ b/tests/script/tsim/insert/update0.sim @@ -35,7 +35,7 @@ sql insert into ct1 values('2022-05-03 16:59:00.016', 18); sql insert into ct1 values('2022-05-03 16:59:00.021', 21); sql insert into ct1 values('2022-05-03 16:59:00.022', 22); -print =============== step3-1 query records from ct1 from memory +print =============== step3-1 query records of ct1 from memory sql select * from ct1; print $data00 $data01 print $data10 $data11 @@ -69,7 +69,7 @@ sql insert into ct2 values('2022-03-02 16:59:00.010', 1),('2022-03-02 16:59:00.0 sql insert into ct2 values('2022-03-02 16:59:00.010', 3),('2022-03-02 16:59:00.010',33),('2022-04-01 16:59:00.011',4),('2022-04-01 16:59:00.011',6),('2022-03-06 16:59:00.013',8); sql insert into ct2 values('2022-03-02 16:59:00.010', 103),('2022-03-02 16:59:00.010',303),('2022-04-01 16:59:00.011',40),('2022-04-01 16:59:00.011',60),('2022-03-06 16:59:00.013',80); -print =============== step3-1 query records from ct2 from memory +print =============== step3-1 query records of ct2 from memory sql select * from ct2; print $data00 $data01 print $data10 $data11 @@ -99,7 +99,7 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -print =============== step3-2 query records from ct1 from file +print =============== step3-2 query records of ct1 from file sql select * from ct1; print $data00 $data01 print $data10 $data11 @@ -128,7 +128,7 @@ if $data51 != 22 then return -1 endi -print =============== step3-2 query records from ct2 from file +print =============== step3-2 query records of ct2 from file sql select * from ct2; print $data00 $data01 print $data10 $data11 @@ -152,4 +152,79 @@ endi if $data21 != 40 then print data21 $data21 != 40 return -1 +endi + +print =============== step3-3 query records of ct1 from memory and file(merge) +sql insert into ct1 values('2022-05-03 16:59:00.010', 100); +sql insert into ct1 values('2022-05-03 16:59:00.022', 200); +sql insert into ct1 values('2022-05-03 16:59:00.016', 160); + +sql select * from ct1; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 +print $data40 $data41 +print $data50 $data51 + +if $rows != 6 then + print rows $rows != 6 + return -1 +endi + +if $data01 != 100 then + print data01 $data01 != 100 + return -1 +endi + +if $data21 != 160 then + print data21 $data21 != 160 + return -1 +endi + +if $data51 != 200 then + print data51 $data51 != 200 + return -1 +endi + +print =============== step3-3 query records of ct2 from memory and file(merge) +sql insert into ct2(ts) values('2022-04-02 16:59:00.016'); +sql insert into ct2 values('2022-03-06 16:59:00.013', NULL); +sql insert into ct2 values('2022-03-01 16:59:00.016', 10); +sql insert into ct2(ts) values('2022-04-01 16:59:00.011'); +sql select * from ct2; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 +print $data40 $data41 + +if $rows != 5 then + print rows $rows != 5 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data11 != 103 then + print data11 $data11 != 103 + return -1 +endi + +if $data21 != NULL then + print data21 $data21 != NULL + return -1 +endi + +if $data31 != 40 then + print data31 $data31 != 40 + return -1 +endi + +if $data41 != NULL then + print data41 $data41 != NULL + return -1 endi \ No newline at end of file From 90f8acf16850f150c0010e0ab643a74931c82090 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 20 May 2022 17:49:32 +0800 Subject: [PATCH 19/23] fix: tq mem leak --- source/libs/transport/src/transComm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index d5c76ccbf2..7014cc481f 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -133,7 +133,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { } else { p->cap = p->total; p->buf = taosMemoryRealloc(p->buf, p->cap); - tTrace("internal malloc mem: %p", p->buf); + tTrace("internal malloc mem: %p, size: %d", p->buf, p->cap); uvBuf->base = p->buf + p->len; uvBuf->len = p->cap - p->len; From 7feeea974c7e0bdaeecacff2906cbc0c570a5ce5 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 20 May 2022 18:22:40 +0800 Subject: [PATCH 20/23] feat: add scenarios that trigger metadata refresh --- include/libs/qcom/query.h | 8 ++- source/client/src/clientImpl.c | 93 ++++++++++++-------------- source/libs/parser/src/parInsert.c | 75 ++++++++++++--------- source/libs/parser/src/parTranslater.c | 16 ++++- 4 files changed, 108 insertions(+), 84 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index daf008108b..f4ccc05208 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -182,8 +182,10 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE -#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \ - ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST) +#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \ + ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST || \ + (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \ + (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED) #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \ ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED) @@ -194,7 +196,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_SCHEDULER_RETRY_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) -#define REQUEST_MAX_TRY_TIMES 5 +#define REQUEST_MAX_TRY_TIMES 1 #define qFatal(...) \ do { \ diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 1dda4c3024..f493f02cd6 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -13,18 +13,18 @@ * along with this program. If not, see . */ +#include "cJSON.h" #include "clientInt.h" #include "clientLog.h" #include "command.h" #include "scheduler.h" #include "tdatablock.h" +#include "tdataformat.h" #include "tdef.h" #include "tglobal.h" #include "tmsgtype.h" #include "tpagedbuf.h" #include "tref.h" -#include "cJSON.h" -#include "tdataformat.h" static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); @@ -189,7 +189,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols); setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision); } - + } + if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) { TSWAP(pRequest->dbList, (*pQuery)->pDbList); TSWAP(pRequest->tableList, (*pQuery)->pTableList); } @@ -293,7 +294,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, - pRequest->metric.start, &res); + pRequest->metric.start, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); @@ -483,7 +484,8 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { int32_t retryNum = 0; int32_t code = 0; - while (retryNum++ < REQUEST_MAX_TRY_TIMES) { + do { + destroyRequest(pRequest); pRequest = launchQuery(pTscObj, sql, sqlLen); if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { break; @@ -494,9 +496,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { pRequest->code = code; break; } - - destroyRequest(pRequest); - } + } while (retryNum++ < REQUEST_MAX_TRY_TIMES); return pRequest; } @@ -805,21 +805,20 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { return TSDB_CODE_SUCCESS; } -static char* parseTagDatatoJson(void *p){ - char* string = NULL; - cJSON *json = cJSON_CreateObject(); - if (json == NULL) - { +static char* parseTagDatatoJson(void* p) { + char* string = NULL; + cJSON* json = cJSON_CreateObject(); + if (json == NULL) { goto end; } int16_t nCols = kvRowNCols(p); - char tagJsonKey[256] = {0}; + char tagJsonKey[256] = {0}; for (int j = 0; j < nCols; ++j) { - SColIdx * pColIdx = kvRowColIdxAt(p, j); - char* val = (char*)(kvRowColVal(p, pColIdx)); - if (j == 0){ - if(*val == TSDB_DATA_TYPE_NULL){ + SColIdx* pColIdx = kvRowColIdxAt(p, j); + char* val = (char*)(kvRowColVal(p, pColIdx)); + if (j == 0) { + if (*val == TSDB_DATA_TYPE_NULL) { string = taosMemoryCalloc(1, 8); sprintf(varDataVal(string), "%s", TSDB_DATA_NULL_STR_L); varDataSetLen(string, strlen(varDataVal(string))); @@ -834,19 +833,18 @@ static char* parseTagDatatoJson(void *p){ // json value val += varDataTLen(val); char* realData = POINTER_SHIFT(val, CHAR_BYTES); - char type = *val; - if(type == TSDB_DATA_TYPE_NULL) { + char type = *val; + if (type == TSDB_DATA_TYPE_NULL) { cJSON* value = cJSON_CreateNull(); - if (value == NULL) - { + if (value == NULL) { goto end; } cJSON_AddItemToObject(json, tagJsonKey, value); - }else if(type == TSDB_DATA_TYPE_NCHAR) { + } else if (type == TSDB_DATA_TYPE_NCHAR) { cJSON* value = NULL; - if (varDataLen(realData) > 0){ - char *tagJsonValue = taosMemoryCalloc(varDataLen(realData), 1); - int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(realData), varDataLen(realData), tagJsonValue); + if (varDataLen(realData) > 0) { + char* tagJsonValue = taosMemoryCalloc(varDataLen(realData), 1); + int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(realData), varDataLen(realData), tagJsonValue); if (length < 0) { tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, val); taosMemoryFree(tagJsonValue); @@ -854,45 +852,41 @@ static char* parseTagDatatoJson(void *p){ } value = cJSON_CreateString(tagJsonValue); taosMemoryFree(tagJsonValue); - if (value == NULL) - { + if (value == NULL) { goto end; } - }else if(varDataLen(realData) == 0){ + } else if (varDataLen(realData) == 0) { value = cJSON_CreateString(""); - }else{ + } else { ASSERT(0); } cJSON_AddItemToObject(json, tagJsonKey, value); - }else if(type == TSDB_DATA_TYPE_DOUBLE){ + } else if (type == TSDB_DATA_TYPE_DOUBLE) { double jsonVd = *(double*)(realData); cJSON* value = cJSON_CreateNumber(jsonVd); - if (value == NULL) - { + if (value == NULL) { goto end; } cJSON_AddItemToObject(json, tagJsonKey, value); -// }else if(type == TSDB_DATA_TYPE_BIGINT){ -// int64_t jsonVd = *(int64_t*)(realData); -// cJSON* value = cJSON_CreateNumber((double)jsonVd); -// if (value == NULL) -// { -// goto end; -// } -// cJSON_AddItemToObject(json, tagJsonKey, value); - }else if (type == TSDB_DATA_TYPE_BOOL) { - char jsonVd = *(char*)(realData); + // }else if(type == TSDB_DATA_TYPE_BIGINT){ + // int64_t jsonVd = *(int64_t*)(realData); + // cJSON* value = cJSON_CreateNumber((double)jsonVd); + // if (value == NULL) + // { + // goto end; + // } + // cJSON_AddItemToObject(json, tagJsonKey, value); + } else if (type == TSDB_DATA_TYPE_BOOL) { + char jsonVd = *(char*)(realData); cJSON* value = cJSON_CreateBool(jsonVd); - if (value == NULL) - { + if (value == NULL) { goto end; } cJSON_AddItemToObject(json, tagJsonKey, value); - }else{ + } else { ASSERT(0); } - } string = cJSON_PrintUnformatted(json); end: @@ -930,7 +924,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; pResultInfo->row[i] = pResultInfo->pCol[i].pData; - }else if (type == TSDB_DATA_TYPE_JSON && colLength[i] > 0) { + } else if (type == TSDB_DATA_TYPE_JSON && colLength[i] > 0) { char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -943,7 +937,6 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int if (pCol->offset[j] != -1) { char* pStart = pCol->offset[j] + pCol->pData; - int32_t jsonInnerType = *pStart; char* jsonInnerData = pStart + CHAR_BYTES; char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; @@ -951,7 +944,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); varDataSetLen(dst, strlen(varDataVal(dst))); } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) { - char *jsonString = parseTagDatatoJson(jsonInnerData); + char* jsonString = parseTagDatatoJson(jsonInnerData); STR_TO_VARSTR(dst, jsonString); taosMemoryFree(jsonString); } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index d1b4a745b9..b452950624 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -41,6 +41,13 @@ sToken = tStrGetToken(pSql, &index, false); \ } while (0) +#define NEXT_VALID_TOKEN(pSql, sToken) \ + do { \ + sToken.n = tGetToken(pSql, &sToken.type); \ + sToken.z = pSql; \ + pSql += sToken.n; \ + } while (TK_NK_SPACE == sToken.type) + typedef struct SInsertParseContext { SParseContext* pComCxt; // input char* pSql; // input @@ -482,9 +489,11 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - return func(pMsgBuf, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, + param); } else if (pToken->type == TK_NK_FLOAT) { - return func(pMsgBuf, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); + return func(pMsgBuf, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, + param); } else { return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); } @@ -685,7 +694,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* isOrdered = false; } if (index < 0) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid column/tag name", sToken.z); + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, sToken.z); } if (pColList->cols[index].valStat == VAL_STAT_HAS) { return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", sToken.z); @@ -895,8 +904,10 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SName* name, char* tb return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, name->tname)); - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_RP != sToken.type) { + NEXT_VALID_TOKEN(pCxt->pSql, sToken); + if (TK_NK_COMMA == sToken.type) { + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_TAGS_NOT_MATCHED); + } else if (TK_NK_RP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); } @@ -996,8 +1007,10 @@ static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlo pDataBlock->size += extendedRowSize; // len; } - NEXT_TOKEN(pCxt->pSql, sToken); - if (TK_NK_RP != sToken.type) { + NEXT_VALID_TOKEN(pCxt->pSql, sToken); + if (TK_NK_COMMA == sToken.type) { + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + } else if (TK_NK_RP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, ") expected", sToken.z); } @@ -1057,10 +1070,10 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; static int32_t parseInsertBody(SInsertParseContext* pCxt) { - int32_t tbNum = 0; - char tbFName[TSDB_TABLE_FNAME_LEN]; - bool autoCreateTbl = false; - STableMeta *pMeta = NULL; + int32_t tbNum = 0; + char tbFName[TSDB_TABLE_FNAME_LEN]; + bool autoCreateTbl = false; + STableMeta* pMeta = NULL; // for each table while (1) { @@ -1121,7 +1134,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { &dataBuf, NULL, &pCxt->createTblReq)); pMeta = pCxt->pTableMeta; pCxt->pTableMeta = NULL; - + if (TK_NK_LP == sToken.type) { // pSql -> field1_name, ...) CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pMeta))); @@ -1160,7 +1173,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); - (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pMeta, tags, tbFName, autoCreateTbl, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); + (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pMeta, tags, tbFName, autoCreateTbl, pCxt->pVgroupsHashObj, + pCxt->pTableBlockHashObj); memset(&pCxt->tags, 0, sizeof(pCxt->tags)); pCxt->pVgroupsHashObj = NULL; @@ -1231,14 +1245,14 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } } - + context.pOutput->payloadType = PAYLOAD_TYPE_KV; int32_t code = skipInsertInto(&context); if (TSDB_CODE_SUCCESS == code) { code = parseInsertBody(&context); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) { SName* pTable = taosHashIterate(context.pTableNameHashObj, NULL); while (NULL != pTable) { taosArrayPush((*pQuery)->pTableList, pTable); @@ -1579,9 +1593,9 @@ typedef struct SmlExecTableHandle { } SmlExecTableHandle; typedef struct SmlExecHandle { - SHashObj* pBlockHash; - SmlExecTableHandle tableExecHandle; - SQuery *pQuery; + SHashObj* pBlockHash; + SmlExecTableHandle tableExecHandle; + SQuery* pQuery; } SSmlExecHandle; static void smlDestroyTableHandle(void* pHandle) { @@ -1673,9 +1687,9 @@ static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedD SSchema* pTagSchema = &pSchema[tags->boundColumns[i] - 1]; // colId starts with 1 param.schema = pTagSchema; SSmlKv* kv = taosArrayGetP(cols, i); - if(IS_VAR_DATA_TYPE(kv->type)){ + if (IS_VAR_DATA_TYPE(kv->type)) { KvRowAppend(msg, kv->value, kv->length, ¶m); - }else{ + } else { KvRowAppend(msg, &(kv->value), kv->length, ¶m); } } @@ -1688,13 +1702,13 @@ static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedD return TSDB_CODE_SUCCESS; } -int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols, bool format, - STableMeta *pTableMeta, char *tableName, char *msgBuf, int16_t msgBufLen) { +int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, + char* tableName, char* msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; - smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table - SSchema* pTagsSchema = getTableTagSchema(pTableMeta); + smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table + SSchema* pTagsSchema = getTableTagSchema(pTableMeta); setBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta)); int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema); if (ret != TSDB_CODE_SUCCESS) { @@ -1702,7 +1716,8 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols return ret; } SKVRow row = NULL; - ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tagsBuilder, &smlHandle->tableExecHandle.tags, pTagsSchema, &row, &pBuf); + ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tagsBuilder, &smlHandle->tableExecHandle.tags, pTagsSchema, + &row, &pBuf); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -1733,7 +1748,7 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); int32_t rowNum = taosArrayGetSize(cols); - if(rowNum <= 0) { + if (rowNum <= 0) { return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); } ret = allocateMemForSize(pDataBlock, extendedRowSize * rowNum); @@ -1744,9 +1759,9 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols for (int32_t r = 0; r < rowNum; ++r) { STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header tdSRowResetBuf(pBuilder, row); - void *rowData = taosArrayGetP(cols, r); + void* rowData = taosArrayGetP(cols, r); size_t rowDataSize = 0; - if(format){ + if (format) { rowDataSize = taosArrayGetSize(rowData); } @@ -1781,9 +1796,9 @@ int32_t smlBindData(void *handle, SArray *tags, SArray *colsSchema, SArray *cols kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); } - if(IS_VAR_DATA_TYPE(kv->type)){ + if (IS_VAR_DATA_TYPE(kv->type)) { MemRowAppend(&pBuf, kv->value, colLen, ¶m); - }else{ + } else { MemRowAppend(&pBuf, &(kv->value), colLen, ¶m); } } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index dbb29699fc..f6d53dd15a 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -120,6 +120,20 @@ static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const return getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pMeta); } +static int32_t refreshGetTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, + STableMeta** pMeta) { + SParseContext* pParCxt = pCxt->pParseCxt; + SName name; + toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name); + int32_t code = + catalogRefreshGetTableMeta(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, &name, pMeta, false); + if (TSDB_CODE_SUCCESS != code) { + parserError("catalogRefreshGetTableMeta error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pDbName, + pTableName); + } + return code; +} + static int32_t getTableDistVgInfo(STranslateContext* pCxt, const SName* pName, SArray** pVgInfo) { SParseContext* pParCxt = pCxt->pParseCxt; int32_t code = collectUseDatabase(pName, pCxt->pDbs); @@ -3201,7 +3215,7 @@ static int32_t translateExplain(STranslateContext* pCxt, SExplainStmt* pStmt) { } static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) { - return getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta); + return refreshGetTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta); } static int32_t translateKillConnection(STranslateContext* pCxt, SKillStmt* pStmt) { From 7391a756f645f42146c4aade1ac8a1dd3a6b7c2a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 20 May 2022 18:53:21 +0800 Subject: [PATCH 21/23] refactor(stream) --- include/common/tmsg.h | 12 --- include/common/tmsgdef.h | 4 + include/libs/stream/tstream.h | 38 +++++++- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 + source/dnode/vnode/src/inc/vnodeInt.h | 14 ++- source/dnode/vnode/src/tq/tq.c | 100 ++++++++++++++++++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 16 ++++ source/libs/stream/src/tstream.c | 22 +++-- 8 files changed, 174 insertions(+), 36 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index addb84046c..d9087f59c6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2589,18 +2589,6 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); } -typedef struct { - int64_t streamId; - int32_t taskId; - int32_t sourceVg; - int64_t sourceVer; - SArray* data; // SArray -} SStreamDispatchReq; - -typedef struct { - int8_t inputStatus; -} SStreamDispatchRsp; - #define TD_AUTO_CREATE_TABLE 0x1 typedef struct { int64_t suid; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 93b2e75360..455898585a 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -200,6 +200,10 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_TASK_WRITE_EXEC, "vnode-task-write-exec", SStreamTaskExecReq, SStreamTaskExecRsp) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_RUN, "vnode-stream-task-run", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_DISPATCH, "vnode-stream-task-dispatch", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TASK_RECOVER, "vnode-stream-task-recover", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index bca947b84c..1604749af8 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -107,7 +107,7 @@ static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) if (ref == 0) { taosMemoryFree(pDataSubmit->data); taosMemoryFree(pDataSubmit->dataRef); - taosFreeQitem(pDataSubmit); + // taosFreeQitem(pDataSubmit); } } @@ -286,6 +286,36 @@ typedef struct { int32_t taskId; } SStreamTaskRunReq; +typedef struct { + int64_t streamId; + int32_t taskId; + int32_t sourceTaskId; + int32_t sourceVg; +#if 0 + int64_t sourceVer; +#endif + SArray* data; // SArray +} SStreamDispatchReq; + +typedef struct { + int64_t streamId; + int32_t taskId; + int8_t inputStatus; +} SStreamDispatchRsp; + +typedef struct { + int64_t streamId; + int32_t taskId; + int32_t sourceTaskId; + int32_t sourceVg; +} SStreamTaskRecoverReq; + +typedef struct { + int64_t streamId; + int32_t taskId; + int8_t inputStatus; +} SStreamTaskRecoverRsp; + int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input); int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input); int32_t streamDequeueOutput(SStreamTask* pTask, void** output); @@ -296,6 +326,12 @@ int32_t streamTaskRun(SStreamTask* pTask); int32_t streamTaskHandleInput(SStreamTask* pTask, void* data); +int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb); +int32_t streamTaskProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pMsg); +int32_t streamTaskProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp); +int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg); +int32_t streamTaskProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 3da3d90ae1..f28209f982 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -314,6 +314,10 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_MERGE_EXEC, vmPutNodeMsgToMergeQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_WRITE_EXEC, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_RUN, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DISPATCH, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_RECOVER, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_VNODE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_VNODE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 1c89649313..23825e6f4a 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -121,10 +121,18 @@ int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); -int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); -int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); -int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); +int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); +#if 0 +int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); +int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); +#endif +int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* data); +int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); // sma int32_t smaOpen(SVnode* pVnode); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9361c0e6d2..a8f9e8db00 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -105,12 +105,11 @@ static void tdSRowDemo() { } int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { - void* pIter = NULL; - STqExec* pExec = NULL; + void* pIter = NULL; while (1) { pIter = taosHashIterate(pTq->execs, pIter); if (pIter == NULL) break; - pExec = (STqExec*)pIter; + STqExec* pExec = (STqExec*)pIter; if (pExec->subType == TOPIC_SUB_TYPE__DB) { if (!isAdd) { int32_t sz = taosArrayGetSize(tbUidList); @@ -275,6 +274,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) } memcpy(data, msg, msgLen); + tqProcessStreamTriggerNew(pTq, data); + +#if 0 SRpcMsg req = { .msgType = TDMT_VND_STREAM_TRIGGER, .pCont = data, @@ -282,6 +284,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) }; tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req); +#endif return 0; } @@ -980,12 +983,24 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { + pTask->status = TASK_STATUS__IDLE; + pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; + pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; + + pTask->inputQ = taosOpenQueue(); + pTask->outputQ = taosOpenQueue(); + pTask->inputQAll = taosAllocateQall(); + pTask->outputQAll = taosAllocateQall(); + + if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL) + goto FAIL; + if (pTask->execType != TASK_EXEC__NONE) { // expand runners pTask->exec.numOfRunners = parallel; pTask->exec.runners = taosMemoryCalloc(parallel, sizeof(SStreamRunner)); if (pTask->exec.runners == NULL) { - return -1; + goto FAIL; } for (int32_t i = 0; i < parallel; i++) { STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); @@ -1007,6 +1022,13 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { } return 0; +FAIL: + if (pTask->inputQ) taosCloseQueue(pTask->inputQ); + if (pTask->outputQ) taosCloseQueue(pTask->outputQ); + if (pTask->inputQAll) taosFreeQall(pTask->inputQAll); + if (pTask->outputQAll) taosFreeQall(pTask->outputQAll); + if (pTask) taosMemoryFree(pTask); + return -1; } int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { @@ -1058,6 +1080,7 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t wo return 0; } +#if 0 int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* data) { SStreamDataSubmit* pSubmit = NULL; @@ -1108,6 +1131,7 @@ FAIL: } return -1; } +#endif int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId) { SStreamTaskExecReq req; @@ -1125,25 +1149,28 @@ int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId) return 0; } -int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) { +int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* pReq) { void* pIter = NULL; bool failed = false; SStreamDataSubmit* pSubmit = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); if (pSubmit == NULL) { failed = true; + goto SET_TASK_FAIL; } pSubmit->dataRef = taosMemoryMalloc(sizeof(int32_t)); if (pSubmit->dataRef == NULL) { failed = true; + goto SET_TASK_FAIL; } - pSubmit->type = STREAM_DATA_TYPE_SUBMIT_BLOCK; - pSubmit->sourceVer = ver; - pSubmit->sourceVg = pTq->pVnode->config.vgId; + pSubmit->type = STREAM_INPUT__DATA_SUBMIT; + /*pSubmit->sourceVer = ver;*/ + /*pSubmit->sourceVg = pTq->pVnode->config.vgId;*/ pSubmit->data = pReq; *pSubmit->dataRef = 1; +SET_TASK_FAIL: while (1) { pIter = taosHashIterate(pTq->pStreamTasks, pIter); if (pIter == NULL) break; @@ -1162,7 +1189,18 @@ int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) { int8_t execStatus = atomic_load_8(&pTask->status); if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { - // TODO dispatch task launch msg to fetch queue + SStreamTaskRunReq* pRunReq = taosMemoryMalloc(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) continue; + // TODO: do we need htonl? + pRunReq->head.vgId = pTq->pVnode->config.vgId; + pRunReq->streamId = pTask->streamId; + pRunReq->taskId = pTask->taskId; + SRpcMsg msg = { + .msgType = TDMT_VND_TASK_RUN, + .pCont = pRunReq, + .contLen = sizeof(SStreamTaskRunReq), + }; + tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &msg); } } else { @@ -1174,11 +1212,53 @@ int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) { streamDataSubmitRefDec(pSubmit); return 0; } else { + if (pSubmit) { + if (pSubmit->dataRef) { + taosMemoryFree(pSubmit->dataRef); + } + taosFreeQitem(pSubmit); + } return -1; } } -int32_t tqProcessTaskExec2(STQ* pTq, char* msg, int32_t msgLen) { +int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { // + SStreamTaskRunReq* pReq = pMsg->pCont; + int32_t taskId = pReq->taskId; + SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + streamTaskProcessRunReq(pTask, &pTq->pVnode->msgCb); + return 0; +} + +int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) { + SStreamDispatchReq* pReq = pMsg->pCont; + int32_t taskId = pReq->taskId; + SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + streamTaskProcessDispatchReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg); + return 0; +} + +int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { + SStreamTaskRecoverReq* pReq = pMsg->pCont; + int32_t taskId = pReq->taskId; + SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + streamTaskProcessRecoverReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg); + return 0; +} + +int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { + SStreamDispatchRsp* pRsp = pMsg->pCont; + int32_t taskId = pRsp->taskId; + SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + streamTaskProcessDispatchRsp(pTask, &pTq->pVnode->msgCb, pRsp); + return 0; +} + +int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { + SStreamTaskRecoverRsp* pRsp = pMsg->pCont; + int32_t taskId = pRsp->taskId; + SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + streamTaskProcessRecoverRsp(pTask, pRsp); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index dfd78d9dca..b6600b5d0b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -106,11 +106,13 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg pMsg->contLen - sizeof(SMsgHead)) < 0) { } } break; +#if 0 case TDMT_VND_TASK_WRITE_EXEC: { if (tqProcessTaskExec(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead), 0) < 0) { } } break; +#endif case TDMT_VND_ALTER_VNODE: break; default: @@ -181,11 +183,25 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return vnodeGetTableMeta(pVnode, pMsg); case TDMT_VND_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId); + + case TDMT_VND_TASK_RUN: + return tqProcessTaskRunReq(pVnode->pTq, pMsg); + case TDMT_VND_TASK_DISPATCH: + return tqProcessTaskDispatchReq(pVnode->pTq, pMsg); + case TDMT_VND_TASK_RECOVER: + return tqProcessTaskRecoverReq(pVnode->pTq, pMsg); + case TDMT_VND_TASK_DISPATCH_RSP: + return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg); + case TDMT_VND_TASK_RECOVER_RSP: + return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg); + +#if 0 case TDMT_VND_TASK_PIPE_EXEC: case TDMT_VND_TASK_MERGE_EXEC: return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, 0); case TDMT_VND_STREAM_TRIGGER: return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, 0); +#endif case TDMT_VND_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg); default: diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index 38b6f2b0e2..66a661481e 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -68,7 +68,7 @@ static int32_t streamBuildExecMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMs // get groupId, compute hash value uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); - // + // get node // TODO: optimize search process SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; @@ -152,13 +152,13 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) // exec while (1) { - SSDataBlock* output; + SSDataBlock* output = NULL; uint64_t ts = 0; if (qExecTask(exec, &output, &ts) < 0) { ASSERT(false); } if (output == NULL) break; - taosArrayPush(pRes, &output); + taosArrayPush(pRes, output); } // destroy @@ -189,7 +189,7 @@ int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) { taosFreeQitem(data); if (taosArrayGetSize(pRes) != 0) { - SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + SStreamDataBlock* resQ = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); resQ->type = STREAM_INPUT__DATA_BLOCK; resQ->blocks = pRes; taosWriteQitem(pTask->outputQ, resQ); @@ -209,7 +209,7 @@ int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) { taosFreeQitem(data); if (taosArrayGetSize(pRes) != 0) { - SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + SStreamDataBlock* resQ = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); resQ->type = STREAM_INPUT__DATA_BLOCK; resQ->blocks = pRes; taosWriteQitem(pTask->outputQ, resQ); @@ -231,7 +231,7 @@ int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) { taosFreeQitem(data); if (taosArrayGetSize(pRes) != 0) { - SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + SStreamDataBlock* resQ = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); resQ->type = STREAM_INPUT__DATA_BLOCK; resQ->blocks = pRes; taosWriteQitem(pTask->outputQ, resQ); @@ -253,7 +253,7 @@ int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) { taosFreeQitem(data); if (taosArrayGetSize(pRes) != 0) { - SStreamDataBlock* resQ = taosAllocateQitem(sizeof(void**), DEF_QITEM); + SStreamDataBlock* resQ = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); resQ->type = STREAM_INPUT__DATA_BLOCK; resQ->blocks = pRes; taosWriteQitem(pTask->outputQ, resQ); @@ -392,12 +392,14 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* // 1.2 enqueue pBlock->type = STREAM_DATA_TYPE_SSDATA_BLOCK; pBlock->sourceVg = pReq->sourceVg; - pBlock->sourceVer = pReq->sourceVer; + /*pBlock->sourceVer = pReq->sourceVer;*/ taosWriteQitem(pTask->inputQ, pBlock); // 1.3 rsp by input status SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp)); pCont->inputStatus = status; + pCont->streamId = pReq->streamId; + pCont->taskId = pReq->sourceTaskId; pRsp->pCont = pCont; pRsp->contLen = sizeof(SStreamDispatchRsp); tmsgSendRsp(pRsp); @@ -439,12 +441,12 @@ int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb) { return 0; } -int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, char* msg) { +int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg) { // return 0; } -int32_t streamTaskProcessRecoverRsp(SStreamTask* pTask, char* msg) { +int32_t streamTaskProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) { // return 0; } From 0b8d3b056d5f130ddaf00be4b3394ef9469fd5ae Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 20 May 2022 18:58:17 +0800 Subject: [PATCH 22/23] fix: bad merge --- source/dnode/vnode/src/vnd/vnodeSvr.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index c09b80af75..297b518ac7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -184,8 +184,11 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { case TDMT_VND_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId); - case TDMT_VND_TASK_RUN: - return tqProcessTaskRunReq(pVnode->pTq, pMsg); + case TDMT_VND_TASK_RUN: { + int32_t code = tqProcessTaskRunReq(pVnode->pTq, pMsg); + pMsg->pCont = NULL; + return code; + } case TDMT_VND_TASK_DISPATCH: return tqProcessTaskDispatchReq(pVnode->pTq, pMsg); case TDMT_VND_TASK_RECOVER: From ace05054e7ca3b81ce2f6c065f851712c77c932a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Fri, 20 May 2022 20:26:10 +0800 Subject: [PATCH 23/23] feat(query): add nested query function --- tests/system-test/2-query/nestedQuery.py | 209 +++++++++++++++++------ 1 file changed, 155 insertions(+), 54 deletions(-) diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py index 8c61451756..871054de3a 100755 --- a/tests/system-test/2-query/nestedQuery.py +++ b/tests/system-test/2-query/nestedQuery.py @@ -52,6 +52,60 @@ class TDTestCase: # return(conn1,cur1) + def data_matrix_equal(self, sql1,row1_s,row1_e,col1_s,col1_e, sql2,row2_s,row2_e,col2_s,col2_e): + # ----row1_start----col1_start---- + # - - - - 是一个矩阵内的数据相等- - - + # - - - - - - - - - - - - - - - - + # ----row1_end------col1_end------ + self.sql1 = sql1 + list1 =[] + tdSql.query(sql1) + for i1 in range(row1_s-1,row1_e): + #print("iiii=%d"%i1) + for j1 in range(col1_s-1,col1_e): + #print("jjjj=%d"%j1) + #print("data=%s" %(tdSql.getData(i1,j1))) + list1.append(tdSql.getData(i1,j1)) + print("=====list1-------list1---=%s" %set(list1)) + + tdSql.execute("reset query cache;") + self.sql2 = sql2 + list2 =[] + tdSql.query(sql2) + for i2 in range(row2_s-1,row2_e): + #print("iiii222=%d"%i2) + for j2 in range(col2_s-1,col2_e): + #print("jjjj222=%d"%j2) + #print("data=%s" %(tdSql.getData(i2,j2))) + list2.append(tdSql.getData(i2,j2)) + print("=====list2-------list2---=%s" %set(list2)) + + if (list1 == list2) and len(list2)>0: + # print(("=====matrix===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2)) + tdLog.info(("===matrix===sql1:'%s' matrix_result = sql2:'%s' matrix_result") %(sql1,sql2)) + elif (set(list2)).issubset(set(list1)): + # 解决不同子表排列结果乱序 + # print(("=====list_issubset==matrix2in1-true===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2)) + tdLog.info(("===matrix_issubset===sql1:'%s' matrix_set_result = sql2:'%s' matrix_set_result") %(sql1,sql2)) + #elif abs(float(str(list1).replace("]","").replace("[","").replace("e+","")) - float(str(list2).replace("]","").replace("[","").replace("e+",""))) <= 0.0001: + elif abs(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace("e+","").replace(", ","").replace("(","").replace(")","").replace("-","")) - float(str(list2).replace("datetime.datetime","").replace("]","").replace("[","").replace("e+","").replace(", ","").replace("(","").replace(")","").replace("-",""))) <= 0.0001: + print(("=====matrix_abs+e+===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2)) + print(("=====matrix_abs+e+replace_after===sql1.list1:'%s',sql2.list2:'%s'") %(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace("e+","").replace(", ","").replace("(","").replace(")","").replace("-","")),float(str(list2).replace("datetime.datetime","").replace("]","").replace("[","").replace("e+","").replace(", ","").replace("(","").replace(")","").replace("-","")))) + tdLog.info(("===matrix_abs+e+===sql1:'%s' matrix_result = sql2:'%s' matrix_result") %(sql1,sql2)) + elif abs(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-","")) - float(str(list2).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-",""))) <= 0.1: + #{datetime.datetime(2021, 8, 27, 1, 46, 40), -441.46841430664057}replace + print(("=====matrix_abs+replace===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2)) + print(("=====matrix_abs+replace_after===sql1.list1:'%s',sql2.list2:'%s'") %(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-","")),float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-","")))) + tdLog.info(("===matrix_abs+replace===sql1:'%s' matrix_result = sql2:'%s' matrix_result") %(sql1,sql2)) + elif abs(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-","")) - float(str(list2).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-",""))) <= 0.5: + print(("=====matrix_abs===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2)) + print(("=====matrix_abs===sql1.list1:'%s',sql2.list2:'%s'") %(float(str(list1).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-","")),float(str(list2).replace("datetime.datetime","").replace("]","").replace("[","").replace(", ","").replace("(","").replace(")","").replace("-","")))) + tdLog.info(("===matrix_abs======sql1:'%s' matrix_result = sql2:'%s' matrix_result") %(sql1,sql2)) + else: + print(("=====matrix_error===sql1.list1:'%s',sql2.list2:'%s'") %(list1,list2)) + tdLog.info(("sql1:'%s' matrix_result != sql2:'%s' matrix_result") %(sql1,sql2)) + return tdSql.checkEqual(list1,list2) + def restartDnodes(self): pass # tdDnodes.stop(1) @@ -1702,10 +1756,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + #TD-15719 tdSql.query(sql) tdSql.query("select 17-2 from stable_1;") for i in range(self.fornum): @@ -1720,10 +1774,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 17-2.2 from stable_1;") for i in range(self.fornum): @@ -1738,10 +1792,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) self.restartDnodes() tdSql.query("select 17-3 from stable_1;") @@ -1758,10 +1812,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + #TD-15770 tdSql.query(sql) tdSql.query("select 17-4 from stable_1;") for i in range(self.fornum): @@ -1775,10 +1829,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 17-4.2 from stable_1;") for i in range(self.fornum): @@ -1792,10 +1846,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 17-5 from stable_1;") for i in range(self.fornum): @@ -1811,10 +1865,10 @@ class TDTestCase: # sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + #TD-15719 tdSql.query(sql) tdSql.query("select 17-6 from stable_1;") for i in range(self.fornum): @@ -1827,10 +1881,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + #TD-15770 tdSql.query(sql) tdSql.query("select 17-7 from stable_1;") for i in range(self.fornum): @@ -1843,10 +1897,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit1_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 17-7.2 from stable_1;") for i in range(self.fornum): @@ -1859,10 +1913,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit1_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) self.restartDnodes() tdSql.query("select 17-8 from stable_1;") @@ -1876,10 +1930,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 17-9 from stable_1;") for i in range(self.fornum): @@ -1892,10 +1946,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 17-10 from stable_1;") for i in range(self.fornum): @@ -1908,10 +1962,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) #18 select apercentile from (select calc_aggregate_alls form regualr_table or stable where <\>\in\and\or session order by limit )interval_sliding tdSql.query("select 18-1 from stable_1;") @@ -1926,10 +1980,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 18-2 from stable_1;") for i in range(self.fornum): @@ -1942,10 +1996,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 18-2.2 from stable_1;") for i in range(self.fornum): @@ -1958,10 +2012,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) self.restartDnodes() tdSql.query("select 18-3 from stable_1;") @@ -1976,10 +2030,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 18-4 from stable_1;") for i in range(self.fornum): @@ -1992,10 +2046,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 18-4.2 from stable_1;") for i in range(self.fornum): @@ -2008,10 +2062,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 18-5 from stable_1;") for i in range(self.fornum): @@ -2025,10 +2079,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_where) sql += "%s " % random.choice(limit1_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + #TD-15770 tdSql.query(sql) tdSql.query("select 18-6 from stable_1;") for i in range(self.fornum): @@ -2041,10 +2095,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 18-7 from stable_1;") for i in range(self.fornum): @@ -2057,10 +2111,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) #19 select apercentile from (select calc_aggregate_alls form regualr_table or stable where <\>\in\and\or session order by limit )interval_sliding #self.dropandcreateDB_random("%s" %db, 1) @@ -2180,10 +2234,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) tdSql.query("select 19-7 from stable_1;") for i in range(self.fornum): @@ -2195,10 +2249,10 @@ class TDTestCase: #sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " - sql += "%s " % random.choice(interval_sliding) + #sql += "%s " % random.choice(interval_sliding) tdLog.info(sql) tdLog.info(len(sql)) - #TD-15721 tdSql.query(sql) + tdSql.query(sql) #20 select * from (select calc_select_fills form regualr_table or stable where <\>\in\and\or fill_where group by order by limit offset ) #self.dropandcreateDB_random("%s" %db, 1) @@ -2351,10 +2405,21 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) - for i in range(self.fornum): - # sql_start = "select * from ( " - # sql_end = ")" - for_num = random.randint(1, 10); + sql2 = "select * from ( select * from ( select " + sql2 += "%s, " % random.choice(s_r_select) + sql2 += "%s, " % random.choice(q_select) + sql2 += "ts from regular_table_1 where " + sql2 += "%s " % random.choice(q_where) + sql2 += ")) " + tdLog.info(sql2) + tdLog.info(len(sql2)) + + self.data_matrix_equal('%s' %sql ,1,10,1,1,'%s' %sql2 ,1,10,1,1) + self.data_matrix_equal('%s' %sql ,1,10,1,1,'%s' %sql ,1,10,3,3) + self.data_matrix_equal('%s' %sql ,1,10,3,3,'%s' %sql2 ,1,10,3,3) + + for i in range(self.fornum): + for_num = random.randint(1, 15); sql = "select ts from (" * for_num sql += "select * from ( select * from ( select " sql += "%s, " % random.choice(s_r_select) @@ -2367,6 +2432,18 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + sql2 = "select * from ( select * from ( select " + sql2 += "%s, " % random.choice(s_r_select) + sql2 += "%s, " % random.choice(q_select) + sql2 += "ts from regular_table_1 where " + sql2 += "%s " % random.choice(q_where) + sql2 += ")) " + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + + self.data_matrix_equal('%s' %sql ,1,10,1,1,'%s' %sql2 ,1,10,1,1) + #2 select * from (select * from (select * form stable where <\>\in\and\or order by limit )) tdSql.query("select 2-1 from stable_1;") for i in range(self.fornum): @@ -2382,9 +2459,21 @@ class TDTestCase: tdLog.info(sql) tdLog.info(len(sql)) tdSql.query(sql) - tdSql.query("select 2-1 from stable_1;") + + sql2 = "select * from ( select * from ( select " + sql2 += "%s, " % random.choice(s_s_select) + sql2 += "%s, " % random.choice(qt_select) + sql2 += "ts from stable_1 where " + sql2 += "%s " % random.choice(q_where) + sql2 += ")) " + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + + self.data_matrix_equal('%s' %sql ,1,10,3,3,'%s' %sql2 ,1,10,3,3) + for i in range(self.fornum): - for_num = random.randint(1, 10); + for_num = random.randint(1, 15); sql = "select ts from (" * for_num sql += "select * from ( select * from ( select " sql += "%s, " % random.choice(s_s_select) @@ -2396,6 +2485,18 @@ class TDTestCase: tdLog.info(sql) tdLog.info(len(sql)) tdSql.query(sql) + + sql2 = "select ts from ( select * from ( select " + sql2 += "%s, " % random.choice(s_s_select) + sql2 += "%s, " % random.choice(qt_select) + sql2 += "ts from stable_1 where " + sql2 += "%s " % random.choice(q_where) + sql2 += ")) " + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.query(sql) + + self.data_matrix_equal('%s' %sql ,1,10,1,1,'%s' %sql2 ,1,10,1,1) #3 select ts ,calc from (select * form stable where <\>\in\and\or order by limit ) #self.dropandcreateDB_random("%s" %db, 1)