From ab604310d3e08ac3f5d34914111b4fa8349e970d Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 13 Oct 2022 17:44:48 +0800 Subject: [PATCH 001/135] test:add test case for alter buffer and pages --- tests/system-test/1-insert/alter_database.py | 56 ++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tests/system-test/1-insert/alter_database.py diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py new file mode 100644 index 0000000000..fa23f9ef4f --- /dev/null +++ b/tests/system-test/1-insert/alter_database.py @@ -0,0 +1,56 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.buffer_boundary = [3,4097,8193,12289,16384] + self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96] + # pages_boundary >= 64 + self.pages_boundary = [64,128,512] + self.pages_error = [256,self.pages_boundary[0]-1] + def alter_buffer(self): + tdSql.execute('create database db') + for buffer in self.buffer_boundary: + tdSql.execute(f'alter database db buffer {buffer}') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][8],buffer) + tdSql.execute('drop database db') + tdSql.execute('create database db vgroups 10') + for buffer in self.buffer_error: + tdSql.error(f'alter database db buffer {buffer}') + tdSql.execute('drop database db') + + def alter_pages(self): + tdSql.execute('create database db') + for pages in self.pages_boundary: + tdSql.execute(f'alter database db pages {pages}') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][10],pages) + tdSql.execute('drop database db') + tdSql.execute('create database db') + for pages in self.pages_error: + tdSql.error(f'alter database db pages {pages}') + tdSql.execute('drop database db') + + def run(self): + tdSql.error('create database db1 vgroups 10 buffer 12289') + self.alter_buffer() + self.alter_pages() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 03469918b8c645c9c5e0e1f1be0fe65d39ad6a7c Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 13 Oct 2022 17:51:54 +0800 Subject: [PATCH 002/135] update --- tests/system-test/1-insert/alter_database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index fa23f9ef4f..d3a55ee0a6 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -18,7 +18,7 @@ class TDTestCase: self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96] # pages_boundary >= 64 self.pages_boundary = [64,128,512] - self.pages_error = [256,self.pages_boundary[0]-1] + self.pages_error = [self.pages_boundary[0]-1] def alter_buffer(self): tdSql.execute('create database db') for buffer in self.buffer_boundary: @@ -39,6 +39,8 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][10],pages) tdSql.execute('drop database db') tdSql.execute('create database db') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + self.pages_error.append(tdSql.queryResult[0][10]) for pages in self.pages_error: tdSql.error(f'alter database db pages {pages}') tdSql.execute('drop database db') From bc803c3f07db1f6b01968d83ad9f6eed826b38d6 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 13 Oct 2022 17:54:19 +0800 Subject: [PATCH 003/135] add test case into ci --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 1c6aef6286..32eaa6dc34 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -18,7 +18,7 @@ python3 ./test.py -f 0-others/sysinfo.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py python3 ./test.py -f 0-others/compatibility.py - +python3 ./test.py -f 1-insert/alter_database.py python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py From bb3bb18c010cf3412a103221c0cb60699fae89fc Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 14 Oct 2022 16:24:00 +0800 Subject: [PATCH 004/135] fix: problem when last is used for system table query --- source/libs/parser/src/parTranslater.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index dbe0c97ab6..4739a852c0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2357,7 +2357,8 @@ static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNo } static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSelect) { - if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable)) { + if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) || + TSDB_SYSTEM_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType) { return TSDB_CODE_SUCCESS; } From 36bf800e154193aa450313e54d9fe8b43fe516ad Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 30 Sep 2022 13:57:38 +0800 Subject: [PATCH 005/135] fix: improve fault tolerance on disk space full with error handling enhanced --- source/dnode/mnode/impl/src/mndMain.c | 2 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 27 +++++---- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 10 +++- source/libs/sync/src/syncMain.c | 2 +- source/libs/sync/src/syncRaftLog.c | 2 - source/libs/tdb/src/db/tdbDb.c | 6 +- source/libs/tdb/src/db/tdbPager.c | 75 ++++++++++++++++++------ source/libs/wal/src/walMeta.c | 20 +++++++ source/libs/wal/src/walMgmt.c | 63 +++++++++----------- 10 files changed, 132 insertions(+), 77 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 275fc76e36..c8459fb53b 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -201,7 +201,7 @@ static int32_t mndInitWal(SMnode *pMnode) { pMnode->pWal = walOpen(path, &cfg); if (pMnode->pWal == NULL) { - mError("failed to open wal since %s", terrstr()); + mError("failed to open wal since %s. wal:%s", terrstr(), path); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 07d9b96261..bfb5cad44c 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -89,6 +89,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { data = NULL; if (vnodeEncodeInfo(pInfo, &data) < 0) { + vError("failed to encode json info."); return -1; } @@ -101,7 +102,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { } if (taosWriteFile(pFile, data, strlen(data)) < 0) { - vError("failed to write info file:%s data:%s", fname, terrstr()); + vError("failed to write info file:%s error:%s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -233,15 +234,15 @@ int vnodeCommit(SVnode *pVnode) { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } if (vnodeSaveInfo(dir, &info) < 0) { - ASSERT(0); + vError("vgId:%d, failed to save vnode info since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } walBeginSnapshot(pVnode->pWal, pVnode->state.applied); // preCommit // smaSyncPreCommit(pVnode->pSma); - if (smaAsyncPreCommit(pVnode->pSma) < 0) { - ASSERT(0); + if(smaAsyncPreCommit(pVnode->pSma) < 0){ + vError("vgId:%d, failed to async pre-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -250,44 +251,44 @@ int vnodeCommit(SVnode *pVnode) { // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit meta since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (VND_IS_RSMA(pVnode)) { if (smaAsyncCommit(pVnode->pSma) < 0) { - ASSERT(0); + vError("vgId:%d, failed to async commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA1(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA2(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } else { if (tsdbCommit(pVnode->pTsdb) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } if (tqCommit(pVnode->pTq) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tq since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // walCommit (TODO) // commit info if (vnodeCommitInfo(dir, &info) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit vnode info since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -296,7 +297,7 @@ int vnodeCommit(SVnode *pVnode) { // postCommit // smaSyncPostCommit(pVnode->pSma); if (smaAsyncPostCommit(pVnode->pSma) < 0) { - ASSERT(0); + vError("vgId:%d, failed to async post-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index de3af7cde6..54ecba6d49 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -140,7 +140,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { - vError("vgId:%d, failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode wal since %s. wal:%s", TD_VID(pVnode), tstrerror(terrno), tdir); goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 131327976f..b8047abe69 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -265,10 +265,16 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp _do_commit: vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change - vnodeCommit(pVnode); + if (vnodeCommit(pVnode) < 0) { + vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } // start a new one - vnodeBegin(pVnode); + if (vnodeBegin(pVnode) < 0) { + vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } } return 0; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 378ac743b3..d6da42fd42 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2726,7 +2726,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); if (code != 0) { // del resp mgr, call FpCommitCb - ASSERT(0); + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 496c8419de..1cfe37dc7e 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -234,8 +234,6 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pEntry->index, err, err, errStr, sysErr, sysErrStr); syncNodeErrorLog(pData->pSyncNode, logBuf); - - ASSERT(0); return -1; } pEntry->index = index; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index ea16e80562..a7fc50dc86 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -106,7 +106,7 @@ int32_t tdbBegin(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerBegin(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to begin pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -121,7 +121,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerCommit(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to commit pager. dbName:%s, txnId:%d", pDb->dbName, pTxn->txnId); return -1; } } @@ -136,7 +136,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerAbort(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to abort pager. dbName:%s, txnId:%d", pDb->dbName, pTxn->txnId); return -1; } } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 04711eb6a0..ee4fc90228 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -210,7 +210,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page to journal since %s", tstrerror(terrno)); return -1; } } @@ -226,6 +226,8 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { // Open the journal pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (pPager->jfd < 0) { + tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -243,9 +245,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { // sync the journal file ret = tdbOsFSync(pPager->jfd); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } // loop to write the dirty pages to file @@ -255,7 +257,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage = (SPage *)pNode; ret = tdbPagerWritePageToDB(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page to db since %s", tstrerror(terrno)); return -1; } } @@ -277,11 +279,25 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { tRBTreeCreate(&pPager->rbt, pageCmpFn); // sync the db file - tdbOsFSync(pPager->fd); + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } // remove the journal file - tdbOsClose(pPager->jfd); - tdbOsRemove(pPager->jFileName); + if (tdbOsClose(pPager->jfd) < 0) { + tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pPager->inTran = 0; return 0; @@ -297,14 +313,14 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // 0, sync the journal file ret = tdbOsFSync(pPager->jfd); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to fsync jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755); if (jfd == NULL) { - return 0; + return -1; } ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); @@ -516,11 +532,15 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno)); if (ret < 0) { + tdbError("failed to write pgno due to %s. file:%s, pgno:%u", strerror(errno), pPager->jFileName, pgno); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -540,13 +560,15 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - ASSERT(0); + tdbError("failed to lseek due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName, pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -590,23 +612,38 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - ASSERT(0); + tdbError("failed to lseek fd due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize); if (ret < 0) { - ASSERT(0); + tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, pPager->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - tdbOsFSync(pPager->fd); + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } tdbOsFree(pageBuf); - tdbOsClose(jfd); - tdbOsRemove(pPager->jFileName); + if (tdbOsClose(jfd) < 0) { + tdbError("failed to close jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } return 0; } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index e49a963191..de9c115067 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -492,14 +492,20 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0); cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField; pRoot = cJSON_Parse(bytes); + if (!pRoot) goto _err; pMeta = cJSON_GetObjectItem(pRoot, "meta"); + if (!pMeta) goto _err; pField = cJSON_GetObjectItem(pMeta, "firstVer"); + if (!pField) goto _err; pWal->vers.firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "snapshotVer"); + if (!pField) goto _err; pWal->vers.snapshotVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "commitVer"); + if (!pField) goto _err; pWal->vers.commitVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "lastVer"); + if (!pField) goto _err; pWal->vers.lastVer = atoll(cJSON_GetStringValue(pField)); pFiles = cJSON_GetObjectItem(pRoot, "files"); @@ -510,16 +516,22 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { SWalFileInfo* pData = pArray->pData; for (int i = 0; i < sz; i++) { cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + if (!pInfoJson) goto _err; SWalFileInfo* pInfo = &pData[i]; pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); + if (!pField) goto _err; pInfo->firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "lastVer"); + if (!pField) goto _err; pInfo->lastVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "createTs"); + if (!pField) goto _err; pInfo->createTs = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "closeTs"); + if (!pField) goto _err; pInfo->closeTs = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "fileSize"); + if (!pField) goto _err; pInfo->fileSize = atoll(cJSON_GetStringValue(pField)); } taosArraySetSize(pArray, sz); @@ -527,6 +539,10 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { pWal->writeCur = sz - 1; cJSON_Delete(pRoot); return 0; + +_err: + cJSON_Delete(pRoot); + return -1; } static int walFindCurMetaVer(SWal* pWal) { @@ -626,6 +642,10 @@ int walLoadMeta(SWal* pWal) { } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); + if (code < 0) { + wError("failed to deserialize wal meta. file:%s", fnameStr); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + } taosCloseFile(&pFile); taosMemoryFree(buf); return code; diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 7974f3e32e..96f9522c9f 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -81,6 +81,12 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } + if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pWal); + return NULL; + } + // set config memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); @@ -98,15 +104,14 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { tstrncpy(pWal->path, path, sizeof(pWal->path)); if (taosMkDir(pWal->path) != 0) { wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - taosMemoryFree(pWal); - return NULL; + goto _err; } // init ref pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (pWal->pRefHash == NULL) { - taosMemoryFree(pWal); - return NULL; + wError("failed to init hash since %s", tstrerror(terrno)); + goto _err; } // open meta @@ -117,9 +122,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - taosHashCleanup(pWal->pRefHash); - taosMemoryFree(pWal); - return NULL; + goto _err; } // init status @@ -131,46 +134,36 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeHead.head.protoVer = WAL_PROTO_VER; pWal->writeHead.magic = WAL_MAGIC; - if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { - taosArrayDestroy(pWal->fileInfoSet); - taosHashCleanup(pWal->pRefHash); - taosMemoryFree(pWal); - return NULL; - } - - pWal->refId = taosAddRef(tsWal.refSetId, pWal); - if (pWal->refId < 0) { - taosHashCleanup(pWal->pRefHash); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - taosMemoryFree(pWal); - return NULL; - } - - walLoadMeta(pWal); + (void)walLoadMeta(pWal); if (walCheckAndRepairMeta(pWal) < 0) { wError("vgId:%d cannot open wal since repair meta file failed", pWal->cfg.vgId); - taosHashCleanup(pWal->pRefHash); - taosRemoveRef(tsWal.refSetId, pWal->refId); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - return NULL; + goto _err; } if (walCheckAndRepairIdx(pWal) < 0) { wError("vgId:%d cannot open wal since repair idx file failed", pWal->cfg.vgId); - taosHashCleanup(pWal->pRefHash); - taosRemoveRef(tsWal.refSetId, pWal->refId); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - return NULL; + goto _err; + } + + // add ref + pWal->refId = taosAddRef(tsWal.refSetId, pWal); + if (pWal->refId < 0) { + wError("failed to add ref for Wal since %s", tstrerror(terrno)); + goto _err; } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); - return pWal; + +_err: + taosArrayDestroy(pWal->fileInfoSet); + taosHashCleanup(pWal->pRefHash); + taosThreadMutexDestroy(&pWal->mutex); + taosMemoryFree(pWal); + pWal = NULL; + return NULL; } int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { From 76865258cf4b82c8ffcc04293b5aa3940ed55d5a Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 30 Sep 2022 15:49:23 +0800 Subject: [PATCH 006/135] enh: protect WAL meta by flushing to a tmpfile at first and then renaming --- source/libs/tdb/src/db/tdbDb.c | 4 +-- source/libs/wal/src/walMeta.c | 56 ++++++++++++++++++++++++++++------ 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index a7fc50dc86..6c01348bc2 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -121,7 +121,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerCommit(pPager, pTxn); if (ret < 0) { - tdbError("failed to commit pager. dbName:%s, txnId:%d", pDb->dbName, pTxn->txnId); + tdbError("failed to commit pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -136,7 +136,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerAbort(pPager, pTxn); if (ret < 0) { - tdbError("failed to abort pager. dbName:%s, txnId:%d", pDb->dbName, pTxn->txnId); + tdbError("failed to abort pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index de9c115067..597fd0000b 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -35,8 +35,12 @@ int64_t FORCE_INLINE walGetCommittedVer(SWal* pWal) { return pWal->vers.commitVe int64_t FORCE_INLINE walGetAppliedVer(SWal* pWal) { return pWal->vers.appliedVer; } -static FORCE_INLINE void walBuildMetaName(SWal* pWal, int metaVer, char* buf) { - sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); +static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { + return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); +} + +static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { + return sprintf(buf, "%s/meta-ver.tmp", pWal->path); } static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { @@ -578,22 +582,51 @@ static int walFindCurMetaVer(SWal* pWal) { int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; - walBuildMetaName(pWal, metaVer + 1, fnameStr); - TdFilePtr pMetaFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE); + char tmpFnameStr[WAL_FILE_LEN]; + int n; + + // flush to a tmpfile + n = walBuildTmpMetaName(pWal, tmpFnameStr); + ASSERT(n < sizeof(tmpFnameStr) && "Buffer overflow of file name"); + + TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pMetaFile == NULL) { + wError("failed to open file due to %s. file:%s", strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); if (len != taosWriteFile(pMetaFile, serialized, len)) { // TODO:clean file - - taosCloseFile(&pMetaFile); - taosRemoveFile(fnameStr); - return -1; + wError("failed to write file due to %s. file:%s", strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosFsyncFile(pMetaFile) < 0) { + wError("failed to sync file due to %s. file:%s", strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosCloseFile(&pMetaFile) < 0) { + wError("failed to close file due to %s. file:%s", strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // rename it + n = walBuildMetaName(pWal, metaVer + 1, fnameStr); + ASSERT(n < sizeof(fnameStr) && "Buffer overflow of file name"); + + if (taosRenameFile(tmpFnameStr, fnameStr) < 0) { + wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } - taosCloseFile(&pMetaFile); // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); @@ -601,6 +634,11 @@ int walSaveMeta(SWal* pWal) { } taosMemoryFree(serialized); return 0; + +_err: + taosCloseFile(&pMetaFile); + taosMemoryFree(serialized); + return -1; } int walLoadMeta(SWal* pWal) { From 9055f63742231dc87284e36e0de60867b9d982f7 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 30 Sep 2022 18:54:49 +0800 Subject: [PATCH 007/135] fix: ftruncate WAL log after the last valid WAL record properly in walScanLogGetLastVer --- source/libs/wal/src/walMeta.c | 79 +++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 597fd0000b..797393c47e 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -45,12 +45,11 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - ASSERT(sz > 0); -#if 0 - for (int i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); + if (sz <= 0) { + wError("No WAL log file found."); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; } -#endif SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1); char fnameStr[WAL_FILE_LEN]; @@ -61,7 +60,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize); pLastFileInfo->fileSize = fileSize; - TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -69,28 +68,30 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { uint64_t magic = WAL_MAGIC; - char* buf = taosMemoryMalloc(readSize + 5); + char* buf = taosMemoryMalloc(readSize + sizeof(uint64_t)); if (buf == NULL) { - taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; - return -1; + goto _err; } int64_t offset; offset = taosLSeekFile(pFile, -readSize, SEEK_END); if (readSize != taosReadFile(pFile, buf, readSize)) { - taosMemoryFree(buf); - taosCloseFile(&pFile); terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + goto _err; } + int64_t walCkHeadSz = sizeof(SWalCkHead); char* found = NULL; while (1) { char* haystack = buf; - char* candidate; + char* candidate = NULL; while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { // read and validate + int64_t len = readSize - (candidate - buf); + if (len < walCkHeadSz) { + break; + } SWalCkHead* logContent = (SWalCkHead*)candidate; if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { found = candidate; @@ -98,35 +99,26 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { haystack = candidate + 1; } if (found || offset == 0) break; - offset = TMIN(0, offset - readSize + sizeof(uint64_t)); - int64_t offset2 = taosLSeekFile(pFile, offset, SEEK_SET); - ASSERT(offset == offset2); - if (readSize != taosReadFile(pFile, buf, readSize)) { - taosMemoryFree(buf); - taosCloseFile(&pFile); + + // go backwards, i.e. by at most one WAL scan buf size + offset = TMAX(0, offset - readSize + walCkHeadSz); + int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); + if (ret < 0) { + wError("failed to lseek file due to %s. offset:%lld", strerror(errno), offset); terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + goto _err; } -#if 0 - if (found == buf) { - SWalCkHead* logContent = (SWalCkHead*)found; - if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { - // file has to be deleted - taosMemoryFree(buf); - taosCloseFile(&pFile); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; - } + ASSERT(offset == ret); + if (readSize != taosReadFile(pFile, buf, readSize)) { + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } -#endif } if (found == NULL) { - // file corrupted, no complete log - // TODO delete and search in previous files - /*ASSERT(0);*/ + wError("WAL log file corrupted: no valid WAL record found. file: %s", fnameStr); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; + goto _err; } // truncate file @@ -137,15 +129,28 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { if (lastEntryEndOffset != fileSize) { wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, fileSize); - taosFtruncateFile(pFile, lastEntryEndOffset); + if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) { + wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (taosFsyncFile(pFile) < 0) { + wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset; pWal->totSize -= (fileSize - lastEntryEndOffset); } taosCloseFile(&pFile); taosMemoryFree(buf); - return retVer; + +_err: + taosCloseFile(&pFile); + taosMemoryFree(buf); + return -1; } int walCheckAndRepairMeta(SWal* pWal) { From 715f53a856c645b5324575ce2bcabd6401f6fe30 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 30 Sep 2022 22:57:51 +0800 Subject: [PATCH 008/135] fix: eliminate asserts on errors of appending or getting log entry --- source/libs/sync/src/syncAppendEntries.c | 4 ++++ source/libs/sync/src/syncCommit.c | 5 ++++- source/libs/sync/src/syncMain.c | 7 +++++-- source/libs/sync/src/syncRaftLog.c | 18 +++++++++++++----- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index e000ba8bf8..4a35a15d3e 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -579,6 +579,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -710,6 +711,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -859,6 +861,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -974,6 +977,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index b604d25816..e8ad22a10d 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -76,7 +76,10 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); } else { pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); - ASSERT(pEntry != NULL); + if (pEntry == NULL) { + sError("failed to get entry since %s. index:%lld", tstrerror(terrno), index); + return; + } } // cannot commit, even if quorum agree. need check term! if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index d6da42fd42..ad0390ac7c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2650,7 +2650,10 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { if (ths->state == TAOS_SYNC_STATE_LEADER) { int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); - ASSERT(code == 0); + if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); + return -1; + } syncNodeReplicate(ths, false); } @@ -2790,8 +2793,8 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); // del resp mgr, call FpCommitCb - ASSERT(0); return -1; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 1cfe37dc7e..c3dad104d1 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -412,13 +412,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { return 0; } -SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { +SSyncRaftEntry* logStoreGetEntryWithoutLock(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { - taosThreadMutexLock(&(pData->mutex)); - // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); SWalReader* pWalHandle = pData->pWalHandle; ASSERT(pWalHandle != NULL); @@ -442,7 +440,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { } } while (0); - ASSERT(0); + sError("failed to read ver since %s. index:%lld", tstrerror(terrno), index); + return NULL; } SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); @@ -463,7 +462,6 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { terrno = saveErr; */ - taosThreadMutexUnlock(&(pData->mutex)); return pEntry; } else { @@ -471,6 +469,16 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { } } +SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { + SSyncLogStoreData* pData = pLogStore->data; + SSyncRaftEntry *pEntry = NULL; + + taosThreadMutexLock(&pData->mutex); + pEntry = logStoreGetEntryWithoutLock(pLogStore, index); + taosThreadMutexUnlock(&pData->mutex); + return pEntry; +} + int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; From 56b32a77a4428b7e2752a83fe2d53ec197e418c0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 8 Oct 2022 15:10:02 +0800 Subject: [PATCH 009/135] fix: allow to restart on disk space full, e.g. in dmCheckDiskSpace --- include/os/osEnv.h | 8 +++++++- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 24 ++++++------------------ source/os/src/osEnv.c | 12 +++++++++--- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 293d9d17f8..d4e94d6173 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -49,9 +49,15 @@ extern SDiskSpace tsTempSpace; void osDefaultInit(); void osUpdate(); void osCleanup(); + bool osLogSpaceAvailable(); bool osDataSpaceAvailable(); bool osTempSpaceAvailable(); + +bool osLogSpaceSufficient(); +bool osDataSpaceSufficient(); +bool osTempSpaceSufficient(); + void osSetTimezone(const char *timezone); void osSetSystemLocale(const char *inLocale, const char *inCharSet); @@ -59,4 +65,4 @@ void osSetSystemLocale(const char *inLocale, const char *inCharSet); } #endif -#endif /*_TD_OS_ENV_H_*/ \ No newline at end of file +#endif /*_TD_OS_ENV_H_*/ diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index d07ec7abb0..076826ebc2 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -51,26 +51,14 @@ static int32_t dmInitMonitor() { static bool dmCheckDiskSpace() { osUpdate(); - if (!osDataSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least , quit", - (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osDataSpaceSufficient()) { + dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); } - if (!osLogSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least, quit", - (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osLogSpaceSufficient()) { + dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least", (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); } - if (!osTempSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least, quit", - (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osTempSpaceSufficient()) { + dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); } return true; } diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index f0442c6fd1..616ab7875d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -87,11 +87,17 @@ void osUpdate() { void osCleanup() {} -bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } +bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; } -bool osDataSpaceAvailable() { return tsDataSpace.reserved <= tsDataSpace.size.avail; } +bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; } -bool osTempSpaceAvailable() { return tsTempSpace.reserved <= tsTempSpace.size.avail; } +bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; } + +bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; } + +bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; } + +bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; } void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, tsTimezoneStr, &tsDaylight, &tsTimezone); } From df18ddbbb52f3b7436f2ff2b0debf862233214f1 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 8 Oct 2022 15:16:45 +0800 Subject: [PATCH 010/135] enh: remove WAL log file corrupted completely --- source/libs/wal/src/walMeta.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 797393c47e..be74b6bfc9 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -118,6 +118,8 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { if (found == NULL) { wError("WAL log file corrupted: no valid WAL record found. file: %s", fnameStr); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + taosRemoveFile(fnameStr); + wWarn("vgId:%d, remove corrupted WAL log file: %s", pWal->cfg.vgId, fnameStr); goto _err; } From 63b13db10f88597ce94f426eec9e22ffe0c372bd Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 8 Oct 2022 20:38:49 +0800 Subject: [PATCH 011/135] fix: eliminate asserts on getting and committing log entries --- source/libs/sync/src/syncCommit.c | 4 +++- source/libs/sync/src/syncMain.c | 5 ++++- source/libs/wal/src/walMeta.c | 1 + source/libs/wal/src/walRead.c | 6 +++++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index e8ad22a10d..511113352e 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -130,7 +130,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { // execute fsm if (pSyncNode->pFsm != NULL) { int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state); - ASSERT(code == 0); + if (code != 0) { + wError("failed to commit sync node since %s", tstrerror(terrno)); + } } } } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ad0390ac7c..bfd933e163 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -3046,7 +3046,10 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); } else { code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); - ASSERT(code == 0); + if (code != 0) { + sError("failed to get log entry since %s. index:%lld", tstrerror(terrno), i); + return -1; + } ASSERT(pEntry != NULL); } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index be74b6bfc9..7cc92c46e9 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -62,6 +62,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); if (pFile == NULL) { + wError("failed to open file due to %s. file:%s", strerror(errno), fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index c4f1a81eeb..248ab91fab 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -181,7 +181,11 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(pRet != NULL); + if (pRet == NULL) { + wError("failed to find WAL log file with ver:%lld", ver); + terrno = TSDB_CODE_WAL_INVALID_VER; + return -1; + } if (pReader->curFileFirstVer != pRet->firstVer) { // error code was set inner if (walReadChangeFile(pReader, pRet->firstVer) < 0) { From 948fa60ef0532fa94d8ff5de54e78a33ddcf2bcb Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 9 Oct 2022 11:52:06 +0800 Subject: [PATCH 012/135] fix: check the length of remaining content for computing CRC32 chksum of WAL record body --- source/libs/wal/src/walMeta.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 7cc92c46e9..7bf707a8ef 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -94,8 +94,10 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { break; } SWalCkHead* logContent = (SWalCkHead*)candidate; - if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { - found = candidate; + if (walValidHeadCksum(logContent) == 0 && + walCkHeadSz + logContent->head.bodyLen <= len && + walValidBodyCksum(logContent) == 0) { + found = candidate; } haystack = candidate + 1; } From c693384ec4c794087d3d7f1f603f9c72cdf1f3ed Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 9 Oct 2022 11:58:28 +0800 Subject: [PATCH 013/135] enh: replace asserts with error logs in walReadVer --- source/libs/wal/src/walRead.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 248ab91fab..1ac15e4bb8 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -476,7 +476,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - ASSERT(0); + wError("vgId:%d, failed to read WAL record head from log file since %s", pReader->pWal->cfg.vgId, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -509,6 +509,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } + wError("vgId:%d, failed to read WAL record body from log file since %s", pReader->pWal->cfg.vgId, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } From 3af05070cf9f1b82c6da496d858e9e9e952c2efe Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 9 Oct 2022 14:11:17 +0800 Subject: [PATCH 014/135] fix: assert on availability of inUse SVBufPool for vnode --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 3 +++ source/dnode/vnode/src/vnd/vnodeBufPool.c | 2 ++ 2 files changed, 5 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index a6628463f8..6679abd247 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -160,6 +160,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid goto _err; } + ASSERT(pPool != NULL); // do delete SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData)); if (pDelData == NULL) { @@ -353,6 +354,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; + ASSERT(pPool != NULL); pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); if (pTbData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -492,6 +494,7 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN // node level = tsdbMemSkipListRandLevel(&pTbData->sl); + ASSERT(pPool != NULL); pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow)); if (pNode == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 3c0a041197..d21cd957a9 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -112,6 +112,8 @@ void vnodeBufPoolReset(SVBufPool *pPool) { void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { SVBufPoolNode *pNode; void *p = NULL; + ASSERT(pPool != NULL); + taosThreadSpinLock(&pPool->lock); if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { // allocate from the anchor node From 7fb827debdea2ad8addb4de84fc3c321cc3d3246 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 9 Oct 2022 20:39:14 +0800 Subject: [PATCH 015/135] fix: refactor walScanLogGetLastVer, walCheckAndRepairMeta, and walCheckAndRepairIdx. * search for the last entry in the contiguous range of valid WAL from the last pos fsynced as firstTrial * find the last entry before the last pos fsynced backwards as second trial * reserve sufficient space for computing CRC32 checksum, esp. of WAL record body * rebuild meta info to resolve potential misalignment between lists of meta and actual log files * retract commitIndex and appliedIndex to lastLogIndex if needed * put an upper size limit on possibly corrupted WAL range to be recovered --- include/libs/wal/wal.h | 1 + include/util/taoserror.h | 1 + source/client/src/clientMain.c | 9 +- source/libs/wal/src/walMeta.c | 671 ++++++++++++++++++++------------- source/libs/wal/src/walMgmt.c | 8 +- source/libs/wal/src/walRead.c | 6 +- source/libs/wal/src/walSeek.c | 10 + source/libs/wal/src/walWrite.c | 5 + source/util/src/terror.c | 3 +- 9 files changed, 442 insertions(+), 272 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index de31a970df..adf244e32a 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -43,6 +43,7 @@ extern "C" { #define WAL_FILE_LEN (WAL_PATH_LEN + 32) #define WAL_MAGIC 0xFAFBFCFDF4F3F2F1ULL #define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3) +#define WAL_RECOV_SIZE_LIMIT (100 * WAL_SCAN_BUF_SIZE) typedef enum { TAOS_WAL_WRITE = 1, diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 6bc0e0e7dd..bbddb539c6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -450,6 +450,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) +#define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006) // tfs #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 6f8cef7c0d..fa0a4ac743 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -146,7 +146,7 @@ void taos_close(TAOS *taos) { int taos_errno(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return terrno; } @@ -154,13 +154,12 @@ int taos_errno(TAOS_RES *res) { return 0; } - return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL - : ((SRequestObj *)res)->code; + return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return (const char *)tstrerror(terrno); } @@ -172,7 +171,7 @@ const char *taos_errstr(TAOS_RES *res) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { - return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL) + return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY) : (const char *)tstrerror(pRequest->code); } } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 7bf707a8ef..42291cd264 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -43,110 +43,177 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { return sprintf(buf, "%s/meta-ver.tmp", pWal->path); } -static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { +static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - if (sz <= 0) { - wError("No WAL log file found."); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; - } + terrno = TSDB_CODE_SUCCESS; + ASSERT(fileIdx >= 0 && fileIdx < sz); - SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1); + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); int64_t fileSize = 0; taosStatFile(fnameStr, &fileSize, NULL); - int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize); - pLastFileInfo->fileSize = fileSize; TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); if (pFile == NULL) { - wError("failed to open file due to %s. file:%s", strerror(errno), fnameStr); + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + // ensure size as non-negative + pFileInfo->fileSize = TMAX(0, pFileInfo->fileSize); + uint64_t magic = WAL_MAGIC; + int64_t walCkHeadSz = sizeof(SWalCkHead); + int64_t end = fileSize; + int64_t offset = 0; + int32_t capacity = 0; + int32_t readSize = 0; + char* buf = NULL; + char* found = NULL; + bool firstTrial = pFileInfo->fileSize < fileSize; - char* buf = taosMemoryMalloc(readSize + sizeof(uint64_t)); - if (buf == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; - goto _err; - } - - int64_t offset; - offset = taosLSeekFile(pFile, -readSize, SEEK_END); - if (readSize != taosReadFile(pFile, buf, readSize)) { - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - int64_t walCkHeadSz = sizeof(SWalCkHead); - char* found = NULL; + // search for the valid last WAL entry, e.g. block by block while (1) { - char* haystack = buf; + offset = (firstTrial) ? pFileInfo->fileSize : TMAX(0, end - WAL_SCAN_BUF_SIZE); + ASSERT(offset <= end); + readSize = end - offset; + capacity = readSize + sizeof(magic); + + int64_t limit = WAL_RECOV_SIZE_LIMIT; + if (limit < readSize) { + wError("vgId:%d, possibly corrupted WAL range exceeds size limit (i.e. %" PRId64 " bytes). offset:%" PRId64 + ", end:%" PRId64 ", file:%s", + pWal->cfg.vgId, limit, offset, end, fnameStr); + terrno = TSDB_CODE_WAL_SIZE_LIMIT; + goto _err; + } + + void* ptr = taosMemoryRealloc(buf, capacity); + if (ptr == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + goto _err; + } + buf = ptr; + + int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (readSize != taosReadFile(pFile, buf, readSize)) { + wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + readSize, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + char* candidate = NULL; - while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { - // read and validate + char* haystack = buf; + + while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) { + // validate head int64_t len = readSize - (candidate - buf); if (len < walCkHeadSz) { - break; + break; } SWalCkHead* logContent = (SWalCkHead*)candidate; - if (walValidHeadCksum(logContent) == 0 && - walCkHeadSz + logContent->head.bodyLen <= len && - walValidBodyCksum(logContent) == 0) { - found = candidate; + if (walValidHeadCksum(logContent) != 0) { + wError("vgId:%d, failed to validate checksum of wal entry header. offset:% %" PRId64 ", file:%s", + ((char*)(logContent)-buf), fnameStr); + haystack = candidate + 1; + if (firstTrial) { + break; + } else { + continue; + } } + + // validate body + int64_t size = walCkHeadSz + logContent->head.bodyLen; + if (len < size) { + int64_t extraSize = size - len; + if (capacity < readSize + extraSize + sizeof(magic)) { + capacity += extraSize; + void* ptr = taosMemoryRealloc(buf, capacity); + if (ptr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + buf = ptr; + } + int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), + offset); + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } + if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) { + wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s", + pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } + } + if (walValidBodyCksum(logContent) != 0) { + terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; + wError("vgId:%d, failed to validate checksum of wal entry body. offset:% %" PRId64 ", file:%s", + ((char*)(logContent)-buf), fnameStr); + haystack = candidate + 1; + if (firstTrial) { + break; + } else { + continue; + } + } + + // found one + found = candidate; haystack = candidate + 1; } + if (found || offset == 0) break; - // go backwards, i.e. by at most one WAL scan buf size - offset = TMAX(0, offset - readSize + walCkHeadSz); - int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); - if (ret < 0) { - wError("failed to lseek file due to %s. offset:%lld", strerror(errno), offset); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - ASSERT(offset == ret); - if (readSize != taosReadFile(pFile, buf, readSize)) { - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // go backwards, e.g. by at most one WAL scan buf size + end = offset + walCkHeadSz - 1; + firstTrial = false; } - if (found == NULL) { - wError("WAL log file corrupted: no valid WAL record found. file: %s", fnameStr); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - taosRemoveFile(fnameStr); - wWarn("vgId:%d, remove corrupted WAL log file: %s", pWal->cfg.vgId, fnameStr); - goto _err; + // determine end of last entry + SWalCkHead* lastEntry = (SWalCkHead*)found; + int64_t retVer = -1; + int64_t lastEntryBeginOffset = 0; + int64_t lastEntryEndOffset = 0; + + if (lastEntry == NULL) { + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + } else { + retVer = lastEntry->head.version; + lastEntryBeginOffset = offset + (int64_t)((char*)lastEntry - (char*)buf); + lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; } // truncate file - SWalCkHead* lastEntry = (SWalCkHead*)found; - int64_t retVer = lastEntry->head.version; - int64_t lastEntryBeginOffset = offset + (int64_t)((char*)found - (char*)buf); - int64_t lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; if (lastEntryEndOffset != fileSize) { - wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, + wWarn("vgId:%d, repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, fileSize); if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) { - wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } if (taosFsyncFile(pFile) < 0) { - wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } - ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset; - pWal->totSize -= (fileSize - lastEntryEndOffset); } + pFileInfo->fileSize = lastEntryEndOffset; taosCloseFile(&pFile); taosMemoryFree(buf); @@ -158,13 +225,69 @@ _err: return -1; } +static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) { + int metaFileNum = taosArrayGetSize(metaLogList); + int actualFileNum = taosArrayGetSize(actualLogList); + int j = 0; + + // both of the lists in asc order + for (int i = 0; i < actualFileNum; i++) { + SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i); + while (j < metaFileNum) { + SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j); + ASSERT(pMetaInfo != NULL); + if (pMetaInfo->firstVer < pLogInfo->firstVer) { + j++; + } else if (pMetaInfo->firstVer == pLogInfo->firstVer) { + (*pLogInfo) = *pMetaInfo; + j++; + break; + } else { + break; + } + } + } + + taosArrayClear(metaLogList); + + for (int i = 0; i < actualFileNum; i++) { + SWalFileInfo* pFileInfo = taosArrayGet(actualLogList, i); + taosArrayPush(metaLogList, pFileInfo); + } +} + +void walAlignVersions(SWal* pWal) { + if (pWal->vers.firstVer > pWal->vers.snapshotVer) { + wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + pWal->vers.firstVer, pWal->vers.snapshotVer); + pWal->vers.firstVer = pWal->vers.snapshotVer; + } + if (pWal->vers.lastVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + pWal->vers.lastVer, pWal->vers.snapshotVer); + pWal->vers.lastVer = pWal->vers.snapshotVer; + } + if (pWal->vers.commitVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + pWal->vers.commitVer, pWal->vers.snapshotVer); + pWal->vers.commitVer = pWal->vers.snapshotVer; + } + if (pWal->vers.appliedVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + pWal->vers.appliedVer, pWal->vers.snapshotVer); + pWal->vers.appliedVer = pWal->vers.snapshotVer; + } + + pWal->vers.commitVer = TMIN(pWal->vers.lastVer, pWal->vers.commitVer); + pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer); +} + int walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info const char* logPattern = "^[0-9]+.log$"; const char* idxPattern = "^[0-9]+.idx$"; regex_t logRegPattern; regex_t idxRegPattern; - bool fixed = false; regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); @@ -198,222 +321,238 @@ int walCheckAndRepairMeta(SWal* pWal) { taosArraySort(actualLog, compareWalFileInfo); - int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); - int actualFileNum = taosArrayGetSize(actualLog); + int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); + int actualFileNum = taosArrayGetSize(actualLog); + int64_t firstVerPrev = pWal->vers.firstVer; + int64_t lastVerPrev = pWal->vers.lastVer; + int64_t totSize = 0; + bool updateMeta = (metaFileNum != actualFileNum); -#if 0 - for (int32_t fileNo = actualFileNum - 1; fileNo >= 0; fileNo--) { - SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, fileNo); + // rebuild meta of file info + walRebuildFileInfoSet(pWal->fileInfoSet, actualLog); + taosArrayDestroy(actualLog); + + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + ASSERT(sz == actualFileNum); + + // scan and determine the lastVer + int32_t fileIdx = sz; + + while (--fileIdx >= 0) { char fnameStr[WAL_FILE_LEN]; + int64_t fileSize = 0; + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - int64_t fileSize = 0; - taosStatFile(fnameStr, &fileSize, NULL); - if (fileSize == 0) { + int32_t code = taosStatFile(fnameStr, &fileSize, NULL); + if (code < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("failed to stat file since %s. file:%s", terrstr(), fnameStr); + return -1; + } + + ASSERT(pFileInfo->firstVer >= 0); + + if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) { + totSize += pFileInfo->fileSize; + continue; + } + updateMeta = true; + + int64_t lastVer = walScanLogGetLastVer(pWal, fileIdx); + if (lastVer < 0) { + if (terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { + wError("failed to scan wal last ver since %s", terrstr()); + return -1; + } + ASSERT(pFileInfo->fileSize == 0); + // remove the empty wal log, and its idx taosRemoveFile(fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); taosRemoveFile(fnameStr); - taosArrayPop(pLogInfoArray); - } else { - break; + // remove its meta entry + taosArrayRemove(pWal->fileInfoSet, fileIdx); + continue; } + + // update lastVer + pFileInfo->lastVer = lastVer; + totSize += pFileInfo->fileSize; } - actualFileNum = taosArrayGetSize(pLogInfoArray); -#endif - - { - int32_t i = 0, j = 0; - while (i < actualFileNum && j < metaFileNum) { - SWalFileInfo* pActualFile = taosArrayGet(actualLog, i); - SWalFileInfo* pMetaFile = taosArrayGet(pWal->fileInfoSet, j); - if (pActualFile->firstVer < pMetaFile->firstVer) { - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - i++; - } else if (pActualFile->firstVer > pMetaFile->firstVer) { - taosArrayRemove(pWal->fileInfoSet, j); - metaFileNum--; - } else { - i++; - j++; - } - } - if (i == actualFileNum && j == metaFileNum) { - if (j > 0) { - SWalFileInfo* pLastInfo = taosArrayGet(pWal->fileInfoSet, j - 1); - int64_t fsize = 0; - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pLastInfo->firstVer, fNameStr); - taosStatFile(fNameStr, &fsize, NULL); - if (pLastInfo->fileSize != fsize) { - fixed = true; - pLastInfo->fileSize = fsize; - pLastInfo->lastVer = walScanLogGetLastVer(pWal); - } - } - } else { - fixed = true; - while (i < actualFileNum) { - SWalFileInfo* pActualFile = taosArrayGet(actualLog, i); - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pActualFile->firstVer, fNameStr); - taosStatFile(fNameStr, &pActualFile->fileSize, NULL); - - if (pActualFile->fileSize == 0) { - ASSERT(i == actualFileNum - 1); - taosRemoveFile(fNameStr); - - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - break; - } - - if (i < actualFileNum - 1) { - pActualFile->lastVer = ((SWalFileInfo*)taosArrayGet(actualLog, i + 1))->firstVer - 1; - taosArrayPush(pWal->fileInfoSet, pActualFile); - i++; - } else { - pActualFile = taosArrayPush(pWal->fileInfoSet, pActualFile); - pActualFile->lastVer = walScanLogGetLastVer(pWal); - if (pActualFile->lastVer == -1) { - taosRemoveFile(fNameStr); - - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - taosArrayPop(pWal->fileInfoSet); - } - break; - } - } - } - } - -#if 0 - if (metaFileNum > actualFileNum) { - taosArrayPopFrontBatch(pWal->fileInfoSet, metaFileNum - actualFileNum); - } else if (metaFileNum < actualFileNum) { - for (int i = metaFileNum; i < actualFileNum; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(actualLog, i); - taosArrayPush(pWal->fileInfoSet, pFileInfo); - } - } -#endif - - taosArrayDestroy(actualLog); - + // reset vers info and so on actualFileNum = taosArrayGetSize(pWal->fileInfoSet); pWal->writeCur = actualFileNum - 1; - + pWal->totSize = totSize; + pWal->vers.firstVer = -1; + pWal->vers.lastVer = -1; if (actualFileNum > 0) { - int64_t fLastVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur))->lastVer; - if (fLastVer != -1 && pWal->vers.lastVer != fLastVer) { - fixed = true; - pWal->vers.lastVer = fLastVer; - } - int64_t fFirstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; - if (fFirstVer != pWal->vers.firstVer) { - fixed = true; - pWal->vers.firstVer = fFirstVer; - } + pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer; + } + (void)walAlignVersions(pWal); + + // update meta file + if (updateMeta) { + (void)walSaveMeta(pWal); + } + return 0; +} + +int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) { + if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } - if (fixed) { - walSaveMeta(pWal); + if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (walValidHeadCksum(pCkHead) != 0) { + terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; + return -1; } return 0; } -int walCheckAndRepairIdx(SWal* pWal) { +int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - for (int32_t i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); + ASSERT(fileIdx >= 0 && fileIdx < sz); + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); + char fLogNameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); + int64_t fileSize = 0; - char fnameStr[WAL_FILE_LEN]; - walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); - int64_t fsize; - TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); - if (pIdxFile == NULL) { - ASSERT(0); + if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) { + wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + ASSERT(pFileInfo->fileSize > 0 && pFileInfo->firstVer >= 0 && pFileInfo->lastVer >= pFileInfo->firstVer); + if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { + return 0; + } + + // start to repair + int64_t offset = fileSize - fileSize % sizeof(SWalIdxEntry); + TdFilePtr pLogFile = NULL; + TdFilePtr pIdxFile = NULL; + SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer - 1, .offset = -sizeof(SWalCkHead)}; + SWalCkHead ckHead; + memset(&ckHead, 0, sizeof(ckHead)); + ckHead.head.version = idxEntry.ver; + + pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); + if (pIdxFile == NULL) { + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); + if (pLogFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); + goto _err; + } + + // determine the last valid entry end, i.e. offset + while ((offset -= sizeof(SWalIdxEntry)) >= 0) { + if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) { + wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fnameStr, terrstr()); - return -1; + goto _err; } - taosFStatFile(pIdxFile, &fsize, NULL); - if (fsize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { - taosCloseFile(&pIdxFile); + if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (idxEntry.ver > pFileInfo->lastVer) { continue; } - int32_t left = fsize % sizeof(SWalIdxEntry); - int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END); - if (left != 0) { - taosFtruncateFile(pIdxFile, offset); - wWarn("vgId:%d wal truncate file %s to offset %ld since size invalid, file size %ld", pWal->cfg.vgId, fnameStr, - offset, fsize); - } - offset -= sizeof(SWalIdxEntry); - - SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer}; - while (1) { - if (offset < 0) { - taosLSeekFile(pIdxFile, 0, SEEK_SET); - taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - break; - } - taosLSeekFile(pIdxFile, offset, SEEK_SET); - int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if ((idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry) != offset) { - taosFtruncateFile(pIdxFile, offset); - wWarn("vgId:%d wal truncate file %s to offset %ld since entry invalid, entry ver %ld, entry offset %ld", - pWal->cfg.vgId, fnameStr, offset, idxEntry.ver, idxEntry.offset); - offset -= sizeof(SWalIdxEntry); - } else { - break; - } + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { + wWarn("vgId:%d, failed to read log file since %s. file:%s, offset:%" PRId64 ", idx entry ver:%" PRId64 "", + pWal->cfg.vgId, terrstr(), fLogNameStr, idxEntry.offset, idxEntry.ver); + continue; } - if (idxEntry.ver < pFileInfo->lastVer) { - char fLogNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); - TdFilePtr pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); - if (pLogFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); - return -1; - } - while (idxEntry.ver < pFileInfo->lastVer) { - if (taosLSeekFile(pLogFile, idxEntry.offset, SEEK_SET) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot seek file %s at %ld, since %s", pWal->cfg.vgId, fLogNameStr, idxEntry.offset, - terrstr()); - return -1; - } - SWalCkHead ckHead; - taosReadFile(pLogFile, &ckHead, sizeof(SWalCkHead)); - if (idxEntry.ver != ckHead.head.version) { - // todo truncate this idx also - taosCloseFile(&pLogFile); - wError("vgId:%d, invalid repair case, log seek to %ld to find ver %ld, actual ver %ld", pWal->cfg.vgId, - idxEntry.offset, idxEntry.ver, ckHead.head.version); - return -1; - } - idxEntry.ver = ckHead.head.version + 1; - idxEntry.offset = idxEntry.offset + sizeof(SWalCkHead) + ckHead.head.bodyLen; - wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); - taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - } - taosCloseFile(&pLogFile); + if (idxEntry.ver == ckHead.head.version) { + break; + } + } + offset += sizeof(SWalIdxEntry); + + // ftruncate idx file + if (offset < fileSize) { + if (taosFtruncateFile(pIdxFile, offset) < 0) { + wError("vgId:%d, failed to ftruncate file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, + strerror(errno), offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } + + // rebuild idx file + if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) { + wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + while (idxEntry.ver < pFileInfo->lastVer) { + ASSERT(idxEntry.ver == ckHead.head.version); + + idxEntry.ver += 1; + idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen; + + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { + wError("vgId:%d, failed to read wal log head since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), + idxEntry.offset, fLogNameStr); + goto _err; + } + wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); + if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) { + wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); + goto _err; + } + } + + if (taosFsyncFile(pIdxFile) < 0) { + wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); + goto _err; + } + + (void)taosCloseFile(&pLogFile); + (void)taosCloseFile(&pIdxFile); + return 0; + +_err: + (void)taosCloseFile(&pLogFile); + (void)taosCloseFile(&pIdxFile); + return -1; +} + +int walCheckAndRepairIdx(SWal* pWal) { + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + int32_t fileIdx = sz; + while (--fileIdx >= 0) { + if (walCheckAndRepairIdxFile(pWal, fileIdx) < 0) { + wError("vgId:%d, failed to repair idx file since %s. fileIdx:%d", pWal->cfg.vgId, terrstr(), fileIdx); + return -1; } - taosCloseFile(&pIdxFile); } return 0; } @@ -529,7 +668,7 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { taosArrayEnsureCap(pArray, sz); SWalFileInfo* pData = pArray->pData; for (int i = 0; i < sz; i++) { - cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); if (!pInfoJson) goto _err; SWalFileInfo* pInfo = &pData[i]; pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); @@ -593,7 +732,20 @@ int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; char tmpFnameStr[WAL_FILE_LEN]; - int n; + int n; + + // fsync the idx and log file at first to ensure validity of meta + if (taosFsyncFile(pWal->pIdxFile) < 0) { + wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (taosFsyncFile(pWal->pLogFile) < 0) { + wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } // flush to a tmpfile n = walBuildTmpMetaName(pWal, tmpFnameStr); @@ -601,7 +753,7 @@ int walSaveMeta(SWal* pWal) { TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pMetaFile == NULL) { - wError("failed to open file due to %s. file:%s", strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -609,20 +761,19 @@ int walSaveMeta(SWal* pWal) { char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); if (len != taosWriteFile(pMetaFile, serialized, len)) { - // TODO:clean file - wError("failed to write file due to %s. file:%s", strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } if (taosFsyncFile(pMetaFile) < 0) { - wError("failed to sync file due to %s. file:%s", strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } if (taosCloseFile(&pMetaFile) < 0) { - wError("failed to close file due to %s. file:%s", strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 96f9522c9f..5972807f50 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -110,8 +110,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // init ref pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (pWal->pRefHash == NULL) { - wError("failed to init hash since %s", tstrerror(terrno)); - goto _err; + wError("failed to init hash since %s", tstrerror(terrno)); + goto _err; } // open meta @@ -149,8 +149,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // add ref pWal->refId = taosAddRef(tsWal.refSetId, pWal); if (pWal->refId < 0) { - wError("failed to add ref for Wal since %s", tstrerror(terrno)); - goto _err; + wError("failed to add ref for Wal since %s", tstrerror(terrno)); + goto _err; } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1ac15e4bb8..cf96c0dae6 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -476,7 +476,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - wError("vgId:%d, failed to read WAL record head from log file since %s", pReader->pWal->cfg.vgId, terrstr()); + wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -509,7 +510,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - wError("vgId:%d, failed to read WAL record body from log file since %s", pReader->pWal->cfg.vgId, terrstr()); + wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 1196914dae..4b75db52b7 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -79,6 +79,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; if (pWal->pLogFile != NULL) { + code = taosFsyncFile(pWal->pLogFile); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } code = taosCloseFile(&pWal->pLogFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -86,6 +91,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { } } if (pWal->pIdxFile != NULL) { + code = taosFsyncFile(pWal->pIdxFile); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } code = taosCloseFile(&pWal->pIdxFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 628b432446..ffadcd04c4 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -540,6 +540,11 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { + wTrace("vgId:%d, fileId:%" PRId64 ".idx, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); + if (taosFsyncFile(pWal->pIdxFile) < 0) { + wError("vgId:%d, file:%" PRId64 ".idx, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), + strerror(errno)); + } wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (taosFsyncFile(pWal->pLogFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), diff --git a/source/util/src/terror.c b/source/util/src/terror.c index eb13a08be4..9a117c6eb4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -447,12 +447,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND, "TQ table schema not f TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no commited offset") // wal -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Wal unexpected generic error") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "WAL unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") // tfs TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") From b6c31ee67d84ef3bb343e1835d5081e010dc399b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 10 Oct 2022 20:17:03 +0800 Subject: [PATCH 016/135] fix: fsync the current wal log and idx files at first to ensure validity of meta in walSaveMeta --- source/libs/wal/src/walMgmt.c | 3 ++- source/libs/wal/src/walWrite.c | 16 +++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 5972807f50..29058cada1 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -134,6 +134,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeHead.head.protoVer = WAL_PROTO_VER; pWal->writeHead.magic = WAL_MAGIC; + // load meta (void)walLoadMeta(pWal); if (walCheckAndRepairMeta(pWal) < 0) { @@ -188,11 +189,11 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { void walClose(SWal *pWal) { taosThreadMutexLock(&pWal->mutex); + (void)walSaveMeta(pWal); taosCloseFile(&pWal->pLogFile); pWal->pLogFile = NULL; taosCloseFile(&pWal->pIdxFile); pWal->pIdxFile = NULL; - walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; taosHashCleanup(pWal->pRefHash); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index ffadcd04c4..e3045cb8e0 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -209,10 +209,12 @@ int32_t walRollback(SWal *pWal, int64_t ver) { taosCloseFile(&pIdxFile); taosCloseFile(&pLogFile); - taosFsyncFile(pWal->pLogFile); - taosFsyncFile(pWal->pIdxFile); - - walSaveMeta(pWal); + code = walSaveMeta(pWal); + if (code < 0) { + wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } // unlock taosThreadMutexUnlock(&pWal->mutex); @@ -384,7 +386,11 @@ int32_t walRollImpl(SWal *pWal) { pWal->lastRollSeq = walGetSeq(); - walSaveMeta(pWal); + code = walSaveMeta(pWal); + if (code < 0) { + wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); + goto END; + } END: return code; From c70d10334cabfb59dc53821e4b7cd1a42ac19513 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 11 Oct 2022 15:26:43 +0800 Subject: [PATCH 017/135] enh: print vgId in error msg within syncNodeCommit --- source/libs/sync/src/syncMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index bfd933e163..cc686ba2fe 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -3047,7 +3047,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, } else { code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); if (code != 0) { - sError("failed to get log entry since %s. index:%lld", tstrerror(terrno), i); + sError("vgId:%d, failed to get log entry since %s. index:%lld", ths->vgId, tstrerror(terrno), i); return -1; } ASSERT(pEntry != NULL); From a4cc3f76d4f8a8698b857e8e45aad8cf085716a1 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 11 Oct 2022 19:58:01 +0800 Subject: [PATCH 018/135] fix: ignore file not exist error of tdbOsRemove --- source/libs/tdb/src/db/tdbPager.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index ee4fc90228..593b8cc589 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -156,6 +156,7 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) { ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); return -1; } From 6f39a36344e5de61af1f673f96547f7c09a9d05b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 11 Oct 2022 20:01:12 +0800 Subject: [PATCH 019/135] fix: replace asserts on failure of tdbPagerWrite with tdbError msg --- source/libs/tdb/src/db/tdbBtree.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a148c1e36b..97e1b3f9cd 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -121,6 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -483,9 +484,8 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN ret = tdbPagerWrite(pPager, pChild); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to write page since %s", terrstr()); + return -1; } // Copy the root page content to the child page @@ -556,8 +556,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pOlds[i]); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } } @@ -583,8 +582,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pParent); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -719,8 +717,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pNews[iNew]); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } } @@ -937,7 +934,7 @@ static int tdbFetchOvflPage(SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt) // mark dirty ret = tdbPagerWrite(pBt->pPager, *ppOfp); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -1942,7 +1939,7 @@ int tdbBtcDelete(SBTC *pBtc) { // drop the cell on the leaf ret = tdbPagerWrite(pPager, pBtc->pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -1964,7 +1961,7 @@ int tdbBtcDelete(SBTC *pBtc) { if (idx < nCells) { ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -2029,7 +2026,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // mark dirty ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } From 12d870ef3b69d8191b11481a1f732c59ed9a332f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 12 Oct 2022 20:48:23 +0800 Subject: [PATCH 020/135] fix: response TSDB_CODE_VND_NOT_SYNCED while not pVnode->inUse in vnodeProcessWriteMsg --- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b8047abe69..5f96fd8807 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -145,6 +145,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t len; int32_t ret; + if (!pVnode->inUse) { + terrno = TSDB_CODE_VND_NOT_SYNCED; + vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); + return -1; + } + vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); From 8f6a81a18cb3d421b07d0326b4d8c0fda4b5f478 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 12 Oct 2022 23:07:14 +0800 Subject: [PATCH 021/135] enh: indicate intentionally restarting taosd on tsdbCommit failure --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 995d7a3155..3db0b4f2b9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1038,7 +1038,9 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; - ASSERT(eno == 0); + ASSERT(eno == 0 && + "tsdbCommit failure" + "Restart taosd"); code = tsdbFSCommit1(pTsdb, &pCommitter->fs); TSDB_CHECK_CODE(code, lino, _exit); From 34ea8996a0dbe4f5e9f2d596dfe0da0b83e9eb33 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 12 Oct 2022 23:08:41 +0800 Subject: [PATCH 022/135] fix: check disk availability for committing vnode --- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index bfb5cad44c..07c4c32955 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -73,7 +73,7 @@ int vnodeBegin(SVnode *pVnode) { int vnodeShouldCommit(SVnode *pVnode) { if (pVnode->inUse) { - return pVnode->inUse->size > pVnode->inUse->node.size; + return osDataSpaceAvailable() && (pVnode->inUse->size > pVnode->inUse->node.size); } return false; } From 90ced77bad5fe9dd2068209ef0395a7dda61161a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 16 Oct 2022 13:07:58 +0800 Subject: [PATCH 023/135] fix: taosbenchmark coverity scan issues (#17378) * fix: taosbenchmark coverity scan issues * fix: taosbenchmark default json case * fix: update taos-tools 217a267 * fix: taosbenchmark coverity scan issues 2849aa4 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 7e441c162a..f0a1ad83ad 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 217a267 + GIT_TAG 2849aa4 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From e3f50c0d481eab72fae604c4f0d2d00af29acda2 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 16 Oct 2022 11:49:53 +0800 Subject: [PATCH 024/135] fix: align vers.firstVer with vers.snapshotVer+1 in WAL meta info --- source/libs/wal/src/walMeta.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 8808363bb2..279f4dc656 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -257,23 +257,23 @@ static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) { } void walAlignVersions(SWal* pWal) { - if (pWal->vers.firstVer > pWal->vers.snapshotVer) { - wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) { + wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId, pWal->vers.firstVer, pWal->vers.snapshotVer); - pWal->vers.firstVer = pWal->vers.snapshotVer; + pWal->vers.firstVer = pWal->vers.snapshotVer + 1; } if (pWal->vers.lastVer < pWal->vers.snapshotVer) { - wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, pWal->vers.lastVer, pWal->vers.snapshotVer); pWal->vers.lastVer = pWal->vers.snapshotVer; } if (pWal->vers.commitVer < pWal->vers.snapshotVer) { - wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, pWal->vers.commitVer, pWal->vers.snapshotVer); pWal->vers.commitVer = pWal->vers.snapshotVer; } if (pWal->vers.appliedVer < pWal->vers.snapshotVer) { - wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, pWal->vers.appliedVer, pWal->vers.snapshotVer); pWal->vers.appliedVer = pWal->vers.snapshotVer; } @@ -384,7 +384,6 @@ int walCheckAndRepairMeta(SWal* pWal) { actualFileNum = taosArrayGetSize(pWal->fileInfoSet); pWal->writeCur = actualFileNum - 1; pWal->totSize = totSize; - pWal->vers.firstVer = -1; pWal->vers.lastVer = -1; if (actualFileNum > 0) { pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; From bcb3d41d08440e3e0c988767563914d991751068 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 16 Oct 2022 13:46:21 +0800 Subject: [PATCH 025/135] fix(query): fix coverity issues. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 9 +++++- source/dnode/vnode/src/tsdb/tsdbRead.c | 11 ++++---- source/libs/executor/src/dataInserter.c | 2 +- source/libs/executor/src/executil.c | 17 +++++++---- source/libs/executor/src/executor.c | 6 ++-- source/libs/executor/src/executorimpl.c | 1 + source/libs/executor/src/scanoperator.c | 16 +++++++++-- source/libs/executor/src/sortoperator.c | 2 +- source/libs/executor/src/tfill.c | 9 ++++-- source/libs/executor/src/timewindowoperator.c | 28 +++++++++++-------- tools/shell/src/shellEngine.c | 2 +- 11 files changed, 69 insertions(+), 34 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 09a46b59bb..a4c5c60e78 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -133,8 +133,15 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { id.uid = pIter->uid; } - tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); + code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } double el = (taosGetTimestampUs() - st) / 1000.0; pInfo->elapsedTime += el; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6b706901ba..c346eeef1d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2676,9 +2676,9 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret int8_t level = 0; int8_t precision = pVnode->config.tsdbCfg.precision; int64_t now = taosGetTimestamp(precision); - int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1 - : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000 - : 1000000); + int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1L + : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L + : 1000000L); for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + level; @@ -3372,14 +3372,13 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl if (pCond->suid != 0) { pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1, 1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:%" PRId64 " , %s", pReader->suid, -1, - pReader->idStr); + tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr); } } else if (taosArrayGetSize(pTableList) > 0) { STableKeyInfo* pKey = taosArrayGet(pTableList, 0); pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:%" PRId64 " , %s", pKey->uid, -1, pReader->idStr); + tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr); } } diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 6a43ea40c6..ce0ff485ab 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -81,7 +81,7 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { } pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows; - qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows, + qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows); tFreeSSubmitRsp(pInserter->submitRes.pRsp); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 0fadacb2c2..8277a4ed88 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1201,14 +1201,21 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pFuncNode->node.aliasName); - pExp->pExpr->_function.functionId = pFuncNode->funcId; - pExp->pExpr->_function.pFunctNode = pFuncNode; + tExprNode* pExprNode = pExp->pExpr; + + pExprNode->_function.functionId = pFuncNode->funcId; + pExprNode->_function.pFunctNode = pFuncNode; + + tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName)); - strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, - tListLen(pExp->pExpr->_function.functionName)); #if 1 // todo refactor: add the parameter for tbname function - if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { + const char* name = "tbname"; + int32_t len = strlen(name); + + if (!pFuncNode->pParameterList && (memcmp(pExprNode->_function.functionName, name, len) == 0) && + pExprNode->_function.functionName[len] == 0) { + pFuncNode->pParameterList = nodesMakeList(); ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e527f7e7c1..413813f58a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -856,15 +856,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); pTaskInfo->streamInfo.schema = mtInfo.schema; - qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %ld", mtInfo.uid, pOffset->ts); + qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %"PRId64, mtInfo.uid, pOffset->ts); } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; if (setForSnapShot(sContext, pOffset->uid) != 0) { - qError("setForSnapShot error. uid:%" PRIi64 " ,version:%" PRIi64, pOffset->uid); + qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); return -1; } - qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %ld", pOffset->uid); + qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %"PRId64, pOffset->uid, pOffset->ts); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; tsdbReaderClose(pInfo->dataReader); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ea01b961b1..e59755787a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1857,6 +1857,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); if (NULL == pMsg) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); return pTaskInfo->code; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c0c38e2b30..66fad27d71 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2568,16 +2568,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { SMetaReader smrChildTable = {0}; metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); - metaGetTableEntryByName(&smrChildTable, condTableName); + int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + return NULL; + } + if (smrChildTable.me.type != TSDB_CHILD_TABLE) { metaReaderClear(&smrChildTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } + SMetaReader smrSuperTable = {0}; metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); - metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByUid + return NULL; + } + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); metaReaderClear(&smrSuperTable); metaReaderClear(&smrChildTable); @@ -3795,6 +3806,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pColList); goto _error; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index c651529c7a..0d5f521034 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -658,7 +658,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataDestroy(p); - qDebug("%s get sorted block, groupId:%0x" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, + qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 88af03775c..8b716531e5 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1681,7 +1681,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); pInfo->pCondition = pPhyFillNode->node.pConditions; pInfo->pColMatchColInfo = pColMatchColInfo; - initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pInfo->srcRowIndex = 0; pOperator->name = "FillOperator"; @@ -1693,7 +1697,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo, NULL, NULL, NULL); - int32_t code = appendDownstream(pOperator, &downstream, 1); + code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1702,5 +1706,6 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi _error: destroyStreamFillOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; return NULL; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c1478bc935..9c219d2765 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1420,8 +1420,9 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* pGpDatas = (uint64_t*)pGpCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; + STimeWindow win = {0}; if (IS_FINAL_OP(pInfo)) { win.skey = startTsCols[i]; @@ -5828,27 +5829,30 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->isFinal = false; - if (pIntervalPhyNode->window.pExprs != NULL) { - int32_t numOfScalar = 0; - SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); - int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } + pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); SExprSupp* pSup = &pOperator->exprSupp; + + initBasicInfo(&pInfo->binfo, pResBlock); + initStreamFunciton(pSup->pCtx, pSup->numOfExprs); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initBasicInfo(&pInfo->binfo, pResBlock); - initStreamFunciton(pSup->pCtx, pSup->numOfExprs); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + if (pIntervalPhyNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } pInfo->invertible = allInvertible(pSup->pCtx, numOfCols); pInfo->invertible = false; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 28375d8a35..37d00c0a7c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -229,7 +229,7 @@ void shellRunSingleCommandImp(char *command) { printf("Query OK, %d of %d rows affected (%.6fs)\r\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); // call auto tab - callbackAutoTab(command, pSql, false); + callbackAutoTab(command, NULL, false); } printf("\r\n"); From a86249ef6b4bf00df5c57d9f651cfd32ae07f69b Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Sun, 16 Oct 2022 18:16:29 +0800 Subject: [PATCH 026/135] test: add coverage test script --- tests/script/coverage_test.sh | 291 ++++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100755 tests/script/coverage_test.sh diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh new file mode 100755 index 0000000000..457c9eae20 --- /dev/null +++ b/tests/script/coverage_test.sh @@ -0,0 +1,291 @@ +#!/bin/bash + +branch= +if [ x$1 != x ];then + branch=$1 + echo "Testing branch: $branch" +else + echo "Please enter branch name as a parameter" + exit 1 +fi + +today=`date +"%Y%m%d"` +TDENGINE_DIR=/root/pxiao/TDengine +TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function buildTDengine { + echo "check if TDengine need build" + cd $TDENGINE_DIR + git remote prune origin > /dev/null + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` + LOCAL_COMMIT=`git rev-parse --short @` + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + + # reset counter + lcov -d . --zerocounters + + + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + fi + + git reset --hard + git checkout -- . + git checkout $branch + git clean -dfx + git pull + git submodule update --init --recursive -f + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug + + echo "rebuild.." + LOCAL_COMMIT=`git rev-parse --short @` + + rm -rf * + if [ "$branch" == "3.0" ]; then + echo "3.0 =============" + cmake -DCOVER=true -DBUILD_TEST=true .. + else + cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null + fi + make -j4 + make install +} + +function runGeneralCaseOneByOne { + while read -r line; do + if [[ $line =~ ^./test.sh* ]]; then + case=`echo $line | grep sim$ | awk '{print $NF}'` + + if [ -n "$case" ]; then + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + fi + done < $1 +} + +function runTestNGCaseOneByOne { + while read -r line; do + if [[ $line =~ ^taostest* ]]; then + case=`echo $line | cut -d' ' -f 3 | cut -d'=' -f 2` + yaml=`echo $line | cut -d' ' -f 2` + + if [ -n "$case" ]; then + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && taostest $yaml --case=$case --keep --disable_collection > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + fi + done < $1 +} + +function runTest { + echo "run Test" + + if [ "$branch" == "3.0" ]; then + echo "start run unit test case ................" + echo " $TDENGINE_DIR/debug " + cd $TDENGINE_DIR/debug + ctest -j12 + echo "3.0 unit test done" + fi + + cd $TDENGINE_DIR/tests/script + + [ -d ../../sim ] && rm -rf ../../sim + [ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT + + runGeneralCaseOneByOne jenkins/basic.txt + + sed -i "1i\SIM cases test result" $TDENGINE_COVERAGE_REPORT + + totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + sed -i -e "2i\ ### Total $totalSuccess SIM test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT + fi + + totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + sed -i "3i\### Total $totalFailed SIM test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT + fi + sed "3G" $TDENGINE_COVERAGE_REPORT + + stopTaosd + echo "run TestNG cases" + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 & + sleep 10 + cd $TDENGINE_DIR/../TestNG/cases + runTestNGCaseOneByOne ../scripts/cases.txt + echo "TestNG cases done" + + cd $TDENGINE_DIR/tests + rm -rf ../sim + /root/pxiao/test-all-coverage.sh full python $branch | tee -a $TDENGINE_COVERAGE_REPORT + + + sed -i "4i\Python cases test result" $TDENGINE_COVERAGE_REPORT + totalPySuccess=`grep 'python case(s) succeed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'` + if [ "$totalPySuccess" -gt "0" ]; then + sed -i -e "5i\ ### Total $totalPySuccess Python test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT + fi + + totalPyFailed=`grep 'python case(s) failed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'` + if [ -z $totalPyFailed ]; then + sed -i "6i\\n" $TDENGINE_COVERAGE_REPORT + else + sed -i "6i\### Total $totalPyFailed Python test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT + fi + + echo "### run JDBC test cases ###" | tee -a $TDENGINE_COVERAGE_REPORT + # Test Connector + stopTaosd + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 & + sleep 10 + + cd $TDENGINE_DIR/src/connector/jdbc + mvn clean package > /dev/null 2>&1 + mvn test > jdbc-out.log 2>&1 + tail -n 20 jdbc-out.log 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + + # Test C Demo + stopTaosd + $TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null & + sleep 10 + yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + + # Test waltest + dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'` + walDir=`find $dataDir -name "wal"|head -n1` + echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT + echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT + if [ -n "$walDir" ]; then + yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + fi + + # run Unit Test + echo "Run Unit Test: utilTest, queryTest and cliTest" + #$TDENGINE_DIR/debug/build/bin/utilTest > /dev/null 2>&1 && echo "utilTest pass!" || echo "utilTest failed!" + #$TDENGINE_DIR/debug/build/bin/queryTest > /dev/null 2>&1 && echo "queryTest pass!" || echo "queryTest failed!" + #$TDENGINE_DIR/debug/build/bin/cliTest > /dev/null 2>&1 && echo "cliTest pass!" || echo "cliTest failed!" + + stopTaosd + + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + +function lcovFunc { + echo "collect data by lcov" + cd $TDENGINE_DIR + + # collect data + lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + + # remove exclude paths + if [ "$branch" == "3.0" ]; then + lcov --remove coverage.info \ + '*/contrib/*' '*/tests/*' '*/test/*'\ + '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c'\ + '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ + '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/tidpool.c' '*/tmempool.c'\ + '*/tthread.c' '*/tversion.c'\ + --rc lcov_branch_coverage=1 -o coverage.info + else + lcov --remove coverage.info \ + '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' '*/ttype.h' '*/tarithoperator.c' '*/TSDBJNIConnector.c' '*/taosdemo.c'\ + --rc lcov_branch_coverage=1 -o coverage.info + fi + + + # generate result + echo "generate result" + lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + + # push result to coveralls.io + echo "push result to coveralls.io" + /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT + + #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT + #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT + cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT + +} + +function sendReport { + echo "send report" + receiver="develop@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDENGINE_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT + BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT` + echo -e "from: \nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \ + /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!" +} + +function stopTaosd { + echo "Stop taosd start" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + echo "Stop tasod end" +} + +function runTestRandomFail { + exec_random_fail_sh=$1 + default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh + [ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1 + + dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py + default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py + [ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1 + + runTest NoRandomFail +} + +WORK_DIR=/root/pxiao + +date >> $WORK_DIR/cron.log +echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log + +stopTaosd +buildTDengine + +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh + +runTest + +lcovFunc +#sendReport +stopTaosd + +date >> $WORK_DIR/cron.log +echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log From ef06a9d265c295e7b9c82be9713ef5fb8df92ae9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 17 Oct 2022 09:58:34 +0800 Subject: [PATCH 027/135] fix more coverity scan problem --- source/dnode/vnode/src/meta/metaTable.c | 7 ++++++- source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 19 +++++++++++-------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 97b0be07c1..1e0e757793 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -51,7 +51,7 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema return -1; } - strcpy(pMetaRsp->tbName, tbName); + strncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN); pMetaRsp->numOfColumns = pSchema->nCols; pMetaRsp->tableType = TSDB_NORMAL_TABLE; pMetaRsp->sversion = pSchema->version; @@ -817,6 +817,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA const void *pData = NULL; int nData = 0; + if (pAlterTbReq->tagName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + // search name index ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index d08b743763..6fd5629592 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -819,7 +819,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(fSet.pHeadF); } } else { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0f93e650c6..3556b47dd3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -41,7 +41,10 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t nReqs; tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); - tStartDecode(&dc); + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } if (tDecodeI32v(&dc, &nReqs) < 0) { code = TSDB_CODE_INVALID_MSG; @@ -167,9 +170,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t ret; if (!pVnode->inUse) { - terrno = TSDB_CODE_VND_NOT_SYNCED; - vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); - return -1; + terrno = TSDB_CODE_VND_NOT_SYNCED; + vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); + return -1; } vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), @@ -293,14 +296,14 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change if (vnodeCommit(pVnode) < 0) { - vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); - goto _err; + vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; } // start a new one if (vnodeBegin(pVnode) < 0) { - vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); - goto _err; + vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; } } From 6f9153c6e3b1de35c1f7f7511e966527fdf736ba Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 10:13:10 +0800 Subject: [PATCH 028/135] fix: error handling --- source/client/src/clientTmq.c | 4 ++-- source/dnode/mnode/impl/src/mndScheduler.c | 2 +- source/libs/wal/src/walWrite.c | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 922cab23c8..73f7c2bcc4 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -966,14 +966,14 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = -1; + tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz); + req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, 256); tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) goto FAIL; - tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz); - for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 511a84290d..1c8ec0a302 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -416,7 +416,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } SStreamTask* pTask = tNewSStreamTask(pStream->uid); - if (pInnerTask == NULL) { + if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index e7161079d9..bb6c3a91ef 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -45,6 +45,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { if (taosRemoveFile(fnameStr) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); return -1; } wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); @@ -53,6 +54,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { if (taosRemoveFile(fnameStr) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); return -1; } wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); From 980fdd6e915d48b15093faf8be4d7ec1a529562e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 10:27:15 +0800 Subject: [PATCH 029/135] fix: error handling --- source/dnode/vnode/src/tq/tq.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e1578b37f1..aaa57d49ed 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1010,16 +1010,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) { qError("stream task input del failed, task id %d", pTask->taskId); + + taosFreeQitem(pRefBlock); continue; } + if (streamSchedExec(pTask) < 0) { qError("stream task launch failed, task id %d", pTask->taskId); continue; } + } else { streamTaskInputFail(pTask); } } + int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { From 837495fa5af3e49ca92a91d39dd0a1386b834f1b Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 17 Oct 2022 10:45:47 +0800 Subject: [PATCH 030/135] test: reopen taosShell.py test case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index cc85b8ddc8..58d8efbed6 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -2,7 +2,7 @@ set -e set -x -#python3 ./test.py -f 0-others/taosShell.py +python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py python3 ./test.py -f 0-others/taosShellNetChk.py python3 ./test.py -f 0-others/telemetry.py From 132681a8bca37652e93c8abd5bad56bb9c0702bf Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 031/135] fix: coverity issues CID: 308488 --- source/common/src/ttime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index d9d4fb274e..bb82863e73 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -795,7 +795,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio // not enough time range if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - while (end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window + while (end < t) { // move forward to the correct time window start += pInterval->sliding; if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { From bcf2cfeed7284e04a8e224431bb718f522e55eb3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 032/135] fix: coverity issues CID: 354713 --- source/libs/scalar/src/filter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 81ee173648..5865370e8e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1709,9 +1709,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) ctx->isrange); if (ctx->isrange) { SFilterRangeNode *r = ctx->rs; + int32_t tlen = 0; while (r) { - char str[256] = {0}; - int32_t tlen = 0; + char str[256] = {0}; if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { strcat(str, "(NULL)"); } else { From 4d0c7732f13a5fd6da82ec0576105678970b868d Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Mon, 17 Oct 2022 11:20:10 +0800 Subject: [PATCH 033/135] test:add test case for create wrong topic --- tests/system-test/7-tmq/create_wrong_topic.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/system-test/7-tmq/create_wrong_topic.py diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py new file mode 100644 index 0000000000..d18cb0260e --- /dev/null +++ b/tests/system-test/7-tmq/create_wrong_topic.py @@ -0,0 +1,77 @@ +import taos +import sys +import time +import socket +import os +import threading +from util.common import * + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.setsql = TDSetSql() + self.rowNum = 10 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + } + self.error_topic = ['avg','count','spread','stddev','sum','hyperloglog'] + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) + def wrong_topic(self): + tdSql.prepare() + tdSql.execute('use db') + stbname = f'db.{tdCom.getLongName(5, "letters")}' + tag_dict = { + 't0':'int' + } + tag_values = [ + f'1' + ] + tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) + tdSql.execute(f"create table {stbname}_tb1 using {stbname} tags({tag_values[0]})") + self.insert_data(self.column_dict,f'{stbname}_tb1',self.rowNum) + for column in self.column_dict.keys(): + for func in self.error_topic: + if func.lower() != 'count' and column.lower() != 'ts': + tdSql.error(f'create topic tpn as select {func}({column}) from {stbname}') + elif func.lower() == 'count' : + tdSql.error(f'create topic tpn as select {func}(*) from {stbname}') + for column in self.column_dict.keys(): + if column.lower() != 'ts': + tdSql.error(f'create topic tpn as select apercentile({column},50) from {stbname}') + tdSql.error(f'create topic tpn as select leastquares({column},1,1) from {stbname}_tb1') + tdSql.error(f'create topic tpn as select HISTOGRAM({column},user_input,[1,3,5,7],0) from {stbname}') + tdSql.error(f'create topic tpn as select percentile({column},1) from {stbname}_tb1') + pass + def run(self): + self.wrong_topic() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 704dba9800668d663f540943b44fd0c33f0da70b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Oct 2022 11:23:53 +0800 Subject: [PATCH 034/135] docs: update examples/python/tmq_example.py with connector repo (#17387) * docs: update examples/python/tmq_example.py with connector repo * Update tmq_example.py * docs: remove tb variable after get_table_name removed Co-authored-by: Liu Jicong --- docs/examples/python/tmq_example.py | 62 ++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index cee036454e..836beb2417 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -1,6 +1,58 @@ import taos -from taos.tmq import TaosConsumer -consumer = TaosConsumer('topic_ctb_column', group_id='vg2') -for msg in consumer: - for row in msg: - print(row) +from taos.tmq import * + +conn = taos.connect() + +print("init") +conn.execute("drop database if exists py_tmq") +conn.execute("create database if not exists py_tmq vgroups 2") +conn.select_db("py_tmq") +conn.execute( + "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)" +) +conn.execute("create table if not exists tb1 using stb1 tags(1)") +conn.execute("create table if not exists tb2 using stb1 tags(2)") +conn.execute("create table if not exists tb3 using stb1 tags(3)") + +print("create topic") +conn.execute("drop topic if exists topic_ctb_column") +conn.execute( + "create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1" +) + +print("build consumer") +conf = TaosTmqConf() +conf.set("group.id", "tg2") +conf.set("td.connect.user", "root") +conf.set("td.connect.pass", "taosdata") +conf.set("enable.auto.commit", "true") + + +def tmq_commit_cb_print(tmq, resp, offset, param=None): + print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") + + +conf.set_auto_commit_cb(tmq_commit_cb_print, None) +tmq = conf.new_consumer() + +print("build topic list") + +topic_list = TaosTmqList() +topic_list.append("topic_ctb_column") + +print("basic consume loop") +tmq.subscribe(topic_list) + +sub_list = tmq.subscription() + +print("subscribed topics: ", sub_list) + +while 1: + res = tmq.poll(1000) + if res: + topic = res.get_topic_name() + vg = res.get_vgroup_id() + db = res.get_db_name() + print(f"topic: {topic}\nvgroup id: {vg}\ndb: {db}") + for row in res: + print(row) From b850c61720e81f6ad3df4aec6f6d5eae1c212b77 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Mon, 17 Oct 2022 11:26:00 +0800 Subject: [PATCH 035/135] test :add test case into ci --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 2686156f0f..fe2337662c 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -279,7 +279,7 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1 - +python3 ./test.py -f 7-tmq/create_wrong_topic.py python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py From 84817bf7438ddcfc959494328c8e29c9c8f4051a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 036/135] fix: coverity issues CID: 354715 --- source/common/src/tvariant.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index a01c393441..830a6199d8 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -366,6 +366,7 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { } if (pBuf != NULL) { + taosMemoryFree(pVariant->pz); *pDest = pBuf; } From 87516ea33243d03a9eb3d6a6b9cf725cccccada2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Oct 2022 13:00:31 +0800 Subject: [PATCH 037/135] fix(query): fix bug in group by tag. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 7 +++- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/scanoperator.c | 32 +++++++++---------- source/libs/executor/src/timewindowoperator.c | 4 +-- source/libs/planner/src/planOptimizer.c | 5 ++- 5 files changed, 26 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c346eeef1d..bc09b4810c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1515,6 +1515,11 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* taosMemoryFree(pReader->pMemSchema); int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + return pReader->pMemSchema; } @@ -3732,7 +3737,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { } int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { - if (isEmptyQueryTimeWindow(&pReader->window)) { + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 413813f58a..01915a7ffb 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -616,7 +616,7 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { int32_t nOptrWithVal = 0; int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); - if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { + if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) { taosMemoryFreeClear(*pOutput); *len = 0; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 66fad27d71..b878112545 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -284,6 +284,19 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, return true; } + +static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + if (pTableScanInfo->pseudoSup.numOfExprs > 0) { + SExprSupp* pSup = &pTableScanInfo->pseudoSup; + + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, + GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + } +} + static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -313,7 +326,6 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; @@ -322,6 +334,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -371,17 +384,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true); - - // currently only the tbname pseudo column - if (pTableScanInfo->pseudoSup.numOfExprs > 0) { - SExprSupp* pSup = &pTableScanInfo->pseudoSup; - - int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, - GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } - } + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); if (pTableScanInfo->pFilterNode != NULL) { int64_t st = taosGetTimestampUs(); @@ -1316,7 +1319,7 @@ static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* blockDataEnsureCapacity(pResBlock, 1); - projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, pTagCalSup->numOfExprs, NULL); + projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, 1, NULL); ASSERT(pResBlock->info.rows == 1); // build tagArray @@ -2120,9 +2123,6 @@ static void destroyStreamScanOperatorInfo(void* param) { taosMemoryFree(pStreamScan->pPseudoExpr); } - cleanupExprSupp(&pStreamScan->tbnameCalSup); - cleanupExprSupp(&pStreamScan->tagCalSup); - updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pUpdateRes); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9c219d2765..cbbd874bb9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2842,14 +2842,13 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc); + initBasicInfo(&pInfo->binfo, pResBlock); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initBasicInfo(&pInfo->binfo, pResBlock); - pInfo->twAggSup.waterMark = pSessionNode->window.watermark; pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType; pInfo->gap = pSessionNode->gap; @@ -4666,7 +4665,6 @@ void destroyStreamStateOperatorInfo(void* param) { SStreamSessionAggOperatorInfo* pChInfo = pChild->info; destroyStreamSessionAggOperatorInfo(pChInfo); taosMemoryFreeClear(pChild); - taosMemoryFreeClear(pChInfo); } } colDataDestroy(&pInfo->twAggSup.timeWindowData); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index cd02e7a136..7fd90dd4e8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -124,9 +124,8 @@ static void optSetParentOrder(SLogicNode* pNode, EOrder order) { EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { - // *((bool*)pContext) = - // (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); - *((bool*)pContext) = true; + *((bool*)pContext) = + (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; From 2df86022c0533dbc03956c728e2c6e2a20107f0c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 17 Oct 2022 10:05:45 +0800 Subject: [PATCH 038/135] fix(stream): coverity scan,type conversion --- source/util/src/tbloomfilter.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 7e1506c140..f3ccbb0aac 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -19,12 +19,12 @@ #include "taos.h" #include "taoserror.h" -#define UNIT_NUM_BITS 64 -#define UNIT_ADDR_NUM_BITS 6 +#define UNIT_NUM_BITS 64ULL +#define UNIT_ADDR_NUM_BITS 6ULL static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1 << (index % UNIT_NUM_BITS); + uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); uint64_t old = buf[unitIndex]; buf[unitIndex] |= mask; return buf[unitIndex] != old; @@ -32,7 +32,7 @@ static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1 << (index % UNIT_NUM_BITS); + uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); return buf[unitIndex] & mask; } From ed9ec795eba10c3745b3cb25ee9768a1a10e4c04 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Oct 2022 14:05:40 +0800 Subject: [PATCH 039/135] fix(query): fix coverity issues. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 15 ++++----- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 ++--- source/libs/executor/src/executil.c | 4 +-- source/libs/executor/src/executor.c | 14 ++++----- source/libs/executor/src/executorimpl.c | 35 +++------------------ source/libs/executor/src/scanoperator.c | 12 +++++-- source/libs/executor/src/tsimplehash.c | 4 +-- tools/shell/src/shellTire.c | 7 ++--- 8 files changed, 37 insertions(+), 62 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index a4c5c60e78..2f4fdfc5f8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -151,9 +151,6 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { ", last file index:%d, last block index:%d, entry:%d, %p, elapsed time:%.2f ms, %s", pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock, el, idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk; tsdbDebug("last block index list:%d, %d, %s", pInfo->blockIndex[0], pInfo->blockIndex[1], idStr); @@ -466,8 +463,8 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { } bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { - int32_t code = 0; int32_t step = pIter->backward ? -1 : 1; + terrno = TSDB_CODE_SUCCESS; // no qualified last file block in current file, no need to fetch row if (pIter->pSttBlk == NULL) { @@ -476,6 +473,10 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { int32_t iBlockL = pIter->iSttBlk; SBlockData *pBlockData = loadLastBlock(pIter, idStr); + if (pBlockData == NULL && terrno != TSDB_CODE_SUCCESS) { + goto _exit; + } + pIter->iRow += step; while (1) { @@ -501,11 +502,7 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow); _exit: - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - } - - return (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); + return (terrno == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); } SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bc09b4810c..670691ab1f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -340,7 +340,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); if (pIter->pLastBlockReader == NULL) { int32_t code = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr); + tsdbError("failed to prepare the last block iterator, code:%s %s", tstrerror(code), pReader->idStr); return code; } } @@ -646,7 +646,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug( - "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " + "load block of %"PRIzu" tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " "time:%.2f ms %s", numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, pReader->idStr); @@ -3090,7 +3090,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, piRow); - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); } int32_t code = tRowMergerGetRow(&merge, pTSRow); @@ -3448,7 +3448,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl return code; _err: - tsdbError("failed to create data reader, code:%s %s", tstrerror(code), pReader->idStr); + tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); return code; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 8277a4ed88..f99c832381 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1138,7 +1138,7 @@ static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, i s.bytes = bytes; s.slotId = slotId; s.precision = precision; - strncpy(s.name, name, tListLen(s.name)); + tstrncpy(s.name, name, tListLen(s.name)); return s; } @@ -1366,7 +1366,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); } else { char* udfName = pExpr->pExpr->_function.pFunctNode->functionName; - strncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN); + tstrncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN); fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet); } pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 01915a7ffb..41f861cd38 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -701,10 +701,10 @@ int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { return 0; } -int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo mtInfo) { +int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) { memset(pCond, 0, sizeof(SQueryTableDataCond)); pCond->order = TSDB_ORDER_ASC; - pCond->numOfCols = mtInfo.schema->nCols; + pCond->numOfCols = pMtInfo->schema->nCols; pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); if (pCond->colList == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -712,15 +712,15 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s } pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; - pCond->suid = mtInfo.suid; + pCond->suid = pMtInfo->suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = sContext->snapVersion; for (int32_t i = 0; i < pCond->numOfCols; ++i) { - pCond->colList[i].type = mtInfo.schema->pSchema[i].type; - pCond->colList[i].bytes = mtInfo.schema->pSchema[i].bytes; - pCond->colList[i].colId = mtInfo.schema->pSchema[i].colId; + pCond->colList[i].type = pMtInfo->schema->pSchema[i].type; + pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes; + pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId; } return TSDB_CODE_SUCCESS; @@ -844,7 +844,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT taosArrayDestroy(pTaskInfo->tableqinfoList.pTableList); if (mtInfo.uid == 0) return 0; // no data - initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, mtInfo); + initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0}); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e59755787a..3ec8085b86 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -895,33 +895,6 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI } #endif -static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { -#if 0 - SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; - uint32_t status = BLK_DATA_NOT_LOAD; - - int32_t numOfOutput = 0; // pTableScanInfo->numOfOutput; - for (int32_t i = 0; i < numOfOutput; ++i) { - int32_t functionId = pCtx[i].functionId; - int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId; - - // group by + first/last should not apply the first/last block filter - if (functionId < 0) { - status |= BLK_DATA_DATA_LOAD; - return status; - } else { - // status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId); - // if ((status & BLK_DATA_DATA_LOAD) == BLK_DATA_DATA_LOAD) { - // return status; - // } - } - } - - return status; -#endif - return 0; -} - int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NOT_LOAD; @@ -1802,7 +1775,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); + qDebug("%s fetch rsp received, index:%d, code:%s", pSourceDataInfo->taskId, index, tstrerror(code)); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -3397,8 +3370,8 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; pSchema->colId = pColNode->colId; pSchema->type = pColNode->node.resType.type; - pSchema->type = pColNode->node.resType.bytes; - strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); + pSchema->bytes = pColNode->node.resType.bytes; + tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } // this the tags and pseudo function columns, we only keep the tag columns @@ -3412,7 +3385,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; pSchema->colId = pColNode->colId; pSchema->type = pColNode->node.resType.type; - pSchema->type = pColNode->node.resType.bytes; + pSchema->bytes = pColNode->node.resType.bytes; strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b878112545..15a7abf1ff 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -325,6 +325,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); pCost->skipBlocks += 1; return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { @@ -1082,12 +1084,13 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) { - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); STimeWindow endWin = win; STimeWindow preWin = win; uint64_t groupId = gpIdCol[*pRowIndex]; + while (1) { if (hasGroup) { (*pRowIndex) += 1; @@ -1151,6 +1154,9 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 pResult->info.rows++; } } + + blockDataDestroy(tmpBlock); + if (pResult->info.rows > 0) { pResult->info.calWin = pInfo->updateWin; return pResult; @@ -1546,7 +1552,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { tsdbReaderClose(pTSInfo->dataReader); pTSInfo->dataReader = NULL; tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer); - qDebug("queue scan tsdb over, switch to wal ver %d", pTaskInfo->streamInfo.snapshotVer + 1); + qDebug("queue scan tsdb over, switch to wal ver %"PRId64, pTaskInfo->streamInfo.snapshotVer + 1); if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) { return NULL; } @@ -2995,7 +3001,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { while (1) { int64_t startTs = taosGetTimestampUs(); - strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); strcpy(pInfo->req.user, pInfo->pUser); int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 16fd11f97d..134862d88c 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -18,7 +18,7 @@ #include "tlog.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 -#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_MAX_CAPACITY (1024 * 1024 * 16L) #define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) #define GET_SHASH_NODE_KEY(_n, _dl) ((char *)(_n) + sizeof(SHNode) + (_dl)) @@ -104,7 +104,7 @@ static void tSimpleHashTableResize(SSHashObj *pHashObj) { int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); if (newCapacity > HASH_MAX_CAPACITY) { - uDebug("current capacity:%zu, maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached", + uDebug("current capacity:%"PRIzu", maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached", pHashObj->capacity, HASH_MAX_CAPACITY); return; } diff --git a/tools/shell/src/shellTire.c b/tools/shell/src/shellTire.c index 2f1ee12d54..b310281119 100644 --- a/tools/shell/src/shellTire.c +++ b/tools/shell/src/shellTire.c @@ -243,8 +243,8 @@ void enumAllWords(STireNode** nodes, char* prefix, SMatch* match) { continue; } else { // combine word string - memset(word, 0, sizeof(word)); - strcpy(word, prefix); + memset(word, 0, tListLen(word)); + strncpy(word, prefix, len); word[len] = FIRST_ASCII + i; // append current char // chain middle node @@ -315,8 +315,7 @@ void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) { } } - // return - return; + taosMemoryFree(root); } SMatch* matchPrefix(STire* tire, char* prefix, SMatch* match) { From a4fd2def98108357e1097afb955cd7177ac151d0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 17 Oct 2022 13:30:23 +0800 Subject: [PATCH 040/135] fix: resolve a typo in errormsg printing in walScanLogGetLastVer --- source/libs/wal/src/walMeta.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 279f4dc656..85791fa682 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -123,8 +123,8 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { } SWalCkHead* logContent = (SWalCkHead*)candidate; if (walValidHeadCksum(logContent) != 0) { - wError("vgId:%d, failed to validate checksum of wal entry header. offset:% %" PRId64 ", file:%s", - ((char*)(logContent)-buf), fnameStr); + wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, + offset + ((char*)(logContent)-buf), fnameStr); haystack = candidate + 1; if (firstTrial) { break; @@ -162,8 +162,8 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { } if (walValidBodyCksum(logContent) != 0) { terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; - wError("vgId:%d, failed to validate checksum of wal entry body. offset:% %" PRId64 ", file:%s", - ((char*)(logContent)-buf), fnameStr); + wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, + offset + ((char*)(logContent)-buf), fnameStr); haystack = candidate + 1; if (firstTrial) { break; From 23a4a3767779d2c95b01787a5db25e88a9ca770e Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 17 Oct 2022 14:17:34 +0800 Subject: [PATCH 041/135] enh: not allowed to start when disk space unavailable --- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 076826ebc2..becefcd6a0 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -51,6 +51,7 @@ static int32_t dmInitMonitor() { static bool dmCheckDiskSpace() { osUpdate(); + // sufficiency if (!osDataSpaceSufficient()) { dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); } @@ -60,7 +61,21 @@ static bool dmCheckDiskSpace() { if (!osTempSpaceSufficient()) { dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); } - return true; + // availability + bool ret = true; + if (!osDataSpaceAvailable()) { + dError("data disk space unavailable, i.e. %s", tsDataDir); + ret = false; + } + if (!osLogSpaceAvailable()) { + dError("log disk space unavailable, i.e. %s", tsLogDir); + ret = false; + } + if (!osTempSpaceAvailable()) { + dError("temp disk space unavailable, i.e. %s", tsTempDir); + ret = false; + } + return ret; } static bool dmCheckDataDirVersion() { From 54e3617a6fb21967a1227164e60e207997815382 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 17 Oct 2022 14:32:18 +0800 Subject: [PATCH 042/135] fix:scan converity defects --- include/common/tmsg.h | 2 +- source/client/src/clientHb.c | 2 +- source/client/src/clientImpl.c | 4 +- source/client/src/clientRawBlockWrite.c | 20 ++++++---- source/client/src/clientSml.c | 36 +++++++++++------ source/client/src/clientStmt.c | 3 ++ source/client/src/clientTmq.c | 3 +- source/common/src/tdataformat.c | 4 ++ tools/taosws-rs | 1 + utils/test/c/sml_test.c | 8 ++-- utils/test/c/tmqSim.c | 51 ++++++++++++++----------- utils/test/c/tmq_taosx_ci.c | 2 +- 12 files changed, 85 insertions(+), 51 deletions(-) create mode 160000 tools/taosws-rs diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9f21ee007f..748bd6afcc 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1902,7 +1902,7 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) { if (pRebInfo == NULL) { return NULL; } - strcpy(pRebInfo->key, key); + tstrncpy(pRebInfo->key, key, sizeof(pRebInfo->key)); pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebInfo->lostConsumers == NULL) { goto _err; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 096e2cdac8..3f24a08cf4 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -173,7 +173,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; - tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, + tscTrace("conn %u hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, pTscObj->pAppInfo->totalDnodes); if (pRsp->query->killRid) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 940dd745c8..b494140da5 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -186,7 +186,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, STscObj* pTscObj = (*pRequest)->pTscObj; if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self))) { - tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, + tscError("%" PRIx64 " failed to add to request container, reqId:0x%" PRIu64 ", conn:%" PRIx64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); taosMemoryFree(param); @@ -371,7 +371,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) { pInfo->pQnodeList = taosArrayDup(pNodeList); taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad); tscDebug("QnodeList updated in cluster 0x%" PRIx64 ", num:%d", pInfo->clusterId, - taosArrayGetSize(pInfo->pQnodeList)); + (int)taosArrayGetSize(pInfo->pQnodeList)); } taosThreadMutexUnlock(&pInfo->qnodeMutex); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index a257335931..1dc319be94 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -410,6 +410,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVAlterTbReq vAlterTbReq = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -419,7 +420,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -524,6 +525,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropStbReq req = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -533,7 +535,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -556,6 +558,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropTbBatchReq req = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -565,7 +568,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -684,7 +687,7 @@ end: static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { SVDropStbReq req = {0}; - SDecoder coder; + SDecoder coder = {0}; SMDropStbReq pReq = {0}; int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; @@ -1212,6 +1215,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; + SSubmitReq* subReq = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); if (!pRequest) { @@ -1228,8 +1232,8 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) } SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - strcpy(pName.dbname, pRequest->pDb); - strcpy(pName.tname, tbname); + tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname)); + tstrncpy(pName.tname, tbname, sizeof(pName.tname)); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); @@ -1278,7 +1282,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; int32_t totalLen = sizeof(SSubmitReq) + submitLen; - SSubmitReq* subReq = taosMemoryCalloc(1, totalLen); + subReq = taosMemoryCalloc(1, totalLen); SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); @@ -1352,6 +1356,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) if (NULL == pQuery) { uError("create SQuery error"); code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(subReq); goto end; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; @@ -1390,6 +1395,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); + taosMemoryFree(subReq); return code; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 24da5f7b70..5c37822222 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -299,6 +299,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { + taosHashCleanup(hashTmp); return -1; } } @@ -430,7 +431,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { STableMeta *pTableMeta = NULL; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); SRequestConnInfo conn = {0}; conn.pTrans = info->taos->pAppInfo->pTransporter; @@ -874,7 +875,8 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra kv->i = ts; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); + return TSDB_CODE_SUCCESS; } @@ -1009,6 +1011,7 @@ static void smlParseTelnetElement(const char **sql, const char **data, int32_t * static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { + if(!cols) return TSDB_CODE_OUT_OF_MEMORY; const char *sql = data; size_t childTableNameLen = strlen(tsSmlChildTableName); while (*sql != '\0') { @@ -1082,7 +1085,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab kv->length = valueLen; kv->type = TSDB_DATA_TYPE_NCHAR; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); } return TSDB_CODE_SUCCESS; @@ -1370,8 +1373,14 @@ static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { SHashObj *s2 = *(SHashObj **)key2; SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN); SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN); - ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); + if(!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP){ + uError("smlKvTimeHashCompare kv1"); + return -1; + } + if(!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP){ + uError("smlKvTimeHashCompare kv2"); + return -1; + } if (kv1->i < kv2->i) { return -1; } else if (kv1->i > kv2->i) { @@ -1735,7 +1744,7 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { kv->i = tsVal; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); return TSDB_CODE_SUCCESS; } @@ -1932,6 +1941,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { } static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { + if(!cols) return TSDB_CODE_OUT_OF_MEMORY; cJSON *metricVal = cJSON_GetObjectItem(root, "value"); if (metricVal == NULL) { return TSDB_CODE_TSC_INVALID_JSON; @@ -1941,7 +1951,7 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { if (!kv) { return TSDB_CODE_OUT_OF_MEMORY; } - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); kv->key = VALUE; kv->keyLen = VALUE_LEN; @@ -1955,7 +1965,9 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { int32_t ret = TSDB_CODE_SUCCESS; - + if (!pKVs){ + return TSDB_CODE_OUT_OF_MEMORY; + } cJSON *tags = cJSON_GetObjectItem(root, "tags"); if (tags == NULL || tags->type != cJSON_Object) { return TSDB_CODE_TSC_INVALID_JSON; @@ -1985,14 +1997,14 @@ static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableN return TSDB_CODE_TSC_INVALID_JSON; } memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); + tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); continue; } // add kv to SSmlKv SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - if (pKVs) taosArrayPush(pKVs, &kv); + taosArrayPush(pKVs, &kv); // key kv->keyLen = keyLen; @@ -2103,6 +2115,8 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { if (!oneTable) { tinfo = smlBuildTableInfo(); if (!tinfo) { + smlDestroyCols(cols); + if (info->dataFormat) taosArrayDestroy(cols); return TSDB_CODE_TSC_OUT_OF_MEMORY; } taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); @@ -2295,7 +2309,7 @@ static int32_t smlInsertData(SSmlHandle *info) { SSmlTableInfo *tableData = *oneTable; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); SRequestConnInfo conn = {0}; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 7a800bd334..bf3fd00f14 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -201,6 +201,9 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if(!pSrc){ + return TSDB_CODE_OUT_OF_MEMORY; + } STableDataBlocks* pDst = NULL; STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 922cab23c8..fc83c52cdd 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -870,8 +870,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("consumer %" PRId64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), - pTmq->groupId); + tscError("consumer setup failed since %s", terrstr()); return NULL; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ad5772e0fe..269e72f151 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -916,6 +916,10 @@ char *tTagValToData(const STagVal *value, bool isJson) { } bool tTagGet(const STag *pTag, STagVal *pTagVal) { + if(!pTag || !pTagVal){ + return false; + } + int16_t lidx = 0; int16_t ridx = pTag->nTag - 1; int16_t midx; diff --git a/tools/taosws-rs b/tools/taosws-rs new file mode 160000 index 0000000000..7a94ffab45 --- /dev/null +++ b/tools/taosws-rs @@ -0,0 +1 @@ +Subproject commit 7a94ffab45f08e16f09b3f430fe75d717054adb6 diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 56ba622a9c..b6d8d75ba0 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -119,7 +119,7 @@ int smlProcess_json1_Test() { " \"dc\": \"lga\"" " }" " }" - "]"}; + "]",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -159,7 +159,7 @@ int smlProcess_json2_Test() { " }," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -227,7 +227,7 @@ int smlProcess_json3_Test() { " }," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -286,7 +286,7 @@ int smlProcess_json4_Test() { " \"t9\": false," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index a043aa7a6d..d36fe0855a 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -155,7 +155,7 @@ static void printHelp() { printf("%s%s\n", indent, "-l"); - printf("%s%s%s\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); + printf("%s%s%s%d\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "producer thread number, default is 0"); printf("%s%s\n", indent, "-b"); @@ -238,7 +238,7 @@ void saveConfigToLogFile() { taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); } taosFprintfFile(g_fp, "\n"); - taosFprintfFile(g_fp, " expect rows: %d\n", g_stConfInfo.stThreads[i].expectMsgCnt); + taosFprintfFile(g_fp, " expect rows: %" PRIx64 "\n", g_stConfInfo.stThreads[i].expectMsgCnt); } char tmpString[128]; @@ -263,11 +263,11 @@ void parseArgument(int32_t argc, char* argv[]) { printHelp(); exit(0); } else if (strcmp(argv[i], "-d") == 0) { - strcpy(g_stConfInfo.dbName, argv[++i]); + tstrncpy(g_stConfInfo.dbName, argv[++i], sizeof(g_stConfInfo.dbName)); } else if (strcmp(argv[i], "-w") == 0) { - strcpy(g_stConfInfo.cdbName, argv[++i]); + tstrncpy(g_stConfInfo.cdbName, argv[++i], sizeof(g_stConfInfo.cdbName)); } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); + tstrncpy(configDir, argv[++i], PATH_MAX); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { @@ -279,9 +279,9 @@ void parseArgument(int32_t argc, char* argv[]) { } else if (strcmp(argv[i], "-e") == 0) { g_stConfInfo.useSnapshot = atol(argv[++i]); } else if (strcmp(argv[i], "-t") == 0) { - char tmpBuf[56]; - strcpy(tmpBuf, argv[++i]); - sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); + char tmpBuf[56] = {0}; + tstrncpy(tmpBuf, argv[++i], sizeof(tmpBuf)); + sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); } else if (strcmp(argv[i], "-x") == 0) { g_stConfInfo.numOfThread = atol(argv[++i]); } else if (strcmp(argv[i], "-l") == 0) { @@ -294,6 +294,10 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.producerRate = atol(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { g_stConfInfo.payloadLen = atol(argv[++i]); + if(g_stConfInfo.payloadLen <= 0 || g_stConfInfo.payloadLen > 1024 * 1024 * 1024){ + pError("%s calloc size is too large: %s %s", GREEN, argv[++i], NC); + exit(-1); + } } else { pError("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); @@ -354,8 +358,8 @@ void ltrim(char* str) { int queryDB(TAOS* taos, char* command) { int retryCnt = 10; - int code; - TAOS_RES* pRes; + int code = 0; + TAOS_RES* pRes = NULL; while (retryCnt--) { pRes = taos_query(taos, command); @@ -363,10 +367,11 @@ int queryDB(TAOS* taos, char* command) { if (code != 0) { taosSsleep(1); taos_free_result(pRes); + pRes = NULL; continue; } taos_free_result(pRes); - return 0; + return 0; } pError("failed to reason:%s, sql: %s", tstrerror(code), command); @@ -418,7 +423,7 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { char sqlStr[1100] = {0}; if (strlen(buf) > 1024) { - taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf)); + taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", (int)strlen(buf)); taosCloseFile(&g_fp); return -1; } @@ -592,7 +597,7 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex); taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId); @@ -644,7 +649,7 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex); taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId); @@ -960,7 +965,7 @@ void parseConsumeInfo() { ltrim(pstr); char* ret = strchr(pstr, ch); memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr); - strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1); + tstrncpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1, sizeof(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey])); // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], // g_stConfInfo.value[g_stConfInfo.numOfKey]); g_stConfInfo.stThreads[i].numOfKey++; @@ -1268,25 +1273,26 @@ void* ombProduceThreadFunc(void* param) { for (int i = 0; i < batchPerTblTimes; ++i) { uint32_t msgsOfSql = g_stConfInfo.batchSize; if ((i == batchPerTblTimes - 1) && (0 != remainder)) { - msgsOfSql = remainder; + msgsOfSql = remainder; } int len = 0; len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "insert into %s values ", ctbName); for (int j = 0; j < msgsOfSql; j++) { - int64_t timeStamp = taosGetTimestampNs(); - len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); + int64_t timeStamp = taosGetTimestampNs(); + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); sendMsgs++; pInfo->totalProduceMsgs++; } - totalMsgLen += len; + totalMsgLen += len; pInfo->totalMsgsLen += len; - int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); + int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); if (affectedRows < 0) { taos_close(pInfo->taos); - pInfo->taos = NULL; - return NULL; + pInfo->taos = NULL; + taosMemoryFree(sqlBuf); + return NULL; } affectedRowsTotal += affectedRows; @@ -1322,6 +1328,7 @@ void* ombProduceThreadFunc(void* param) { printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); taos_close(pInfo->taos); pInfo->taos = NULL; + taosMemoryFree(sqlBuf); return NULL; } diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 3a598ba98b..6540fdac4c 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -663,7 +663,7 @@ void initLogFile() { int main(int argc, char* argv[]) { for (int32_t i = 1; i < argc; i++) { if(strcmp(argv[i], "-c") == 0){ - strcpy(g_conf.dir, argv[++i]); + tstrncpy(g_conf.dir, argv[++i], sizeof(g_conf.dir)); }else if(strcmp(argv[i], "-s") == 0){ g_conf.snapShot = true; }else if(strcmp(argv[i], "-d") == 0){ From 4c2896236f0a1065a7d7cbf51400fba4aee65685 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 17 Oct 2022 14:34:34 +0800 Subject: [PATCH 043/135] fix: reserve enough capacity for inter/final result block when performing scalar calculation of fill operator --- source/libs/executor/src/executorimpl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e59755787a..64f09cf766 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2816,6 +2816,8 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId); blockDataCleanup(pInfo->pRes); + blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); + blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.groupId) { From 8fe672feab8edac2bece5b3843a9dd13932ba4c7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 17 Oct 2022 14:34:51 +0800 Subject: [PATCH 044/135] fix: crash whle reconfig mnode will cause taosd reboot failure --- source/dnode/mnode/impl/src/mndMnode.c | 30 ++++++++++++++------------ source/libs/sync/src/syncMain.c | 18 ++++++++++++++++ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index c27241317c..63ce538536 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -394,7 +394,6 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; - if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -478,7 +477,6 @@ static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeO static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; - int32_t numOfReplicas = 0; SDDropMnodeReq dropReq = {0}; SEpSet dropEpSet = {0}; @@ -505,9 +503,8 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { if (pObj == NULL) return 0; - if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1; - if (mndTransAppendNullLog(pTrans) != 0) return -1; + if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; return 0; } @@ -715,7 +712,8 @@ static void mndReloadSyncConfig(SMnode *pMnode) { SMnodeObj *pObj = NULL; ESdbStatus objStatus = 0; void *pIter = NULL; - bool hasUpdatingMnode = false; + int32_t updatingMnodes = 0; + int32_t readyMnodes = 0; SSyncCfg cfg = {.myIndex = -1}; while (1) { @@ -723,7 +721,11 @@ static void mndReloadSyncConfig(SMnode *pMnode) { if (pIter == NULL) break; if (objStatus == SDB_STATUS_CREATING || objStatus == SDB_STATUS_DROPPING) { mInfo("vgId:1, has updating mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus)); - hasUpdatingMnode = true; + updatingMnodes++; + } + if (objStatus == SDB_STATUS_READY) { + mInfo("vgId:1, has ready mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus)); + readyMnodes++; } if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) { @@ -739,18 +741,18 @@ static void mndReloadSyncConfig(SMnode *pMnode) { sdbReleaseLock(pSdb, pObj, false); } + if (readyMnodes <= 0 || updatingMnodes <= 0) { + mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes); + return; + } + if (cfg.myIndex == -1) { - mInfo("vgId:1, mnode not reload since selfIndex is -1"); + mInfo("vgId:1, mnode sync not reconfig since selfIndex is -1"); return; } - if (!mndGetRestored(pMnode)) { - mInfo("vgId:1, mnode not reload since restore not finished"); - return; - } - - if (hasUpdatingMnode) { - mInfo("vgId:1, start to reload mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); + if (updatingMnodes > 0) { + mInfo("vgId:1, mnode sync reconfig, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); for (int32_t i = 0; i < cfg.replicaNum; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 907eb21f4c..b255d83933 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1891,8 +1891,26 @@ inline bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config) { return b1; } +static bool syncIsConfigChanged(const SSyncCfg* pOldCfg, const SSyncCfg* pNewCfg) { + if (pOldCfg->replicaNum != pNewCfg->replicaNum) return true; + if (pOldCfg->myIndex != pNewCfg->myIndex) return true; + for (int32_t i = 0; i < pOldCfg->replicaNum; ++i) { + const SNodeInfo* pOldInfo = &pOldCfg->nodeInfo[i]; + const SNodeInfo* pNewInfo = &pNewCfg->nodeInfo[i]; + if (strcmp(pOldInfo->nodeFqdn, pNewInfo->nodeFqdn) != 0) return true; + if (pOldInfo->nodePort != pNewInfo->nodePort) return true; + } + + return false; +} + void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex lastConfigChangeIndex) { SSyncCfg oldConfig = pSyncNode->pRaftCfg->cfg; + if (!syncIsConfigChanged(&oldConfig, pNewConfig)) { + sInfo("vgId:1, sync not reconfig since not changed"); + return; + } + pSyncNode->pRaftCfg->cfg = *pNewConfig; pSyncNode->pRaftCfg->lastConfigIndex = lastConfigChangeIndex; From 73662b3058eb670198675dcd5354fe569e76af00 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 11:17:41 +0800 Subject: [PATCH 045/135] refactor: table sink --- source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tq.c | 12 +- source/dnode/vnode/src/tq/tqSink.c | 206 ++++++++++++++++++++ tests/system-test/7-tmq/tmqAutoCreateTbl.py | 2 +- 4 files changed, 214 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index f96afe6fba..69e6cdce9f 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -115,7 +115,6 @@ typedef struct { typedef struct { SMqDataRsp dataRsp; - SMqRspHead rspHead; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; SRpcHandleInfo pInfo; } STqPushEntry; @@ -183,6 +182,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t ver); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index aaa57d49ed..5e1cc15063 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -192,7 +192,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { return -1; } - memcpy(buf, &pPushEntry->rspHead, sizeof(SMqRspHead)); + memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); @@ -215,7 +215,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { tFormatOffset(buf1, 80, &pRsp->reqOffset); tFormatOffset(buf2, 80, &pRsp->rspOffset); tqDebug("vgId:%d, from consumer:%" PRId64 ", (epoch %d) push rsp, block num: %d, reqOffset:%s, rspOffset:%s", - TD_VID(pTq->pVnode), pPushEntry->rspHead.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); + TD_VID(pTq->pVnode), pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); return 0; } @@ -560,9 +560,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { memcpy(pPushEntry->subKey, pHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN); dataRsp.withTbName = 0; memcpy(&pPushEntry->dataRsp, &dataRsp, sizeof(SMqDataRsp)); - pPushEntry->rspHead.consumerId = consumerId; - pPushEntry->rspHead.epoch = reqEpoch; - pPushEntry->rspHead.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + pPushEntry->dataRsp.head.consumerId = consumerId; + pPushEntry->dataRsp.head.epoch = reqEpoch; + pPushEntry->dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey) + 1, &pPushEntry, sizeof(void*)); tqDebug("tmq poll: consumer %ld, subkey %s, vg %d save handle to push mgr", consumerId, pHandle->subKey, TD_VID(pTq->pVnode)); @@ -924,7 +924,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqTableSink; + pTask->tbSink.tbSinkFunc = tqTableSink1; ASSERT(pTask->tbSink.pSchemaWrapper); ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 521d12fdab..76bfa72caa 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -284,6 +284,212 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem return ret; } +void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper; + + int32_t blockSz = taosArrayGetSize(pBlocks); + bool createTb = true; + + SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + + tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + for (int32_t i = 0; i < blockSz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + SBatchDeleteReq deleteReq = {0}; + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + deleteReq.suid = suid; + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq); + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + taosArrayDestroy(deleteReq.deleteReqs); + + ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; + + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = serializedDeleteReq, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(serializedDeleteReq); + tqDebug("failed to put delete req into write-queue since %s", terrstr()); + } + } else { + SVCreateTbReq createTbReq = {0}; + + // set const + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, + }; + taosArrayPush(tagArray, &tagVal); + createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(tagArray); + tdDestroySVCreateTbReq(&createTbReq); + return; + } + createTbReq.ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + createTbReq.ctb.tagName = tagName; + + // set table name + if (pDataBlock->info.parTbName[0]) { + createTbReq.name = strdup(pDataBlock->info.parTbName); + } else { + createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + } + + int32_t schemaLen; + int32_t code; + tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + // save schema str + void* schemaStr = taosMemoryMalloc(schemaLen); + if (schemaStr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, schemaStr, schemaLen); + code = tEncodeSVCreateTbReq(&encoder, &createTbReq); + if (code < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + tEncoderClear(&encoder); + taosMemoryFree(schemaStr); + return; + } + tEncoderClear(&encoder); + tdDestroySVCreateTbReq(&createTbReq); + + int32_t cap = sizeof(SSubmitReq); + + int32_t rows = pDataBlock->info.rows; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + + cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; + + SSubmitReq* ret = rpcMallocCont(cap); + ret->header.vgId = pVnode->config.vgId; + ret->length = sizeof(SSubmitReq); + ret->numOfBlocks = htonl(1); + + SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); + + blkHead->numOfRows = htonl(pDataBlock->info.rows); + blkHead->sversion = htonl(pTSchema->version); + blkHead->suid = htobe64(suid); + // uid is assigned by vnode + blkHead->uid = 0; + blkHead->schemaLen = 0; + + tqDebug("tq sink, convert block %d, rows: %d", i, rows); + + int32_t dataLen = 0; + void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); + STSRow* rowData = blkSchema; + if (createTb) { + memcpy(blkSchema, schemaStr, schemaLen); + blkHead->schemaLen = htonl(schemaLen); + rowData = POINTER_SHIFT(blkSchema, schemaLen); + } + + taosMemoryFree(schemaStr); + + for (int32_t j = 0; j < rows; j++) { + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTSchema->version); + tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); + tdSRowResetBuf(&rb, rowData); + + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pColumn = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); + } else { + void* colData = colDataGetData(pColData, j); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k); + } + } + tdSRowEnd(&rb); + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + dataLen += rowLen; + } + blkHead->dataLen = htonl(dataLen); + + ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; + ret->length = htonl(ret->length); + + SRpcMsg msg = { + .msgType = TDMT_VND_SUBMIT, + .pCont = ret, + .contLen = ntohl(ret->length), + }; + + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(ret); + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + } + taosArrayDestroy(tagArray); +} + void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pRes = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index a613f11267..de95fef148 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -52,7 +52,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1, wal_retention_size=-1,wal_retention_period=-1) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) # tdLog.info("create ctb") From 2abd0eec40474558a8d6a93b21156e7fcce56c55 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 046/135] fix: coverity issues CID: 354727 --- source/common/src/tvariant.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 830a6199d8..0533a49514 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -28,10 +28,6 @@ *exti = -1; \ break; \ } \ - if ((res) > (maxv)) { \ - *exti = 1; \ - break; \ - } \ assert(0); \ } while (0) From d2fc6215c6a0c9247b14c1fe49f320db573b8ca3 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 15:06:05 +0800 Subject: [PATCH 047/135] fix: client mem leak --- source/client/src/clientTmq.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 73f7c2bcc4..51005e3c30 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -709,6 +709,7 @@ void tmqSendHbReq(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { + taosMemoryFree(param); return; } int64_t consumerId = tmq->consumerId; @@ -939,10 +940,9 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { return NULL; } - int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); - *pRefId = pTmq->refId; - if (pTmq->hbBgEnable) { + int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); + *pRefId = pTmq->refId; pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer); } From 7e54b77024ab488222f6d7bf6e909f79e4fa5ffd Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 048/135] fix: coverity issues CID: 354728 --- source/common/src/tvariant.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 0533a49514..d0f4855549 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -28,6 +28,10 @@ *exti = -1; \ break; \ } \ + if ((res) > (maxv)) { \ + *exti = -1; \ + break; \ + } \ assert(0); \ } while (0) @@ -685,7 +689,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc case TSDB_DATA_TYPE_BIGINT: { if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { - SET_EXT_INFO(converted, (int64_t)result, INT64_MIN + 1, INT64_MAX, extInfo); + SET_EXT_INFO(converted, result, INT64_MIN + 1, INT64_MAX, extInfo); return -1; } *((int64_t *)payload) = (int64_t)result; @@ -694,7 +698,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc case TSDB_DATA_TYPE_UBIGINT: { if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) { - SET_EXT_INFO(converted, (uint64_t)result, 0, UINT64_MAX - 1, extInfo); + SET_EXT_INFO(converted, result, 0, UINT64_MAX - 1, extInfo); return -1; } *((uint64_t *)payload) = (uint64_t)result; From 5c66712b9123770465a784e8246154a4942ca4be Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 17 Oct 2022 15:28:10 +0800 Subject: [PATCH 049/135] fix: coverity issues --- source/dnode/vnode/src/meta/metaQuery.c | 2 +- source/dnode/vnode/src/meta/metaTable.c | 7 ++++--- source/libs/tdb/src/db/tdbBtree.c | 1 + source/libs/tdb/src/db/tdbPCache.c | 7 ++++--- source/libs/tdb/src/db/tdbPager.c | 21 +++++++++++++++------ source/libs/tdb/src/db/tdbTable.c | 2 ++ source/libs/tdb/src/inc/tdbOs.h | 4 +++- 7 files changed, 30 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 84ce087e55..631ef09d4b 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -675,7 +675,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv SSchemaWrapper *pSchemaWrapper = &schema; tDecoderInit(&dc, pData, nData); - tDecodeSSchemaWrapper(&dc, pSchemaWrapper); + (void)tDecodeSSchemaWrapper(&dc, pSchemaWrapper); tDecoderClear(&dc); tdbFree(pData); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 1e0e757793..2b7982d381 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -944,6 +944,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaUpdateTagIdx(pMeta, &ctbEntry); } + ASSERT(ctbEntry.ctbEntry.pTags); SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); @@ -952,7 +953,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tDecoderClear(&dc1); tDecoderClear(&dc2); - if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); + taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); tdbTbcClose(pTbDbc); @@ -1202,8 +1203,8 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { SMetaEntry stbEntry = {0}; STagIdxKey *pTagIdxKey = NULL; int32_t nTagIdxKey; - const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; - const void *pTagData = NULL; // + const SSchema *pTagColumn; + const void *pTagData = NULL; int32_t nTagData = 0; SDecoder dc = {0}; int32_t ret = 0; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 13b7e0ac0a..9e17f50dce 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1006,6 +1006,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const nLeft -= kLen; // pack partial val to local if any space left if (nLocal > nHeader + kLen + sizeof(SPgno)) { + ASSERT(pVal != NULL && vLen != 0); memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno)); nLeft -= nLocal - nHeader - kLen - sizeof(SPgno); } diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index b6d1f95c0e..c73ddce74c 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -105,6 +105,7 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) { for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) { if (tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL) < 0) { // TODO: handle error + tdbOsFree(pCache->aPage); return -1; } @@ -267,7 +268,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) // 4. Try a create new page if (!pPage) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); - if (ret < 0) { + if (ret < 0 && pPage != NULL) { // TODO ASSERT(0); return NULL; @@ -300,8 +301,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) pPage->pPager = pPageH->pPager; memcpy(pPage->pData, pPageH->pData, pPage->pageSize); - tdbDebug("pcache/pPageH: %p %d %p %p %d", pPageH, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize, pPage, - TDB_PAGE_PGNO(pPageH)); + // tdbDebug("pcache/pPageH: %p %d %p %p %d", pPageH, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize, pPage, + // TDB_PAGE_PGNO(pPageH)); tdbPageInit(pPage, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize); pPage->kLen = pPageH->kLen; pPage->vLen = pPageH->vLen; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index dd2416e5b8..57ad2a783a 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -84,7 +84,8 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { pPager->pCache = pCache; pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); - if (pPager->fd < 0) { + if (TDB_FD_INVALID(pPager->fd)) { + // if (pPager->fd < 0) { return -1; } @@ -226,7 +227,7 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { // Open the journal pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); - if (pPager->jfd < 0) { + if (TDB_FD_INVALID(pPager->jfd)) { tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -365,7 +366,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // 4, remove the journal file tdbOsClose(pPager->jfd); - tdbOsRemove(pPager->jFileName); + (void)tdbOsRemove(pPager->jFileName); pPager->inTran = 0; return 0; @@ -540,7 +541,8 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { - tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, pPage->pageSize); + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, + pPage->pageSize); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -568,7 +570,8 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize); if (ret < 0) { - tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName, pPage->pageSize); + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName, + pPage->pageSize); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -603,11 +606,13 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); if (ret < 0) { + tdbOsFree(pageBuf); return -1; } ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { + tdbOsFree(pageBuf); return -1; } @@ -615,13 +620,16 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { tdbError("failed to lseek fd due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); return -1; } ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize); if (ret < 0) { - tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, pPager->pageSize); + tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, + pPager->pageSize); terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); return -1; } } @@ -629,6 +637,7 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { if (tdbOsFSync(pPager->fd) < 0) { tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); return -1; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 380baefb46..008907ca77 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -106,11 +106,13 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF // pTb->pBt ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, &(pTb->pBt)); if (ret < 0) { + tdbOsFree(pTb); return -1; } ret = tdbPagerRestore(pPager, pTb->pBt); if (ret < 0) { + tdbOsFree(pTb); return -1; } diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index 503e109adb..b5dd27052c 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -37,6 +37,8 @@ extern "C" { /* file */ typedef TdFilePtr tdb_fd_t; +#define TDB_FD_INVALID(fd) (fd == NULL) + #define TDB_O_CREAT TD_FILE_CREATE #define TDB_O_WRITE TD_FILE_WRITE #define TDB_O_READ TD_FILE_READ @@ -141,4 +143,4 @@ typedef pthread_mutex_t tdb_mutex_t; } #endif -#endif /*_TDB_OS_H_*/ \ No newline at end of file +#endif /*_TDB_OS_H_*/ From 9abea767912eafa6211ad1704e00947fdcf4e2a8 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 17 Oct 2022 15:43:42 +0800 Subject: [PATCH 050/135] fix: do coverity scan --- include/libs/nodes/plannodes.h | 2 +- include/libs/nodes/querynodes.h | 2 +- source/libs/command/src/command.c | 146 +++++++++++++++------- source/libs/executor/src/dataDispatcher.c | 2 + source/libs/parser/src/parInsert.c | 81 +++++------- source/libs/parser/src/parInsertData.c | 4 +- source/libs/parser/src/parTokenizer.c | 2 + source/libs/parser/src/parTranslater.c | 23 ++-- source/libs/parser/src/parUtil.c | 2 +- source/libs/planner/src/planOptimizer.c | 3 +- source/libs/planner/src/planSpliter.c | 6 +- 11 files changed, 165 insertions(+), 108 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index c57e9718b2..d0e1c3d258 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -525,7 +525,7 @@ typedef struct SDataInserterNode { SDataSinkNode sink; int32_t numOfTables; uint32_t size; - char* pData; + void* pData; } SDataInserterNode; typedef struct SQueryInserterNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 8d85c139df..5f2ce66f25 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -356,7 +356,7 @@ typedef struct SVgDataBlocks { SVgroupInfo vg; int32_t numOfTables; // number of tables in current submit block uint32_t size; - char* pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ... + void* pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ... } SVgDataBlocks; typedef struct SVnodeModifOpStmt { diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 18d839e109..33e5b0e1e4 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -43,7 +43,6 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe blockEncode(pBlock, (*pRsp)->data, &len, numOfCols, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); - blockDataDestroy(pBlock); return TSDB_CODE_SUCCESS; } @@ -59,21 +58,33 @@ static int32_t getSchemaBytes(const SSchema* pSchema) { } } -static SSDataBlock* buildDescResultDataBlock() { +static int32_t buildDescResultDataBlock(SSDataBlock** pOutput) { SSDataBlock* pBlock = createDataBlock(); + if (NULL == pBlock) { + return TSDB_CODE_OUT_OF_MEMORY; + } SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_FIELD_LEN, 1); - blockDataAppendColInfo(pBlock, &infoData); + int32_t code = blockDataAppendColInfo(pBlock, &infoData); + if (TSDB_CODE_SUCCESS == code) { + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2); + code = blockDataAppendColInfo(pBlock, &infoData); + } + if (TSDB_CODE_SUCCESS == code) { + infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, tDataTypes[TSDB_DATA_TYPE_INT].bytes, 3); + code = blockDataAppendColInfo(pBlock, &infoData); + } + if (TSDB_CODE_SUCCESS == code) { + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_NOTE_LEN, 4); + code = blockDataAppendColInfo(pBlock, &infoData); + } - infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2); - blockDataAppendColInfo(pBlock, &infoData); - - infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, tDataTypes[TSDB_DATA_TYPE_INT].bytes, 3); - blockDataAppendColInfo(pBlock, &infoData); - - infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_NOTE_LEN, 4); - blockDataAppendColInfo(pBlock, &infoData); - return pBlock; + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pBlock; + } else { + blockDataDestroy(pBlock); + } + return code; } static void setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) { @@ -109,22 +120,39 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** SDescribeStmt* pDesc = (SDescribeStmt*)pStmt; int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta); - SSDataBlock* pBlock = buildDescResultDataBlock(); - setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta); - - return buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp); + SSDataBlock* pBlock = NULL; + int32_t code = buildDescResultDataBlock(&pBlock); + if (TSDB_CODE_SUCCESS == code) { + setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta); + } + if (TSDB_CODE_SUCCESS == code) { + code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp); + } + blockDataDestroy(pBlock); + return code; } static int32_t execResetQueryCache() { return catalogClearCache(); } -static SSDataBlock* buildCreateDBResultDataBlock() { - SSDataBlock* pBlock = createDataBlock(); - SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1); - blockDataAppendColInfo(pBlock, &infoData); +static int32_t buildCreateDBResultDataBlock(SSDataBlock** pOutput) { + SSDataBlock* pBlock = createDataBlock(); + if (NULL == pBlock) { + return TSDB_CODE_OUT_OF_MEMORY; + } - infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2); - blockDataAppendColInfo(pBlock, &infoData); - return pBlock; + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1); + int32_t code = blockDataAppendColInfo(pBlock, &infoData); + if (TSDB_CODE_SUCCESS == code) { + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2); + code = blockDataAppendColInfo(pBlock, &infoData); + } + + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pBlock; + } else { + blockDataDestroy(pBlock); + } + return code; } int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { @@ -259,21 +287,37 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S } static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) { - SSDataBlock* pBlock = buildCreateDBResultDataBlock(); - setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->pCfg); - return buildRetrieveTableRsp(pBlock, SHOW_CREATE_DB_RESULT_COLS, pRsp); + SSDataBlock* pBlock = NULL; + int32_t code = buildCreateDBResultDataBlock(&pBlock); + if (TSDB_CODE_SUCCESS == code) { + setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->pCfg); + } + if (TSDB_CODE_SUCCESS == code) { + code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_DB_RESULT_COLS, pRsp); + } + blockDataDestroy(pBlock); + return code; } -static SSDataBlock* buildCreateTbResultDataBlock() { +static int32_t buildCreateTbResultDataBlock(SSDataBlock** pOutput) { SSDataBlock* pBlock = createDataBlock(); + if (NULL == pBlock) { + return TSDB_CODE_OUT_OF_MEMORY; + } SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD1_LEN, 1); - blockDataAppendColInfo(pBlock, &infoData); + int32_t code = blockDataAppendColInfo(pBlock, &infoData); + if (TSDB_CODE_SUCCESS == code) { + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2); + code = blockDataAppendColInfo(pBlock, &infoData); + } - infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2); - blockDataAppendColInfo(pBlock, &infoData); - - return pBlock; + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pBlock; + } else { + blockDataDestroy(pBlock); + } + return code; } void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { @@ -482,12 +526,16 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p } static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { - SSDataBlock* pBlock = buildCreateTbResultDataBlock(); - int32_t code = setCreateTBResultIntoDataBlock(pBlock, pStmt->pDbCfg, pStmt->tableName, pStmt->pTableCfg); - if (code) { - return code; + SSDataBlock* pBlock = NULL; + int32_t code = buildCreateTbResultDataBlock(&pBlock); + if (TSDB_CODE_SUCCESS == code) { + code = setCreateTBResultIntoDataBlock(pBlock, pStmt->pDbCfg, pStmt->tableName, pStmt->pTableCfg); } - return buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp); + if (TSDB_CODE_SUCCESS == code) { + code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp); + } + blockDataDestroy(pBlock); + return code; } static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { @@ -556,8 +604,12 @@ _return: return TSDB_CODE_SUCCESS; } -static SSDataBlock* buildLocalVariablesResultDataBlock() { +static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + if (NULL == pBlock) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pBlock->info.hasVarCol = true; pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); @@ -572,7 +624,8 @@ static SSDataBlock* buildLocalVariablesResultDataBlock() { infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN; taosArrayPush(pBlock->pDataBlock, &infoData); - return pBlock; + *pOutput = pBlock; + return TSDB_CODE_SUCCESS; } int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { @@ -604,12 +657,16 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { } static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) { - SSDataBlock* pBlock = buildLocalVariablesResultDataBlock(); - int32_t code = setLocalVariablesResultIntoDataBlock(pBlock); - if (code) { - return code; + SSDataBlock* pBlock = NULL; + int32_t code = buildLocalVariablesResultDataBlock(&pBlock); + if (TSDB_CODE_SUCCESS == code) { + code = setLocalVariablesResultIntoDataBlock(pBlock); } - return buildRetrieveTableRsp(pBlock, SHOW_LOCAL_VARIABLES_RESULT_COLS, pRsp); + if (TSDB_CODE_SUCCESS == code) { + code = buildRetrieveTableRsp(pBlock, SHOW_LOCAL_VARIABLES_RESULT_COLS, pRsp); + } + blockDataDestroy(pBlock); + return code; } static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** pOutput) { @@ -659,6 +716,7 @@ static int32_t execSelectWithoutFrom(SSelectStmt* pSelect, SRetrieveTableRsp** p if (TSDB_CODE_SUCCESS == code) { code = buildRetrieveTableRsp(pBlock, LIST_LENGTH(pSelect->pProjectionList), pRsp); } + blockDataDestroy(pBlock); return code; } diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 8f8e486b3a..0b5c8372f1 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -170,6 +170,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); + ASSERT(NULL != pBuf); memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); @@ -258,6 +259,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD dispatcher->pDataBlocks = taosOpenQueue(); taosThreadMutexInit(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { + taosMemoryFree(dispatcher); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 5b861ef79a..e3bcabe946 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -543,7 +543,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_UTINYINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(uv)) { + } else if (uv > UINT8_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); } uint8_t tmpVal = (uint8_t)uv; @@ -563,7 +563,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_USMALLINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(uv)) { + } else if (uv > UINT16_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); } uint16_t tmpVal = (uint16_t)uv; @@ -583,7 +583,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_UINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(uv)) { + } else if (uv > UINT32_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); } uint32_t tmpVal = (uint32_t)uv; @@ -600,8 +600,6 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_UBIGINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT(uv)) { - return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); } return func(pMsgBuf, &uv, pSchema->bytes, param); } @@ -844,7 +842,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_UTINYINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(uv)) { + } else if (uv > UINT8_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); } *(uint8_t*)(&val->i64) = uv; @@ -864,7 +862,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_USMALLINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(uv)) { + } else if (uv > UINT16_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); } *(uint16_t*)(&val->i64) = uv; @@ -884,7 +882,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_UINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(uv)) { + } else if (uv > UINT32_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); } *(uint32_t*)(&val->i64) = uv; @@ -902,8 +900,6 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_UBIGINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT(uv)) { - return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); } *(uint64_t*)(&val->i64) = uv; break; @@ -994,7 +990,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint isParseBindParam = true; if (NULL == pCxt->pStmtCb) { code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); - goto end; + break; } continue; @@ -1002,57 +998,47 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint if (isParseBindParam) { code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); - goto end; + break; } SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // todo this can be optimize with parse column code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - if (!isNullValue(pTagSchema->type, &sToken)) { - taosArrayPush(tagName, pTagSchema->name); - } - if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { - if (sToken.n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - code = buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", sToken.z); - goto end; + if (TSDB_CODE_SUCCESS == code) { + if (!isNullValue(pTagSchema->type, &sToken)) { + taosArrayPush(tagName, pTagSchema->name); } - if (isNullValue(pTagSchema->type, &sToken)) { - code = tTagNew(pTagVals, 1, true, &pTag); + if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { + isJson = true; + if (sToken.n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + code = buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", sToken.z); + break; + } + if (isNullValue(pTagSchema->type, &sToken)) { + code = tTagNew(pTagVals, 1, true, &pTag); + } else { + code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg); + } } else { - code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg); + STagVal val = {0}; + code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, &val, &pCxt->msg); + if (TSDB_CODE_SUCCESS == code) { + taosArrayPush(pTagVals, &val); + } } - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - isJson = true; - } else { - STagVal val = {0}; - code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, &val, &pCxt->msg); - if (TSDB_CODE_SUCCESS != code) { - goto end; - } - - taosArrayPush(pTagVals, &val); } } - if (isParseBindParam) { - code = TSDB_CODE_SUCCESS; - goto end; + if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !isJson) { + code = tTagNew(pTagVals, 1, false, &pTag); } - if (!isJson && (code = tTagNew(pTagVals, 1, false, &pTag)) != TSDB_CODE_SUCCESS) { - goto end; + if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { + buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid, pCxt->sTableName, tagName, + pCxt->pTableMeta->tableInfo.numOfTags); + pTag = NULL; } - buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid, pCxt->sTableName, tagName, - pCxt->pTableMeta->tableInfo.numOfTags); - -end: for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); if (IS_VAR_DATA_TYPE(p->type)) { @@ -1061,6 +1047,7 @@ end: } taosArrayDestroy(pTagVals); taosArrayDestroy(tagName); + tTagFree(pTag); return code; } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 997e680c99..09e40cad60 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -244,7 +244,7 @@ int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32 } } - taosHashPut(pHashList, (const char*)id, idLen, (char*)dataBlocks, POINTER_BYTES); + taosHashPut(pHashList, id, idLen, dataBlocks, POINTER_BYTES); if (pBlockList) { taosArrayPush(pBlockList, dataBlocks); } @@ -666,7 +666,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB } } else { for (int32_t i = 0; i < numOfRows; ++i) { - char* payload = (blkKeyTuple + i)->payloadAddr; + void* payload = (blkKeyTuple + i)->payloadAddr; TDRowLenT rowTLen = TD_ROW_LEN((STSRow*)payload); memcpy(pDataBlock, payload, rowTLen); pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 76c01cfd90..afcd3610f4 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -597,6 +597,8 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { *tokenId = TK_NK_BOOL; return i; } + *tokenId = tKeywordCode(z, i); + return i; } default: { if (((*z & 0x80) != 0) || !isIdChar[(uint8_t)*z]) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index dbe0c97ab6..a8a10312fd 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -995,13 +995,17 @@ static int32_t parseTimeFromValueNode(STranslateContext* pCxt, SValueNode* pVal) if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; } + int64_t value = 0; if (IS_UNSIGNED_NUMERIC_TYPE(pVal->node.resType.type)) { - pVal->datum.i = (int64_t)pVal->datum.u; + value = pVal->datum.u; } else if (IS_FLOAT_TYPE(pVal->node.resType.type)) { - pVal->datum.i = (int64_t)pVal->datum.d; + value = pVal->datum.d; } else if (TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) { - pVal->datum.i = pVal->datum.b; + value = pVal->datum.b; + } else { + value = pVal->datum.i; } + pVal->datum.i = value; return TSDB_CODE_SUCCESS; } else if (IS_VAR_DATA_TYPE(pVal->node.resType.type) || TSDB_DATA_TYPE_TIMESTAMP == pVal->node.resType.type) { if (TSDB_CODE_SUCCESS == taosParseTime(pVal->literal, &pVal->datum.i, pVal->node.resType.bytes, @@ -1083,7 +1087,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_UTINYINT: { code = toUInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.u); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_UTINYINT(pVal->datum.u))) { + if (strict && (TSDB_CODE_SUCCESS != code || pVal->datum.u > UINT8_MAX)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(uint8_t*)&pVal->typeData = pVal->datum.u; @@ -1091,7 +1095,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_USMALLINT: { code = toUInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.u); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_USMALLINT(pVal->datum.u))) { + if (strict && (TSDB_CODE_SUCCESS != code || pVal->datum.u > UINT16_MAX)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(uint16_t*)&pVal->typeData = pVal->datum.u; @@ -1099,7 +1103,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_UINT: { code = toUInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.u); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_UINT(pVal->datum.u))) { + if (strict && (TSDB_CODE_SUCCESS != code || pVal->datum.u > UINT32_MAX)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(uint32_t*)&pVal->typeData = pVal->datum.u; @@ -1107,7 +1111,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_UBIGINT: { code = toUInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.u); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_UBIGINT(pVal->datum.u))) { + if (strict && TSDB_CODE_SUCCESS != code) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(uint64_t*)&pVal->typeData = pVal->datum.u; @@ -4286,8 +4290,9 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); } if (TSDB_CODE_SUCCESS == code) { - if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) || - (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_NCHAR_LEN)) { + pCol->dataType.bytes = calcTypeBytes(pCol->dataType); + if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_NCHAR_LEN)) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index ee82e6c04b..95eb094edb 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -1200,7 +1200,7 @@ int32_t getTableMetaFromCacheForInsert(SArray* pTableMetaPos, SParseMetaCache* p int32_t reqIndex = *(int32_t*)taosArrayGet(pTableMetaPos, tableNo); SMetaRes* pRes = taosArrayGet(pMetaCache->pTableMetaData, reqIndex); if (TSDB_CODE_SUCCESS == pRes->code) { - *pMeta = tableMetaDup(pRes->pRes); + *pMeta = tableMetaDup((const STableMeta*)pRes->pRes); if (NULL == *pMeta) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index cd02e7a136..f4b8ff9fdf 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1960,7 +1960,8 @@ static SNode* rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNode if (NULL != pSelectValue) { strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName); } else { - snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, (void*)pFunc); + int64_t pointer = (int64_t)pFunc; + snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%" PRId64 "", pFunc->functionName, pointer); } int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pCol)); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 9567a2388f..a2bbc0af60 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -342,7 +342,8 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWStart->functionName, "_wstart"); - snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, (void*)pWStart); + int64_t pointer = (int64_t)pWStart; + snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%" PRId64 "", pWStart->functionName, pointer); int32_t code = fmGetFuncInfo(pWStart, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pFuncs, (SNode*)pWStart); @@ -367,7 +368,8 @@ static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWEnd->functionName, "_wend"); - snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, (void*)pWEnd); + int64_t pointer = (int64_t)pWEnd; + snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%" PRId64 "", pWEnd->functionName, pointer); int32_t code = fmGetFuncInfo(pWEnd, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pWin->pFuncs, (SNode*)pWEnd); From 51dd72ec82b019fe7f44b00ef1a3b5a4112e0e56 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 051/135] fix: coverity issues CID: 372918 --- source/libs/scalar/src/filter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 5865370e8e..0cfeeedde8 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1644,7 +1644,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SColumnNode *refNode = (SColumnNode *)left->desc; - if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS) { + if (unit->compare.optr <= OP_TYPE_JSON_CONTAINS) { len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->dataBlockId, refNode->slotId, operatorTypeStr(unit->compare.optr)); } @@ -1664,7 +1664,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); - if (unit->compare.optr2 >= 0 && unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS) { + if (unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS) { sprintf(str + strlen(str), "[%d][%d] %s [", refNode->dataBlockId, refNode->slotId, operatorTypeStr(unit->compare.optr2)); } From c6956061847a3fc4297132d20cf811392e37d489 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 052/135] fix: coverity issues CID: 399357 --- source/common/src/tdataformat.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ad5772e0fe..ef550891b7 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -602,7 +602,7 @@ _exit: } int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { - int32_t n; + int32_t n = 0; TSROW_LEN(pRow, n); if (p) { @@ -613,7 +613,7 @@ int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { } int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) { - int32_t n; + int32_t n = 0; *ppRow = (STSRow2 *)p; TSROW_LEN(*ppRow, n); From 4639051e683eced1583508d47aa20d43b0500512 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Oct 2022 16:03:45 +0800 Subject: [PATCH 053/135] fix(query): fix coverity issues. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 69 +++++++++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 - 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 670691ab1f..1da5297396 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1506,7 +1506,12 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* if (pReader->pMemSchema == NULL) { int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); - return pReader->pMemSchema; + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } else { + return pReader->pMemSchema; + } } if (pReader->pMemSchema->version == sversion) { @@ -1518,9 +1523,9 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; + } else { + return pReader->pMemSchema; } - - return pReader->pMemSchema; } static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, @@ -1816,7 +1821,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -1826,8 +1835,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow1); } else { init = true; - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } @@ -1837,7 +1850,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pSchema); + int32_t code = tRowMergerInit(&merge, piRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); } @@ -1847,7 +1863,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, pRow); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } @@ -1855,7 +1874,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } @@ -1865,7 +1888,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pSchema); + int32_t code = tRowMergerInit(&merge, piRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); } @@ -1876,7 +1902,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow1); } else { init = true; - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } @@ -1884,7 +1913,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } else { tRowMerge(&merge, &fRow); } @@ -3046,19 +3078,30 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SRowMerger merge = {0}; // get the correct schema for data in memory + terrno = 0; STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } if (pReader->pSchema == NULL) { pReader->pSchema = pTSchema; } - tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + int32_t code = tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + if(pTSchema1 == NULL) { + return terrno; + } + tRowMergerAdd(&merge, pNextRow, pTSchema1); doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); - int32_t code = tRowMergerGetRow(&merge, pTSRow); + code = tRowMergerGetRow(&merge, pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 47c836d9c1..4e02a28cdf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -719,8 +719,6 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { } pMerger->version = key.version; - -_exit: return code; } From 89f80436c65d95166cc8f704c86d2548270d2790 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Oct 2022 16:17:16 +0800 Subject: [PATCH 054/135] fix(query): fix coverity issues. --- source/libs/executor/src/tsimplehash.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 134862d88c..b66a6f6770 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -16,6 +16,7 @@ #include "tsimplehash.h" #include "taoserror.h" #include "tlog.h" +#include "tdef.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 #define HASH_MAX_CAPACITY (1024 * 1024 * 16L) @@ -110,14 +111,14 @@ static void tSimpleHashTableResize(SSHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, POINTER_BYTES * newCapacity); if (!pNewEntryList) { uWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; } size_t inc = newCapacity - pHashObj->capacity; - memset((char *)pNewEntryList + pHashObj->capacity * sizeof(void *), 0, inc * sizeof(void *)); + memset((char *)pNewEntryList + pHashObj->capacity * POINTER_BYTES, 0, inc * sizeof(void *)); pHashObj->hashList = pNewEntryList; pHashObj->capacity = newCapacity; From b286b0cf26617eb745d0d8f1a25bc024370e6bd6 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 17 Oct 2022 16:18:35 +0800 Subject: [PATCH 055/135] fix: do coverity scan --- source/libs/parser/src/parTranslater.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a8a10312fd..666d74ef5e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4248,7 +4248,7 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN code = taosHashPut(pHash, pTag->colName, len, &pTag, POINTER_BYTES); } if (TSDB_CODE_SUCCESS == code) { - tagsSize += pTag->dataType.bytes; + tagsSize += calcTypeBytes(pTag->dataType); } else { break; } @@ -4290,9 +4290,8 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); } if (TSDB_CODE_SUCCESS == code) { - pCol->dataType.bytes = calcTypeBytes(pCol->dataType); - if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_BINARY_LEN) || - (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && pCol->dataType.bytes > TSDB_MAX_NCHAR_LEN)) { + if ((TSDB_DATA_TYPE_VARCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_BINARY_LEN) || + (TSDB_DATA_TYPE_NCHAR == pCol->dataType.type && calcTypeBytes(pCol->dataType) > TSDB_MAX_NCHAR_LEN)) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); } } @@ -4300,7 +4299,7 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES); } if (TSDB_CODE_SUCCESS == code) { - rowSize += pCol->dataType.bytes; + rowSize += calcTypeBytes(pCol->dataType); } else { break; } From 6d965b8b819b7116d0d76357ae5cad92383d4b7c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Oct 2022 16:27:44 +0800 Subject: [PATCH 056/135] fix(query): fix coverity issues. --- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/scanoperator.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 41f861cd38..0f4118257b 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -294,7 +294,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo if (!exists) { #endif taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo); - taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(uid), &keyInfo.groupId, sizeof(keyInfo.groupId)); + taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(*uid), &keyInfo.groupId, sizeof(keyInfo.groupId)); } /*}*/ diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 15a7abf1ff..c6bd3532ef 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -942,8 +942,7 @@ static FORCE_INLINE void doClearBufferedBlocks(SStreamScanInfo* pInfo) { } static bool isSessionWindow(SStreamScanInfo* pInfo) { - return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || - pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; + return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; } static bool isStateWindow(SStreamScanInfo* pInfo) { From 68e0ddfa8831905c1546600718b5178b2776c7d2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 16:28:55 +0800 Subject: [PATCH 057/135] refactor: tmq client --- include/common/tmsg.h | 2 +- source/client/src/clientTmq.c | 44 +++++++++++++++++++---------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9f21ee007f..5e2ca8896a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1902,7 +1902,7 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) { if (pRebInfo == NULL) { return NULL; } - strcpy(pRebInfo->key, key); + tstrncpy(pRebInfo->key, key, TSDB_SUBSCRIBE_KEY_LEN); pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebInfo->lostConsumers == NULL) { goto _err; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 51005e3c30..8f94d88b1c 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -233,12 +233,12 @@ void tmq_conf_destroy(tmq_conf_t* conf) { tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { if (strcmp(key, "group.id") == 0) { - strcpy(conf->groupId, value); + tstrncpy(conf->groupId, value, TSDB_CGROUP_LEN); return TMQ_CONF_OK; } if (strcmp(key, "client.id") == 0) { - strcpy(conf->clientId, value); + tstrncpy(conf->clientId, value, 256); return TMQ_CONF_OK; } @@ -452,7 +452,6 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT int32_t code; tEncodeSize(tEncodeSTqOffset, pOffset, len, code); if (code < 0) { - ASSERT(0); return -1; } void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); @@ -464,15 +463,22 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT SEncoder encoder; tEncoderInit(&encoder, abuf, len); tEncodeSTqOffset(&encoder, pOffset); + tEncoderClear(&encoder); // build param SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); + if (pParam == NULL) { + taosMemoryFree(buf); + return -1; + } pParam->params = pParamSet; pParam->pOffset = pOffset; // build send info SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { + taosMemoryFree(buf); + taosMemoryFree(pParam); return -1; } pMsgSendInfo->msgInfo = (SDataBuf){ @@ -547,6 +553,8 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { + tsem_destroy(&pParamSet->rspSem); + taosMemoryFree(pParamSet); goto FAIL; } goto HANDLE_RSP; @@ -565,6 +573,7 @@ HANDLE_RSP: tsem_wait(&pParamSet->rspSem); code = pParamSet->rspErr; tsem_destroy(&pParamSet->rspSem); + taosMemoryFree(pParamSet); return code; } else { code = 0; @@ -587,7 +596,14 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + if (async) { + if (automatic) { + tmq->commitCb(tmq, code, tmq->commitCbUserParam); + } else { + userCb(tmq, code, userParam); + } + } return -1; } @@ -642,16 +658,6 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t code = pParamSet->rspErr; tsem_destroy(&pParamSet->rspSem); taosMemoryFree(pParamSet); - } else { - code = 0; - } - - if (code != 0 && async) { - if (automatic) { - tmq->commitCb(tmq, code, tmq->commitCbUserParam); - } else { - userCb(tmq, code, userParam); - } } #if 0 @@ -722,6 +728,7 @@ void tmqSendHbReq(void* param, void* tmrId) { SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { taosMemoryFree(pReq); + goto OVER; } sendInfo->msgInfo = (SDataBuf){ .pData = pReq, @@ -871,8 +878,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("consumer %" PRId64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), - pTmq->groupId); + tscError("setting up new consumer failed since %s, consumer group %s", terrstr(), conf->groupId); return NULL; } @@ -1061,9 +1067,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = 0; FAIL: if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree); - if (code != 0 && buf) { - taosMemoryFree(buf); - } + taosMemoryFree(buf); + return code; } @@ -1101,7 +1106,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { - taosMemoryFree(pMsg->pData); tscWarn("msg discard from vgId:%d, epoch %d since out of memory", vgId, epoch); goto CREATE_MSG_FAIL; } From c9aadfadee9fd0d6245b8c48fb6533833787f710 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 17 Oct 2022 16:29:50 +0800 Subject: [PATCH 058/135] fix coverity scan problem --- source/dnode/vnode/src/meta/metaQuery.c | 32 ++++++++++++------------- source/dnode/vnode/src/meta/metaTable.c | 4 ++++ source/libs/index/inc/indexInt.h | 2 +- source/libs/index/src/index.c | 2 +- source/libs/index/src/indexCache.c | 12 ++++++---- source/libs/index/src/indexComm.c | 1 + source/libs/index/src/indexFilter.c | 26 ++++++++++---------- source/libs/index/src/indexFst.c | 2 +- source/libs/index/src/indexFstFile.c | 6 +++-- source/libs/index/src/indexTfile.c | 11 +++++++-- source/libs/transport/src/thttp.c | 2 +- source/libs/transport/src/trans.c | 8 ++----- source/libs/transport/src/transCli.c | 18 +++++++------- source/os/src/osSocket.c | 8 ++++++- source/util/src/tsched.c | 1 + 15 files changed, 76 insertions(+), 59 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 84ce087e55..0852f566d6 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1059,7 +1059,7 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { if (param->val == NULL) { metaError("vgId:%d, failed to filter NULL data", TD_VID(pMeta->pVnode)); - return -1; + goto END; } else { if (IS_VAR_DATA_TYPE(param->type)) { tagData = varDataVal(param->val); @@ -1111,27 +1111,25 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { break; } } - if (p->suid != pKey->suid) { + if (p == NULL || p->suid != pKey->suid) { break; } first = false; - if (p != NULL) { - int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); - if (cmp == 0) { - // match - tb_uid_t tuid = 0; - if (IS_VAR_DATA_TYPE(pKey->type)) { - tuid = *(tb_uid_t *)(p->data + varDataTLen(p->data)); - } else { - tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes); - } - taosArrayPush(pUids, &tuid); - } else if (cmp == 1) { - // not match but should continue to iter + int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); + if (cmp == 0) { + // match + tb_uid_t tuid = 0; + if (IS_VAR_DATA_TYPE(pKey->type)) { + tuid = *(tb_uid_t *)(p->data + varDataTLen(p->data)); } else { - // not match and no more result - break; + tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes); } + taosArrayPush(pUids, &tuid); + } else if (cmp == 1) { + // not match but should continue to iter + } else { + // not match and no more result + break; } valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); if (valid < 0) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 97b0be07c1..385e59d152 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -119,6 +119,8 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const taosArrayDestroy(pTagVals); indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); indexMultiTermDestroy(terms); + + taosArrayDestroy(pTagVals); #endif return 0; } @@ -159,6 +161,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); type = TSDB_DATA_TYPE_VARCHAR; term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, val, len); + taosMemoryFree(val); } else if (pTagVal->nData == 0) { term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0); } @@ -177,6 +180,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche } indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); indexMultiTermDestroy(terms); + taosArrayDestroy(pTagVals); #endif return 0; } diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 9605528ad6..4e7c0a0ef7 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -138,7 +138,7 @@ void idxReleaseRef(int64_t ref); #define IDX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ do { \ uint8_t oldTy = ty; \ - ty = (ty >> 4) | exTy; \ + ty = ((ty >> 4) & 0xFF) | exTy; \ ty = (ty << 4) | oldTy; \ } while (0) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index cb671cfff9..76dc84ae42 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -139,7 +139,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { END: if (idx != NULL) { - indexClose(idx); + indexDestroy(idx); } *index = NULL; return ret; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 72f693f7e5..0bb454571a 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -538,7 +538,7 @@ int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) { idxCacheRef(pCache); // encode data CacheTerm* ct = taosMemoryCalloc(1, sizeof(CacheTerm)); - if (cache == NULL) { + if (ct == NULL) { return -1; } // set up key @@ -730,15 +730,17 @@ static int32_t idxCacheJsonTermCompare(const void* l, const void* r) { return cmp; } static MemTable* idxInternalCacheCreate(int8_t type) { - int ttype = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : TSDB_DATA_TYPE_BINARY; + // int ttype = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : + // TSDB_DATA_TYPE_BINARY; + int ttype = TSDB_DATA_TYPE_BINARY; int32_t (*cmpFn)(const void* l, const void* r) = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare; MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable)); idxMemRef(tbl); - if (ttype == TSDB_DATA_TYPE_BINARY || ttype == TSDB_DATA_TYPE_NCHAR) { - tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, ttype, MAX_INDEX_KEY_LEN, cmpFn, SL_ALLOW_DUP_KEY, idxCacheTermGet); - } + // if (ttype == TSDB_DATA_TYPE_BINARY || ttype == TSDB_DATA_TYPE_NCHAR) { + tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, ttype, MAX_INDEX_KEY_LEN, cmpFn, SL_ALLOW_DUP_KEY, idxCacheTermGet); + //} return tbl; } diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index ecf9136073..25036996fc 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -427,6 +427,7 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { *dst = taosMemoryCalloc(1, bufSize + 1); idxInt2str(*(uint64_t*)src, *dst, 1); tlen = strlen(*dst); + break; case TSDB_DATA_TYPE_FLOAT: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%.9lf", *(float*)src); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 6d81499dbc..a503b82d3a 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -231,7 +231,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SIF_ERR_RET(sifGetValueFromNode(node, ¶m->condValue)); param->colId = -1; param->colValType = (uint8_t)(vn->node.resType.type); - memcpy(param->colName, vn->literal, strlen(vn->literal)); + memcpy(param->colName, vn->literal, sizeof(param->colName)); break; } case QUERY_NODE_COLUMN: { @@ -400,54 +400,52 @@ static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reve static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { int8_t ltype = left->colValType, rtype = right->colValType; if (ltype == TSDB_DATA_TYPE_FLOAT) { - float f; + float f = 0; SIF_DATA_CONVERT(rtype, right->condValue, f); typedata->f = f; param->val = &typedata->f; } else if (ltype == TSDB_DATA_TYPE_DOUBLE) { - double d; + double d = 0; SIF_DATA_CONVERT(rtype, right->condValue, d); typedata->d = d; param->val = &typedata->d; } else if (ltype == TSDB_DATA_TYPE_BIGINT) { - int64_t i64; + int64_t i64 = 0; SIF_DATA_CONVERT(rtype, right->condValue, i64); typedata->i64 = i64; param->val = &typedata->i64; } else if (ltype == TSDB_DATA_TYPE_INT) { - int32_t i32; + int32_t i32 = 0; SIF_DATA_CONVERT(rtype, right->condValue, i32); typedata->i32 = i32; param->val = &typedata->i32; } else if (ltype == TSDB_DATA_TYPE_SMALLINT) { - int16_t i16; - + int16_t i16 = 0; SIF_DATA_CONVERT(rtype, right->condValue, i16); typedata->i16 = i16; param->val = &typedata->i16; } else if (ltype == TSDB_DATA_TYPE_TINYINT) { - int8_t i8; + int8_t i8 = 0; SIF_DATA_CONVERT(rtype, right->condValue, i8) typedata->i8 = i8; param->val = &typedata->i8; } else if (ltype == TSDB_DATA_TYPE_UBIGINT) { - uint64_t u64; + uint64_t u64 = 0; SIF_DATA_CONVERT(rtype, right->condValue, u64); typedata->u64 = u64; param->val = &typedata->u64; - } else if (ltype == TSDB_DATA_TYPE_UINT) { - uint32_t u32; + uint32_t u32 = 0; SIF_DATA_CONVERT(rtype, right->condValue, u32); typedata->u32 = u32; param->val = &typedata->u32; } else if (ltype == TSDB_DATA_TYPE_USMALLINT) { - uint16_t u16; + uint16_t u16 = 0; SIF_DATA_CONVERT(rtype, right->condValue, u16); typedata->u16 = u16; param->val = &typedata->u16; } else if (ltype == TSDB_DATA_TYPE_UTINYINT) { - uint8_t u8; + uint8_t u8 = 0; SIF_DATA_CONVERT(rtype, right->condValue, u8); typedata->u8 = u8; param->val = &typedata->u8; @@ -663,7 +661,7 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { // ugly code, refactor later if (nParam > 1 && params[1].status == SFLT_NOT_INDEX) { output->status = SFLT_NOT_INDEX; - return code; + goto _return; } SIF_ERR_JRET(sifGetOperFn(node->opType, &operFn, &output->status)); } diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 01dffa782d..aed2ec3ee3 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -338,7 +338,7 @@ uint8_t fstStateCommInput(FstState* s, bool* null) { return v; } // 0 indicate that common_input is None - return v == 0 ? 0 : COMMON_INPUT(v); + return COMMON_INPUT(v); } // input_len diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 7021fdfae3..33960ad8c2 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -72,7 +72,8 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of if (offset >= ctx->file.size) return 0; do { - char key[128] = {0}; + char key[1024] = {0}; + assert(strlen(ctx->file.buf) + 1 + 64 < sizeof(key)); idxGenLRUKey(key, ctx->file.buf, blkId); LRUHandle* h = taosLRUCacheLookup(ctx->lru, key, strlen(key)); @@ -99,6 +100,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of assert(blk->nread <= kBlockSize); if (blk->nread < kBlockSize && blk->nread < len) { + taosMemoryFree(blk); break; } @@ -150,7 +152,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int if (ctx->type == TFILE) { // ugly code, refactor later ctx->file.readOnly = readOnly; - memcpy(ctx->file.buf, path, strlen(path)); + memcpy(ctx->file.buf, path, sizeof(ctx->file.buf)); if (readOnly == false) { ctx->file.pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); taosFtruncateFile(ctx->file.pFile, 0); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index e746bb4303..2f879ff552 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -506,7 +506,9 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const c tfh.suid = suid; tfh.version = version; tfh.colType = colType; - memcpy(tfh.colName, colName, strlen(colName)); + if (strlen(colName) <= sizeof(tfh.colName)) { + memcpy(tfh.colName, colName, strlen(colName)); + } return tfileWriterCreate(wcx, &tfh); } @@ -580,8 +582,13 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { if (cap < ttsz) { cap = ttsz; - buf = (char*)taosMemoryRealloc(buf, cap); + char* t = (char*)taosMemoryRealloc(buf, cap); + if (t == NULL) { + taosMemoryFree(buf); + return -1; + } } + char* p = buf; tfileSerialTableIdsToBuf(p, v->tableId); tw->ctx->write(tw->ctx, buf, ttsz); diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index c1d8dbef62..2399891f6d 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -146,7 +146,7 @@ static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const if (nread < 0) { uError("http-report recv error:%s", uv_err_name(nread)); } else { - uTrace("http-report succ to recv %d bytes", nread); + uTrace("http-report succ to recv %d bytes", (int32_t)nread); } uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index c82af0d0e9..237a4b6059 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -54,11 +54,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->retry = pInit->rfp; pRpc->startTimer = pInit->tfp; - if (pInit->connType == TAOS_CONN_SERVER) { - pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - } else { - pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - } + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; uint32_t ip = 0; if (pInit->connType == TAOS_CONN_SERVER) { @@ -79,7 +75,7 @@ void* rpcOpen(const SRpcInit* pInit) { } pRpc->parent = pInit->parent; if (pInit->user) { - memcpy(pRpc->user, pInit->user, strlen(pInit->user)); + memcpy(pRpc->user, pInit->user, TSDB_UNI_LEN); } int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 359b9824cf..48b3097449 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -267,11 +267,12 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { #define EPSET_GET_SIZE(epSet) (epSet)->numOfEps #define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) -#define EPSET_FORWARD_INUSE(epSet) \ - do { \ - if ((epSet)->numOfEps != 0) { \ - (epSet)->inUse = (++((epSet)->inUse)) % ((epSet)->numOfEps); \ - } \ +#define EPSET_FORWARD_INUSE(epSet) \ + do { \ + if ((epSet)->numOfEps != 0) { \ + ++((epSet)->inUse); \ + (epSet)->inUse = ((epSet)->inUse) % ((epSet)->numOfEps); \ + } \ } while (0) #define EPSET_DEBUG_STR(epSet, tbuf) \ @@ -503,6 +504,7 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { SConnList list = {0}; taosHashPut((SHashObj*)pool, key, strlen(key), (void*)&list, sizeof(list)); plist = taosHashGet((SHashObj*)pool, key, strlen(key)); + if (plist == NULL) return NULL; QUEUE_INIT(&plist->conns); } @@ -1157,7 +1159,7 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, SCliObj* cli = taosMemoryCalloc(1, sizeof(SCliObj)); STrans* pTransInst = shandle; - memcpy(cli->label, label, strlen(label)); + memcpy(cli->label, label, TSDB_LABEL_LEN); cli->numOfThreads = numOfThreads; cli->pThreadObj = (SCliThrd**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrd*)); @@ -1611,8 +1613,8 @@ int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { SCvtAddr cvtAddr = {0}; if (ip != NULL && fqdn != NULL) { - memcpy(cvtAddr.ip, ip, strlen(ip)); - memcpy(cvtAddr.fqdn, fqdn, strlen(fqdn)); + if (strlen(ip) <= sizeof(cvtAddr.ip)) memcpy(cvtAddr.ip, ip, strlen(ip)); + if (strlen(fqdn) <= sizeof(cvtAddr.fqdn)) memcpy(cvtAddr.fqdn, fqdn, strlen(fqdn)); cvtAddr.cvt = true; } for (int i = 0; i < pTransInst->numOfThreads; i++) { diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index bfeef248cd..fc943d4d76 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -590,7 +590,9 @@ TdSocketPtr taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t taosCloseSocket(&pSocket); return NULL; } else { - taosKeepTcpAlive(pSocket); + if (taosKeepTcpAlive(pSocket) == -1) { + return NULL; + } } return pSocket; @@ -1059,18 +1061,22 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { } #if defined(WINDOWS) if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) { + taosCloseSocketNoCheck1(fd); return -1; } #elif defined(_TD_DARWIN_64) uint32_t conn_timeout_ms = timeout * 1000; if (0 != setsockopt(fd, IPPROTO_TCP, TCP_CONNECTIONTIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { + taosCloseSocketNoCheck1(fd); return -1; } #else // Linux like systems uint32_t conn_timeout_ms = timeout * 1000; if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { + taosCloseSocketNoCheck1(fd); return -1; } #endif + return (int)fd; } diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 9cf9e2c431..f524680331 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -38,6 +38,7 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab if (pSched->queue == NULL) { uError("%s: no enough memory for queue", label); taosCleanUpScheduler(pSched); + taosMemoryFree(pSched); return NULL; } From 51e5522ca442ad53d1b2653d8109bbaf10a4c95f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 059/135] fix: coverity issues CID: 399471 --- examples/c/tmq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 19adaad116..d3fc803c94 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -212,6 +212,7 @@ tmq_list_t* build_topic_list() { tmq_list_t* topicList = tmq_list_new(); int32_t code = tmq_list_append(topicList, "topicname"); if (code) { + tmq_list_destroy(topicList); return NULL; } return topicList; From 93aee94a3fed846a8a1eff0b257d5f5a675b4d93 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 17 Oct 2022 16:54:12 +0800 Subject: [PATCH 060/135] fix: do coverity scan --- source/libs/parser/src/parInsert.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index e3bcabe946..f59a6c7f9b 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1027,6 +1027,9 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } } } + if (TSDB_CODE_SUCCESS != code) { + break; + } } if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !isJson) { From 1a3b2fa4f2a3377df76a60f4bf6fb9b15d7fc219 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 061/135] fix: coverity issues CID: 399527 --- source/libs/scalar/src/filter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 0cfeeedde8..a04d5a3a3a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1214,7 +1214,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit *u, uint32_t *uidx) { SFilterFieldId left, right, *pright = &right; - int32_t type = FILTER_UNIT_DATA_TYPE(u); + uint8_t type = FILTER_UNIT_DATA_TYPE(u); uint16_t flag = 0; filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); @@ -2614,7 +2614,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum int32_t usize = (int32_t)taosArrayGetSize((SArray *)colInfo->info); for (int32_t n = 0; n < usize; ++n) { - SFilterUnit *u = taosArrayGetP((SArray *)colInfo->info, n); + SFilterUnit *u = (SFilterUnit *)taosArrayGetP((SArray *)colInfo->info, n); filterAddUnitFromUnit(info, &oinfo, u, &uidx); filterAddUnitToGroup(&ng, uidx); From bcae6e1f4e60bf0f69470e406f3bf5ce082db97c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 17:18:57 +0800 Subject: [PATCH 062/135] refactor(stream): optimize auto create child table --- source/dnode/vnode/src/tq/tqSink.c | 176 +++++++++++++++++------------ 1 file changed, 105 insertions(+), 71 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 76bfa72caa..366cd6268e 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -293,7 +293,6 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper; int32_t blockSz = taosArrayGetSize(pBlocks); - bool createTb = true; SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); if (!tagArray) { @@ -303,6 +302,7 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); for (int32_t i = 0; i < blockSz; i++) { + bool createTb = true; SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); if (pDataBlock->info.type == STREAM_DELETE_RESULT) { SBatchDeleteReq deleteReq = {0}; @@ -337,83 +337,115 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { tqDebug("failed to put delete req into write-queue since %s", terrstr()); } } else { - SVCreateTbReq createTbReq = {0}; - - // set const - createTbReq.flags = 0; - createTbReq.type = TSDB_CHILD_TABLE; - createTbReq.ctb.suid = suid; - - // set super table name - SName name = {0}; - tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); - - // set tag content - taosArrayClear(tagArray); - STagVal tagVal = { - .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .i64 = (int64_t)pDataBlock->info.groupId, - }; - taosArrayPush(tagArray, &tagVal); - createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); - - STag* pTag = NULL; - tTagNew(tagArray, 1, false, &pTag); - if (pTag == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(tagArray); - tdDestroySVCreateTbReq(&createTbReq); - return; - } - createTbReq.ctb.pTag = (uint8_t*)pTag; - - // set tag name - SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); - char tagNameStr[TSDB_COL_NAME_LEN] = {0}; - strcpy(tagNameStr, "group_id"); - taosArrayPush(tagName, tagNameStr); - createTbReq.ctb.tagName = tagName; - - // set table name + char* ctbName = NULL; + // set child table name if (pDataBlock->info.parTbName[0]) { - createTbReq.name = strdup(pDataBlock->info.parTbName); + ctbName = strdup(pDataBlock->info.parTbName); } else { - createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); } - int32_t schemaLen; - int32_t code; - tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - if (code < 0) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - return; - } + int32_t schemaLen = 0; + void* schemaStr = NULL; - // save schema str - void* schemaStr = taosMemoryMalloc(schemaLen); - if (schemaStr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - return; - } + int64_t uid = 0; + SMetaReader mr = {0}; + metaReaderInit(&mr, pVnode->pMeta, 0); + if (metaGetTableEntryByName(&mr, ctbName) < 0) { + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); - SEncoder encoder = {0}; - tEncoderInit(&encoder, schemaStr, schemaLen); - code = tEncodeSVCreateTbReq(&encoder, &createTbReq); - if (code < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); + SVCreateTbReq createTbReq = {0}; + + // set const + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + createTbReq.name = ctbName; + ctbName = NULL; + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, + }; + taosArrayPush(tagArray, &tagVal); + createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(tagArray); + tdDestroySVCreateTbReq(&createTbReq); + return; + } + createTbReq.ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + createTbReq.ctb.tagName = tagName; + + int32_t code; + tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + // set schema str + schemaStr = taosMemoryMalloc(schemaLen); + if (schemaStr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, schemaStr, schemaLen); + code = tEncodeSVCreateTbReq(&encoder, &createTbReq); + if (code < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + tEncoderClear(&encoder); + taosMemoryFree(schemaStr); + return; + } tEncoderClear(&encoder); - taosMemoryFree(schemaStr); - return; + tdDestroySVCreateTbReq(&createTbReq); + } else { + if (mr.me.type != TSDB_CHILD_TABLE) { + tqError("vgId:%d, failed to write into %s, since table type incorrect, type %d", TD_VID(pVnode), ctbName, + mr.me.type); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; + } + if (mr.me.ctbEntry.suid != suid) { + tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid: %ld, actual suid %ld", + TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; + } + + createTb = false; + uid = mr.me.uid; + metaReaderClear(&mr); + taosMemoryFreeClear(ctbName); } - tEncoderClear(&encoder); - tdDestroySVCreateTbReq(&createTbReq); int32_t cap = sizeof(SSubmitReq); @@ -445,9 +477,11 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { memcpy(blkSchema, schemaStr, schemaLen); blkHead->schemaLen = htonl(schemaLen); rowData = POINTER_SHIFT(blkSchema, schemaLen); + } else { + blkHead->uid = uid; } - taosMemoryFree(schemaStr); + taosMemoryFreeClear(schemaStr); for (int32_t j = 0; j < rows; j++) { SRowBuilder rb = {0}; From 0029ddd232c2ebb338996b10538c3ecdb7a0f9be Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 17 Oct 2022 17:48:30 +0800 Subject: [PATCH 063/135] feat:add new interface for schemaless --- include/client/taos.h | 1 + source/client/src/clientSml.c | 220 ++++++++++++++++++++++------------ 2 files changed, 145 insertions(+), 76 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 270b647a77..1182a9c2f7 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -198,6 +198,7 @@ DLL_EXPORT const void *taos_get_raw_block(TAOS_RES *res); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); +DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw(TAOS* taos, char* lines, int len, int32_t *totalRows, int protocol, int precision); /* --------------------------TMQ INTERFACE------------------------------- */ diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 24da5f7b70..a7b0aff42d 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -28,8 +28,8 @@ #define QUOTE '"' #define SLASH '\\' -#define JUMP_SPACE(sql) \ - while (*sql != '\0') { \ +#define JUMP_SPACE(sql, sqlEnd) \ + while (sql < sqlEnd) { \ if (*sql == SPACE) \ sql++; \ else \ @@ -915,16 +915,17 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return TSDB_CODE_TSC_INVALID_VALUE; } -static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSmlMsgBuf *msg) { +static int32_t smlParseInfluxString(const char *sql, const char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { if (!sql) return TSDB_CODE_SML_INVALID_DATA; - JUMP_SPACE(sql) + JUMP_SPACE(sql, sqlEnd) if (*sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; elements->measure = sql; // parse measure - while (*sql != '\0') { + while (sql < sqlEnd) { if ((sql != elements->measure) && IS_SLASH_LETTER(sql)) { - MOVE_FORWARD_ONE(sql, strlen(sql) + 1); + MOVE_FORWARD_ONE(sql, sqlEnd - sql); + sqlEnd--; continue; } if (IS_COMMA(sql)) { @@ -948,7 +949,7 @@ static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSm } else { if (*sql == COMMA) sql++; elements->tags = sql; - while (*sql != '\0') { + while (sql < sqlEnd) { if (IS_SPACE(sql)) { break; } @@ -959,10 +960,10 @@ static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSm elements->measureTagsLen = sql - elements->measure; // parse cols - JUMP_SPACE(sql) + JUMP_SPACE(sql, sqlEnd) elements->cols = sql; bool isInQuote = false; - while (*sql != '\0') { + while (sql < sqlEnd) { if (IS_QUOTE(sql)) { isInQuote = !isInQuote; } @@ -982,10 +983,10 @@ static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSm } // parse timestamp - JUMP_SPACE(sql) + JUMP_SPACE(sql, sqlEnd) elements->timestamp = sql; - while (*sql != '\0') { - if (*sql == SPACE) { + while (sql < sqlEnd) { + if (isspace(*sql)) { break; } sql++; @@ -995,8 +996,8 @@ static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSm return TSDB_CODE_SUCCESS; } -static void smlParseTelnetElement(const char **sql, const char **data, int32_t *len) { - while (**sql != '\0') { +static void smlParseTelnetElement(const char **sql, const char *sqlEnd, const char **data, int32_t *len) { + while (*sql < sqlEnd) { if (**sql != SPACE && !(*data)) { *data = *sql; } else if (**sql == SPACE && *data) { @@ -1007,19 +1008,19 @@ static void smlParseTelnetElement(const char **sql, const char **data, int32_t * } } -static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTableName, SHashObj *dumplicateKey, +static int32_t smlParseTelnetTags(const char *data, const char *sqlEnd, SArray *cols, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { const char *sql = data; size_t childTableNameLen = strlen(tsSmlChildTableName); - while (*sql != '\0') { - JUMP_SPACE(sql) + while (sql < sqlEnd) { + JUMP_SPACE(sql, sqlEnd) if (*sql == '\0') break; const char *key = sql; int32_t keyLen = 0; // parse key - while (*sql != '\0') { + while (sql < sqlEnd) { if (*sql == SPACE) { smlBuildInvalidDataMsg(msg, "invalid data", sql); return TSDB_CODE_SML_INVALID_DATA; @@ -1044,7 +1045,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab // parse value const char *value = sql; int32_t valueLen = 0; - while (*sql != '\0') { + while (sql < sqlEnd) { // parse value if (*sql == SPACE) { break; @@ -1089,11 +1090,11 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab } // format: =[ =] -static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTableInfo *tinfo, SArray *cols) { +static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, const char *sqlEnd, SSmlTableInfo *tinfo, SArray *cols) { if (!sql) return TSDB_CODE_SML_INVALID_DATA; // parse metric - smlParseTelnetElement(&sql, &tinfo->sTableName, &tinfo->sTableNameLen); + smlParseTelnetElement(&sql, sqlEnd, &tinfo->sTableName, &tinfo->sTableNameLen); if (!(tinfo->sTableName) || IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; @@ -1102,7 +1103,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTable // parse timestamp const char *timestamp = NULL; int32_t tLen = 0; - smlParseTelnetElement(&sql, ×tamp, &tLen); + smlParseTelnetElement(&sql, sqlEnd, ×tamp, &tLen); if (!timestamp || tLen == 0) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); return TSDB_CODE_SML_INVALID_DATA; @@ -1117,7 +1118,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTable // parse value const char *value = NULL; int32_t valueLen = 0; - smlParseTelnetElement(&sql, &value, &valueLen); + smlParseTelnetElement(&sql, sqlEnd, &value, &valueLen); if (!value || valueLen == 0) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); return TSDB_CODE_TSC_INVALID_VALUE; @@ -1135,7 +1136,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTable } // parse tags - ret = smlParseTelnetTags(sql, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); + ret = smlParseTelnetTags(sql, sqlEnd, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); return ret; @@ -2061,11 +2062,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo * } /************* TSDB_SML_JSON_PROTOCOL function end **************/ -static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { +static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { SSmlLineInfo elements = {0}; uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql); - int ret = smlParseInfluxString(sql, &elements, &info->msgBuf); + int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); return ret; @@ -2170,7 +2171,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseTelnetLine(SSmlHandle *info, void *data) { +static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { int ret = TSDB_CODE_SUCCESS; SSmlTableInfo *tinfo = smlBuildTableInfo(); if (!tinfo) { @@ -2184,7 +2185,7 @@ static int32_t smlParseTelnetLine(SSmlHandle *info, void *data) { } if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ret = smlParseTelnetString(info, (const char *)data, tinfo, cols); + ret = smlParseTelnetString(info, (const char *)data, data + len, tinfo, cols); } else if (info->protocol == TSDB_SML_JSON_PROTOCOL) { ret = smlParseJSONString(info, (cJSON *)data, tinfo, cols); } else { @@ -2275,7 +2276,7 @@ static int32_t smlParseJSON(SSmlHandle *info, char *payload) { for (int32_t i = 0; i < payloadNum; ++i) { cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(root)) ? root : cJSON_GetArrayItem(root, i); - ret = smlParseTelnetLine(info, dataPoint); + ret = smlParseTelnetLine(info, dataPoint, -1); if (ret != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); goto end; @@ -2364,10 +2365,14 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); } -static int32_t smlParseLine(SSmlHandle *info, char *lines[], int numLines) { +static int32_t smlParseLine(SSmlHandle *info, char *lines[], char* rawLine, char* rawLineEnd, int numLines) { int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { - code = smlParseJSON(info, *lines); + if(lines){ + code = smlParseJSON(info, *lines); + }else if(rawLine){ + code = smlParseJSON(info, rawLine); + } if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseJSON failed:%s", info->id, *lines); return code; @@ -2376,10 +2381,25 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], int numLines) { } for (int32_t i = 0; i < numLines; ++i) { + char *tmp = NULL; + int len = 0; + if(lines){ + tmp = lines[i]; + len = strlen(tmp); + }else if(rawLine){ + tmp = rawLine; + while(rawLine < rawLineEnd){ + if(*(rawLine++) == '\n'){ + break; + } + len++; + } + } + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - code = smlParseInfluxLine(info, lines[i]); + code = smlParseInfluxLine(info, tmp, len); } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - code = smlParseTelnetLine(info, lines[i]); + code = smlParseTelnetLine(info, tmp, len); } else { ASSERT(0); } @@ -2391,13 +2411,13 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], int numLines) { return code; } -static int smlProcess(SSmlHandle *info, char *lines[], int numLines) { +static int smlProcess(SSmlHandle *info, char *lines[], char* rawLine, char* rawLineEnd, int numLines) { int32_t code = TSDB_CODE_SUCCESS; int32_t retryNum = 0; info->cost.parseTime = taosGetTimestampUs(); - code = smlParseLine(info, lines, numLines); + code = smlParseLine(info, lines, rawLine, rawLineEnd, numLines); if (code != 0) { uError("SML:0x%" PRIx64 " smlParseLine error : %s", info->id, tstrerror(code)); return code; @@ -2490,39 +2510,8 @@ static void smlInsertCallback(void *param, void *res, int32_t code) { smlDestroyInfo(info); } -/** - * taos_schemaless_insert() parse and insert data points into database according to - * different protocol. - * - * @param $lines input array may contain multiple lines, each line indicates a data point. - * If protocol=2 is used input array should contain single JSON - * string(e.g. char *lines[] = {"$JSON_string"}). If need to insert - * multiple data points in JSON format, should include them in $JSON_string - * as a JSON array. - * @param $numLines indicates how many data points in $lines. - * If protocol = 2 is used this param will be ignored as $lines should - * contain single JSON string. - * @param $protocol indicates which protocol to use for parsing: - * 0 - influxDB line protocol - * 1 - OpenTSDB telnet line protocol - * 2 - OpenTSDB JSON format protocol - * @return return zero for successful insertion. Otherwise return none-zero error code of - * failure reason. - * - */ - -TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } +TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd, int numLines, int protocol, int precision) { int batchs = 0; STscObj *pTscObj = request->pTscObj; @@ -2546,12 +2535,6 @@ TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int pr goto end; } - if (!lines) { - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - goto end; - } - if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); @@ -2602,15 +2585,28 @@ TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int pr info->affectedRows = perBatch; info->pRequest->body.queryFp = smlInsertCallback; info->pRequest->body.param = info; - int32_t code = smlProcess(info, lines, perBatch); - lines += perBatch; + int32_t code = smlProcess(info, lines, rawLine, rawLineEnd, perBatch); + if(lines){ + lines += perBatch; + } + if(rawLine){ + int num = 0; + while(rawLine < rawLineEnd){ + if(*(rawLine++) == '\n'){ + num++; + } + if(num == perBatch){ + break; + } + } + } if (code != TSDB_CODE_SUCCESS) { info->pRequest->body.queryFp(info, req, code); } } tsem_wait(¶ms.sem); -end: + end: taosThreadSpinDestroy(¶ms.lock); tsem_destroy(¶ms.sem); // ((STscObj *)taos)->schemalessType = 0; @@ -2618,3 +2614,75 @@ end: uDebug("resultend:%s", request->msgBuf); return (TAOS_RES *)request; } + +/** + * taos_schemaless_insert() parse and insert data points into database according to + * different protocol. + * + * @param $lines input array may contain multiple lines, each line indicates a data point. + * If protocol=2 is used input array should contain single JSON + * string(e.g. char *lines[] = {"$JSON_string"}). If need to insert + * multiple data points in JSON format, should include them in $JSON_string + * as a JSON array. + * @param $numLines indicates how many data points in $lines. + * If protocol = 2 is used this param will be ignored as $lines should + * contain single JSON string. + * @param $protocol indicates which protocol to use for parsing: + * 0 - influxDB line protocol + * 1 - OpenTSDB telnet line protocol + * 2 - OpenTSDB JSON format protocol + * @return return zero for successful insertion. Otherwise return none-zero error code of + * failure reason. + * + */ + +TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT); + if (!request) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; + } + + if (!lines || numLines <= 0) { + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + request->code = TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&msg, "lines is null", NULL); + return (TAOS_RES *)request; + } + + return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision); +} + +TAOS_RES *taos_schemaless_insert_raw(TAOS* taos, char* lines, int len, int32_t *totalRows, int protocol, int precision){ + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT); + if (!request) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; + } + + if (!lines || len <= 0) { + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + request->code = TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&msg, "lines is null", NULL); + return (TAOS_RES *)request; + } + + int numLines = 0; + for(int i = 0; i < len; i++){ + if(lines[i] == '\n' || i == len - 1){ + numLines++; + } + } + *totalRows = numLines; + return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision); +} From 89800c9d54c0efdaa642c3332f6c8102b9d0b890 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Oct 2022 18:26:16 +0800 Subject: [PATCH 064/135] chore: taostools sanitizer flag align with tdengine (#17407) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index f0a1ad83ad..73409a950a 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 2849aa4 + GIT_TAG 4d02980 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 16fb3525aad0e15a4711fb02188d96042e90ece8 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 065/135] fix: coverity issues CID: 399527 --- source/common/src/tvariant.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index d0f4855549..9660a37f48 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -29,7 +29,7 @@ break; \ } \ if ((res) > (maxv)) { \ - *exti = -1; \ + *exti = 1; \ break; \ } \ assert(0); \ From 5d5c0b876c650fe038cebf36cbbd1bee10d9141a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 17 Oct 2022 18:42:07 +0800 Subject: [PATCH 066/135] fix: coverity scan of deadcode when print submit msg --- source/dnode/vnode/src/vnd/vnodeSvr.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 3556b47dd3..d0a00a8b6d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -802,15 +802,11 @@ static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, tInitSubmitBlkIter(msgIter, pBlock, &blkIter); if (blkIter.row == NULL) return 0; - if (!pSchema || (suid != msgIter->suid) || rv != TD_ROW_SVER(blkIter.row)) { - if (pSchema) { - taosMemoryFreeClear(pSchema); - } - pSchema = metaGetTbTSchema(pMeta, msgIter->suid, TD_ROW_SVER(blkIter.row), 1); // TODO: use the real schema - if (pSchema) { - suid = msgIter->suid; - rv = TD_ROW_SVER(blkIter.row); - } + + pSchema = metaGetTbTSchema(pMeta, msgIter->suid, TD_ROW_SVER(blkIter.row), 1); // TODO: use the real schema + if (pSchema) { + suid = msgIter->suid; + rv = TD_ROW_SVER(blkIter.row); } if (!pSchema) { printf("%s:%d no valid schema\n", tags, __LINE__); From 34f576b681357dc5a66bbc1096bef108968c8128 Mon Sep 17 00:00:00 2001 From: Zhiqiang Wang <1296468573@qq.com> Date: Mon, 17 Oct 2022 19:16:11 +0800 Subject: [PATCH 067/135] fix: show help install error (#17408) --- packaging/tools/make_install.sh | 83 ++++++++++++++++---------------- tools/shell/src/shellArguments.c | 2 +- 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index b56a5a203d..d0e94e3eb3 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -168,7 +168,7 @@ function install_bin() { ${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || : - [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : + [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : @@ -182,21 +182,21 @@ function install_bin() { ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : - [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : - [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || : - [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : - [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : - [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : - [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || : - [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : - [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor > /dev/null 2>&1 || : + [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} > /dev/null 2>&1 || : else ${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || : - [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : + [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : @@ -206,14 +206,14 @@ function install_bin() { ${csudo}cp -r ${script_dir}/remove.sh ${install_main_dir}/bin || : ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : - [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : - [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || : - [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : - [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : - [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : - [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || : + [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} > /dev/null 2>&1 || : fi } @@ -238,7 +238,7 @@ function install_jemalloc() { if [ -f "${binary_dir}/build/lib/libjemalloc.so.2" ]; then ${csudo}/usr/bin/install -c -d /usr/local/lib ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.so.2 /usr/local/lib - ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so > /dev/null 2>&1 ${csudo}/usr/bin/install -c -d /usr/local/lib [ -f ${binary_dir}/build/lib/libjemalloc.a ] && ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.a /usr/local/lib @@ -274,8 +274,8 @@ function install_avro() { if [ -f "${binary_dir}/build/$1/libavro.so.23.0.0" ] && [ -d /usr/local/$1 ]; then ${csudo}/usr/bin/install -c -d /usr/local/$1 ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.so.23.0.0 /usr/local/$1 - ${csudo}ln -sf libavro.so.23.0.0 /usr/local/$1/libavro.so.23 - ${csudo}ln -sf libavro.so.23 /usr/local/$1/libavro.so + ${csudo}ln -sf libavro.so.23.0.0 /usr/local/$1/libavro.so.23 > /dev/null 2>&1 + ${csudo}ln -sf libavro.so.23 /usr/local/$1/libavro.so > /dev/null 2>&1 ${csudo}/usr/bin/install -c -d /usr/local/$1 [ -f ${binary_dir}/build/$1/libavro.a ] && ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.a /usr/local/$1 @@ -304,11 +304,11 @@ function install_lib() { ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/libtaos.so.${verNumber} - ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 - ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 > /dev/null 2>&1 + ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so > /dev/null 2>&1 if [ -d "${lib64_link_dir}" ]; then - ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 - ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so + ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 > /dev/null 2>&1 + ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so > /dev/null 2>&1 fi if [ -f ${binary_dir}/build/lib/libtaosws.so ]; then @@ -316,23 +316,23 @@ function install_lib() { ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so ||: - ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : + ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so > /dev/null 2>&1 || : fi else ${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \ ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* ${csudo}ln -sf ${install_main_dir}/driver/libtaos.${verNumber}.dylib \ - ${lib_link_dir}/libtaos.1.dylib || : + ${lib_link_dir}/libtaos.1.dylib > /dev/null 2>&1 || : - ${csudo}ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib || : + ${csudo}ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib > /dev/null 2>&1 || : if [ -f ${binary_dir}/build/lib/libtaosws.dylib ]; then ${csudo}cp ${binary_dir}/build/lib/libtaosws.dylib \ ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.dylib ||: - ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib || : + ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib > /dev/null 2>&1 || : fi fi @@ -346,6 +346,7 @@ function install_lib() { } function install_header() { + ${csudo}mkdir -p ${inc_link_dir} ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h ||: ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ @@ -353,13 +354,13 @@ function install_header() { if [ -f ${binary_dir}/build/include/taosws.h ]; then ${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h ||: - ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h ||: + ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h > /dev/null 2>&1 ||: fi - ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h - ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h - ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h - ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h + ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h > /dev/null 2>&1 + ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h > /dev/null 2>&1 + ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h > /dev/null 2>&1 + ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h > /dev/null 2>&1 ${csudo}chmod 644 ${install_main_dir}/include/* } @@ -374,7 +375,7 @@ function install_config() { ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ ${cfg_install_dir}/${configFile}.${verNumber} ${csudo}ln -s ${cfg_install_dir}/${configFile} \ - ${install_main_dir}/cfg/${configFile} + ${install_main_dir}/cfg/${configFile} > /dev/null 2>&1 else ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ ${cfg_install_dir}/${configFile}.${verNumber} @@ -395,7 +396,7 @@ function install_taosadapter_config() { ${cfg_install_dir}/taosadapter.toml.${verNumber} || : [ -f ${cfg_install_dir}/taosadapter.toml ] && ${csudo}ln -s ${cfg_install_dir}/taosadapter.toml \ - ${install_main_dir}/cfg/taosadapter.toml || : + ${install_main_dir}/cfg/taosadapter.toml > /dev/null 2>&1 || : else if [ -f "${binary_dir}/test/cfg/taosadapter.toml" ]; then ${csudo}cp -f ${binary_dir}/test/cfg/taosadapter.toml \ @@ -408,12 +409,12 @@ function install_taosadapter_config() { function install_log() { ${csudo}rm -rf ${log_dir} || : ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} - ${csudo}ln -s ${log_dir} ${install_main_dir}/log + ${csudo}ln -s ${log_dir} ${install_main_dir}/log > /dev/null 2>&1 } function install_data() { ${csudo}mkdir -p ${data_dir} && ${csudo}chmod 777 ${data_dir} - ${csudo}ln -s ${data_dir} ${install_main_dir}/data + ${csudo}ln -s ${data_dir} ${install_main_dir}/data > /dev/null 2>&1 } function install_connector() { @@ -533,7 +534,7 @@ function install_taosadapter_service() { function install_service_on_launchctl() { ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : ${csudo}cp ${script_dir}/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist - ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist || : + ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : } function install_service() { diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index eedb6d7295..4d40de66bd 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -22,7 +22,7 @@ #define TAOS_CONSOLE_PROMPT_HEADER "taos> " #define TAOS_CONSOLE_PROMPT_CONTINUE " -> " -#define SHELL_HOST "The auth string to use when connecting to the server." +#define SHELL_HOST "TDengine server FQDN to connect. The default host is localhost." #define SHELL_PORT "The TCP/IP port number to use for the connection." #define SHELL_USER "The user name to use when connecting to the server." #define SHELL_PASSWORD "The password to use when connecting to the server." From b7222bc4ead1237d3f75eb7406d7baae272de7f0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 17 Oct 2022 19:21:11 +0800 Subject: [PATCH 068/135] feat:add new interface for schemaless --- source/client/src/clientSml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index a7b0aff42d..82f543f0dd 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2648,7 +2648,7 @@ TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int pr return NULL; } - if (!lines || numLines <= 0) { + if (!lines) { SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; request->code = TSDB_CODE_SML_INVALID_DATA; smlBuildInvalidDataMsg(&msg, "lines is null", NULL); From c73535493d1d012ad8d5c8fd03f4f0106666a184 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 17 Oct 2022 19:32:13 +0800 Subject: [PATCH 069/135] test: add insert rows --- tests/system-test/7-tmq/tmqShow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index c0f33d9204..0691da6786 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -37,7 +37,7 @@ class TDTestCase: 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], 'ctbPrefix': 'ctb', - 'ctbNum': 10, + 'ctbNum': 100, 'rowsPerTbl': 4000, 'batchNum': 15, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 @@ -124,7 +124,8 @@ class TDTestCase: tdLog.info("async insert data") pThread = tmqCom.asyncInsertData(paraDict) - time.sleep(5) + tmqCom.getStartConsumeNotifyFromTmqsim(); + #time.sleep(5) tdLog.info("check show consumers") tdSql.query("show consumers") # tdLog.info(tdSql.queryResult) From a0cd578668792352695545797a3ca4ebbca3b123 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 17 Oct 2022 19:37:34 +0800 Subject: [PATCH 070/135] feat:add new interface for schemaless --- source/client/src/clientSml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 55d117a3c8..fe1946d77d 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2199,7 +2199,7 @@ static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { } if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ret = smlParseTelnetString(info, (const char *)data, data + len, tinfo, cols); + ret = smlParseTelnetString(info, (const char *)data, POINTER_SHIFT(data, len), tinfo, cols); } else if (info->protocol == TSDB_SML_JSON_PROTOCOL) { ret = smlParseJSONString(info, (cJSON *)data, tinfo, cols); } else { From 57cdf726b38c90c2b3fa72e217da5b18440ac010 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 17 Oct 2022 19:46:23 +0800 Subject: [PATCH 071/135] fix(shell): fixed tire tree free match error --- tools/shell/src/shellTire.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tools/shell/src/shellTire.c b/tools/shell/src/shellTire.c index b310281119..346757b76f 100644 --- a/tools/shell/src/shellTire.c +++ b/tools/shell/src/shellTire.c @@ -275,7 +275,6 @@ void matchPrefixFromList(STire* tire, char* prefix, SMatch* match) { // match prefix words, if match is not NULL , put all item to match and return match void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) { - SMatch* root = match; int m = 0; STireNode* c = 0; int len = strlen(prefix); @@ -299,23 +298,14 @@ void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) { // previous items already matched if (i == len - 1) { - // malloc match if not pass by param match - if (root == NULL) { - root = (SMatch*)taosMemoryMalloc(sizeof(SMatch)); - memset(root, 0, sizeof(SMatch)); - strcpy(root->pre, prefix); - } - // prefix is match to end char - if (c->d) enumAllWords(c->d, prefix, root); + if (c->d) enumAllWords(c->d, prefix, match); } else { // move to next node continue match if (c->d == NULL) break; nodes = c->d; } } - - taosMemoryFree(root); } SMatch* matchPrefix(STire* tire, char* prefix, SMatch* match) { From 77c4c8fd88e5b40b95b7a2e3597ef2d8e6be3206 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 17:44:41 +0800 Subject: [PATCH 072/135] test: fix failed case --- source/dnode/vnode/src/tq/tqSink.c | 6 +++++- tests/system-test/7-tmq/tmqAutoCreateTbl.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 366cd6268e..9bdcf04e89 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -352,6 +352,7 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { SMetaReader mr = {0}; metaReaderInit(&mr, pVnode->pMeta, 0); if (metaGetTableEntryByName(&mr, ctbName) < 0) { + metaReaderClear(&mr); tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); SVCreateTbReq createTbReq = {0}; @@ -444,6 +445,9 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { createTb = false; uid = mr.me.uid; metaReaderClear(&mr); + + tqDebug("vgId:%d, stream write, table %s, uid %ld already exist, skip create", TD_VID(pVnode), ctbName, uid); + taosMemoryFreeClear(ctbName); } @@ -478,7 +482,7 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { blkHead->schemaLen = htonl(schemaLen); rowData = POINTER_SHIFT(blkSchema, schemaLen); } else { - blkHead->uid = uid; + blkHead->uid = htobe64(uid); } taosMemoryFreeClear(schemaStr); diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index de95fef148..587472ed82 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -42,7 +42,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 500, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 3, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -87,7 +87,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 500, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} From cf9abe7258a4b50d8f037fb5e5fa9de1280e0f09 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 17 Oct 2022 19:48:36 +0800 Subject: [PATCH 073/135] fix: fix coverity check issues --- include/util/tdef.h | 3 +++ source/client/src/clientHb.c | 5 +++- source/dnode/mnode/impl/inc/mndQuery.h | 2 ++ source/dnode/mnode/impl/src/mndProfile.c | 2 +- source/dnode/mnode/impl/src/mndQuery.c | 7 +++++ source/dnode/vnode/src/vnd/vnodeQuery.c | 6 +++++ source/libs/catalog/inc/ctgRemote.h | 2 ++ source/libs/catalog/src/catalog.c | 16 ++++++++---- source/libs/catalog/src/ctgAsync.c | 15 +++-------- source/libs/catalog/src/ctgCache.c | 33 ++++++++++++------------ source/libs/catalog/src/ctgRemote.c | 6 +++++ source/libs/command/inc/commandInt.h | 4 +-- source/libs/command/src/command.c | 1 + source/libs/command/src/explain.c | 1 + source/libs/executor/src/dataDeleter.c | 2 ++ source/libs/executor/src/dataInserter.c | 1 + source/libs/qcom/src/queryUtil.c | 6 +++++ source/libs/qcom/src/querymsg.c | 6 +++-- source/libs/qworker/src/qworker.c | 5 ++-- source/libs/scheduler/src/schJob.c | 1 - source/libs/scheduler/src/schRemote.c | 8 +++--- source/libs/scheduler/src/schTask.c | 24 +++++++++++++---- source/libs/scheduler/src/schUtil.c | 2 +- 23 files changed, 103 insertions(+), 55 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index e6d28ae341..d9bb558b74 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -239,6 +239,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 +#define TSDB_MAX_COL_TAG_NUM (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS) #define TSDB_MAX_JSON_TAG_LEN 16384 #define TSDB_MAX_JSON_KEY_LEN 256 @@ -496,6 +497,8 @@ enum { #define MAX_NUM_STR_SIZE 40 +#define MAX_META_MSG_IN_BATCH 1048576 + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 096e2cdac8..aa725a4988 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -173,7 +173,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; - tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, + tscTrace("conn %d hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, pTscObj->pAppInfo->totalDnodes); if (pRsp->query->killRid) { @@ -440,6 +440,7 @@ int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S } if (userNum <= 0) { + taosMemoryFree(users); return TSDB_CODE_SUCCESS; } @@ -476,6 +477,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl } if (dbNum <= 0) { + taosMemoryFree(dbs); return TSDB_CODE_SUCCESS; } @@ -514,6 +516,7 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC } if (stbNum <= 0) { + taosMemoryFree(stbs); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/mnode/impl/inc/mndQuery.h b/source/dnode/mnode/impl/inc/mndQuery.h index 4b359c0a14..69ec54f330 100644 --- a/source/dnode/mnode/impl/inc/mndQuery.h +++ b/source/dnode/mnode/impl/inc/mndQuery.h @@ -22,6 +22,8 @@ extern "C" { #endif +#define MAX_META_MSG_IN_BATCH 1048576 + int32_t mndInitQuery(SMnode *pMnode); void mndCleanupQuery(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 1909b168bd..d42b66455f 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -369,7 +369,7 @@ static void mndReleaseApp(SMnode *pMnode, SAppObj *pApp) { taosCacheRelease(pMgmt->appCache, (void **)&pApp, false); } -void *mndGetNextApp(SMnode *pMnode, SCacheIter *pIter) { +SAppObj *mndGetNextApp(SMnode *pMnode, SCacheIter *pIter) { SAppObj *pApp = NULL; bool hasNext = taosCacheIterNext(pIter); if (hasNext) { diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index f185cdeaec..2e01fadbae 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -77,6 +77,12 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { void *pRsp = NULL; SMnode *pMnode = pMsg->info.node; + if (msgNum >= MAX_META_MSG_IN_BATCH) { + code = TSDB_CODE_INVALID_MSG; + mError("too many msgs %d in mnode batch meta req", msgNum); + goto _exit; + } + SArray *batchRsp = taosArrayInit(msgNum, sizeof(SBatchRsp)); if (NULL == batchRsp) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -106,6 +112,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { if (fp == NULL) { mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index e18379fe8a..8c1f858cbb 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -272,6 +272,12 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { SRpcMsg rspMsg = {0}; void *pRsp = NULL; + if (msgNum >= MAX_META_MSG_IN_BATCH) { + code = TSDB_CODE_INVALID_MSG; + qError("too many msgs %d in vnode batch meta req", msgNum); + goto _exit; + } + SArray *batchRsp = taosArrayInit(msgNum, sizeof(SBatchRsp)); if (NULL == batchRsp) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/catalog/inc/ctgRemote.h b/source/libs/catalog/inc/ctgRemote.h index 8edd3b3880..746344b7ed 100644 --- a/source/libs/catalog/inc/ctgRemote.h +++ b/source/libs/catalog/inc/ctgRemote.h @@ -20,6 +20,8 @@ extern "C" { #endif +#define CTG_MAX_REQ_IN_BATCH 1048576 + #ifdef __cplusplus } #endif diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b027dbcfa5..b960103d94 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -173,15 +173,19 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput)); } - CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, output, syncReq)); + code = ctgUpdateTbMetaEnqueue(pCtg, output, syncReq); + output = NULL; + CTG_ERR_JRET(code); return TSDB_CODE_SUCCESS; _return: - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); - + if (output) { + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); + } + CTG_RET(code); } @@ -290,7 +294,9 @@ int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp* rspMsg, bool syncOp) { CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, rspMsg->tableType == TSDB_SUPER_TABLE, &output->tbMeta)); } - CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, output, syncOp)); + code = ctgUpdateTbMetaEnqueue(pCtg, output, syncOp); + output = NULL; + CTG_ERR_JRET(code); return TSDB_CODE_SUCCESS; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 1305540a50..1d2e3640a1 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -71,7 +71,7 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", pJob->queryId, taskIdx, - ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); + ctgTaskTypeStr(task.type), (int32_t)taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); return TSDB_CODE_SUCCESS; } @@ -202,7 +202,7 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", pJob->queryId, taskIdx, - ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); + ctgTaskTypeStr(task.type), (int32_t)taosArrayGetSize(ctx->pNames), pJob->tbHashNum); return TSDB_CODE_SUCCESS; } @@ -1056,7 +1056,6 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; @@ -1223,7 +1222,6 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; @@ -1309,7 +1307,6 @@ int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } _return: @@ -1346,7 +1343,6 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } _return: @@ -1382,7 +1378,6 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { @@ -1672,7 +1667,7 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { int32_t baseResIdx = 0; for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); - ctgDebug("start to check tb metas in db %s, tbNum %d", pReq->dbFName, taosArrayGetSize(pReq->pTables)); + ctgDebug("start to check tb metas in db %s, tbNum %d", pReq->dbFName, (int32_t)taosArrayGetSize(pReq->pTables)); CTG_ERR_RET(ctgGetTbMetasFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); } @@ -2010,10 +2005,6 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask* pTask) { _return: - if (dbCache) { - ctgReleaseVgInfoToCache(pCtg, dbCache); - } - CTG_RET(code); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 3a774afdcc..89571ebf55 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -597,6 +597,8 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + taosMemoryFree(operation->data); + taosMemoryFree(operation); CTG_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -648,6 +650,7 @@ int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) SCtgDropDBMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDBMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDBMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -668,7 +671,6 @@ int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -681,6 +683,7 @@ int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) SCtgDropDbVgroupMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDbVgroupMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDbVgroupMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -700,7 +703,6 @@ int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -714,6 +716,7 @@ int32_t ctgDropStbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SCtgDropStbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropStbMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropStbMetaMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -731,7 +734,6 @@ int32_t ctgDropStbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -744,6 +746,7 @@ int32_t ctgDropTbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SCtgDropTblMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTblMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTblMetaMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -760,7 +763,6 @@ int32_t ctgDropTbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -773,6 +775,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); + taosMemoryFree(op); ctgFreeVgInfo(dbInfo); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -796,8 +799,6 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId _return: ctgFreeVgInfo(dbInfo); - taosMemoryFreeClear(op->data); - taosMemoryFreeClear(op); CTG_RET(code); } @@ -810,6 +811,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool sy SCtgUpdateTbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbMetaMsg)); + taosMemoryFree(op); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -834,8 +836,6 @@ _return: taosMemoryFree(output); } - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -847,6 +847,7 @@ int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEp SCtgUpdateEpsetMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateEpsetMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateEpsetMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -863,8 +864,6 @@ int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEp _return: - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -877,6 +876,7 @@ int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -892,7 +892,6 @@ int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp _return: tFreeSGetUserAuthRsp(pAuth); - taosMemoryFreeClear(msg); CTG_RET(code); } @@ -906,6 +905,7 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog *pCtg, STableIndex **pIndex, bool syncO SCtgUpdateTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbIndexMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbIndexMsg)); + taosMemoryFree(op); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -923,7 +923,6 @@ _return: taosArrayDestroyEx((*pIndex)->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(*pIndex); - taosMemoryFreeClear(msg); CTG_RET(code); } @@ -937,6 +936,7 @@ int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { SCtgDropTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTbIndexMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbIndexMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -952,8 +952,6 @@ int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { _return: - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -968,6 +966,7 @@ int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool freeCtg, bool stopQueue, bool SCtgClearCacheMsg *msg = taosMemoryMalloc(sizeof(SCtgClearCacheMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgClearCacheMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -981,8 +980,6 @@ int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool freeCtg, bool stopQueue, bool _return: - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -1483,7 +1480,9 @@ int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool sync int32_t code = 0; CTG_ERR_RET(ctgCloneMetaOutput(pOut, &pOutput)); - CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, pOutput, syncReq)); + code = ctgUpdateTbMetaEnqueue(pCtg, pOutput, syncReq); + pOutput = NULL; + CTG_ERR_JRET(code); return TSDB_CODE_SUCCESS; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 744d6eab06..23bccb0835 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -19,6 +19,7 @@ #include "tname.h" #include "tref.h" #include "trpc.h" +#include "ctgRemote.h" int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; @@ -579,6 +580,11 @@ int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg) { int32_t num = taosArrayGetSize(pBatch->pMsgs); SBatchReq* pBatchReq = (SBatchReq*)(*msg); + if (num >= CTG_MAX_REQ_IN_BATCH) { + qError("too many msgs %d in one batch request", num); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + pBatchReq->header.vgId = htonl(vgId); pBatchReq->msgNum = htonl(num); offset += sizeof(SBatchReq); diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index ad677a81ff..ce4c4cf09b 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -89,8 +89,8 @@ extern "C" { #define EXPLAIN_STRING_TYPE_FORMAT "%s" #define EXPLAIN_INPUT_ORDER_FORMAT "input_order=%s" #define EXPLAIN_OUTPUT_ORDER_TYPE_FORMAT "output_order=%s" -#define EXPLAIN_OFFSET_FORMAT "offset=%d" -#define EXPLAIN_SOFFSET_FORMAT "soffset=%d" +#define EXPLAIN_OFFSET_FORMAT "offset=%" PRId64 +#define EXPLAIN_SOFFSET_FORMAT "soffset=%" PRId64 #define EXPLAIN_PARTITIONS_FORMAT "partitions=%d" #define COMMAND_RESET_LOG "resetLog" diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 18d839e109..a9134de537 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -485,6 +485,7 @@ static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRs SSDataBlock* pBlock = buildCreateTbResultDataBlock(); int32_t code = setCreateTBResultIntoDataBlock(pBlock, pStmt->pDbCfg, pStmt->tableName, pStmt->pTableCfg); if (code) { + blockDataDestroy(pBlock); return code; } return buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp); diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index c5de44e1f6..e0a4218243 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1598,6 +1598,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); + blockDataDestroy(pBlock); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 1ade4b89e6..55978855d1 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -267,6 +267,8 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData taosThreadMutexInit(&deleter->mutex, NULL); if (NULL == deleter->pDataBlocks) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + destroyDataSinker((SDataSinkHandle*)deleter); + taosMemoryFree(deleter); return TSDB_CODE_QRY_OUT_OF_MEMORY; } *pHandle = deleter; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 6a43ea40c6..18923ffb67 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -323,6 +323,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat int32_t code = tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); if (code) { + destroyDataSinker((SDataSinkHandle*)pInserterNode); return code; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 8162b922ce..a917397d02 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -424,6 +424,12 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } + if ((pSrc->tableInfo.numOfColumns + pSrc->tableInfo.numOfTags) > TSDB_MAX_COL_TAG_NUM) { + *pDst = NULL; + qError("too many column and tag num:%d,%d", pSrc->tableInfo.numOfColumns, pSrc->tableInfo.numOfTags); + return TSDB_CODE_INVALID_PARA; + } + int32_t metaSize = sizeof(STableMeta) + (pSrc->tableInfo.numOfColumns + pSrc->tableInfo.numOfTags) * sizeof(SSchema); *pDst = taosMemoryMalloc(metaSize); if (NULL == *pDst) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 80ba356e8f..a6f26088de 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -271,8 +271,8 @@ int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t SBuildTableInput *pInput = input; STableCfgReq cfgReq = {0}; cfgReq.header.vgId = pInput->vgId; - strcpy(cfgReq.dbFName, pInput->dbFName); - strcpy(cfgReq.tbName, pInput->tbName); + strncpy(cfgReq.dbFName, pInput->dbFName, sizeof(cfgReq.dbFName)); + strncpy(cfgReq.tbName, pInput->tbName, sizeof(cfgReq.tbName)); int32_t bufLen = tSerializeSTableCfgReq(NULL, 0, &cfgReq); void *pBuf = (*mallcFp)(bufLen); @@ -615,6 +615,8 @@ int32_t queryProcessGetTbCfgRsp(void *output, char *msg, int32_t msgSize) { STableCfgRsp *out = taosMemoryCalloc(1, sizeof(STableCfgRsp)); if (tDeserializeSTableCfgRsp(msg, msgSize, out) != 0) { qError("tDeserializeSTableCfgRsp failed, msgSize:%d", msgSize); + tFreeSTableCfgRsp(out); + taosMemoryFree(out); return TSDB_CODE_INVALID_MSG; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index d2c16a1cb5..0ff40f3b9a 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -258,7 +258,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); if (len < 0) { - QW_TASK_ELOG("invalid length from dsGetDataLength, length:%d", len); + QW_TASK_ELOG("invalid length from dsGetDataLength, length:%" PRId64, len); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -292,7 +292,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, } // Got data from sink - QW_TASK_DLOG("there are data in sink, dataLength:%d", len); + QW_TASK_DLOG("there are data in sink, dataLength:%" PRId64, len); *dataLen += len; @@ -408,7 +408,6 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu // QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); - break; } QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC)); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index bb8451ba37..8fcfdce2b2 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -122,7 +122,6 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_DROP: SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); - break; default: SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus)); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1b59c06140..c26ae4e646 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -169,7 +169,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) { SCH_LOCK(SCH_WRITE, &pJob->resLock); if (NULL == pJob->execRes.res) { - pJob->execRes.res = taosArrayInit(batchRsp.nRsps, POINTER_BYTES); + pJob->execRes.res = (void*)taosArrayInit(batchRsp.nRsps, POINTER_BYTES); pJob->execRes.msgType = TDMT_VND_CREATE_TABLE; } @@ -386,7 +386,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa // NEVER REACH HERE SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:0x%" PRIx64, pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); - break; } case TDMT_SCH_LINK_BROKEN: SCH_TASK_ELOG("link broken received, error:%x - %s", rspCode, tstrerror(rspCode)); @@ -981,7 +980,7 @@ _return: } int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { - uint32_t msgSize = 0; + int32_t msgSize = 0; void *msg = NULL; int32_t code = 0; bool isCandidateAddr = false; @@ -1133,12 +1132,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, default: SCH_TASK_ELOG("unknown msg type to send, msgType:%d", msgType); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); - break; } #if 1 SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; - code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); + code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); msg = NULL; SCH_ERR_JRET(code); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 33bf50d853..c641b88152 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -904,7 +904,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { } else if (tsQueryPlannerTrace) { char *msg = NULL; int32_t msgLen = 0; - qSubPlanToString(plan, &msg, &msgLen); + SCH_ERR_RET(qSubPlanToString(plan, &msg, &msgLen)); SCH_TASK_DLOGL("physical plan len:%d, %s", msgLen, msg); taosMemoryFree(msg); } @@ -926,6 +926,7 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { } SArray *explainRes = NULL; + int32_t code = 0; SQWMsg qwMsg = {0}; qwMsg.msgInfo.taskType = TASK_TYPE_TEMP; qwMsg.msgInfo.explain = SCH_IS_EXPLAIN_JOB(pJob); @@ -938,14 +939,21 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); } - SCH_ERR_RET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, + SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); + explainRes = NULL; } - SCH_RET(schProcessOnTaskSuccess(pJob, pTask)); + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + +_return: + + taosArrayDestroy(explainRes); + + SCH_RET(code); } int32_t schLaunchTaskImpl(void *param) { @@ -1097,22 +1105,28 @@ int32_t schExecRemoteFetch(SSchJob *pJob, SSchTask *pTask) { int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { void *pRsp = NULL; + int32_t code = 0; SArray *explainRes = NULL; if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); } - SCH_ERR_RET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, + SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); + explainRes = NULL; } SCH_ERR_RET(schProcessFetchRsp(pJob, pTask, pRsp, TSDB_CODE_SUCCESS)); - return TSDB_CODE_SUCCESS; +_return: + + taosArrayDestroy(explainRes); + + SCH_RET(code); } // Note: no more error processing, handled in function internal diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 7099dbb2c0..e20ed9d58e 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -242,7 +242,7 @@ uint64_t schGenUUID(void) { if (hashId == 0) { char uid[64] = {0}; - int32_t code = taosGetSystemUUID(uid, tListLen(uid)); + int32_t code = taosGetSystemUUID(uid, tListLen(uid) - 1); if (code != TSDB_CODE_SUCCESS) { qError("Failed to get the system uid, reason:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } else { From 5ff8bb5bc0b8af92bd0c62b53646b50ac60cb145 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 17 Oct 2022 19:56:59 +0800 Subject: [PATCH 074/135] fix invalid packet --- source/libs/transport/src/transComm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index f843c8559c..4ae1a1083a 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -128,10 +128,10 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) { static const int HEADSIZE = sizeof(STransMsgHead); SConnBuffer* p = connBuf; - if (p->left != 0) { + if (p->left != 0 || p->total <= 0) { return -1; } - int total = connBuf->total; + int total = p->total; if (total >= HEADSIZE && !p->invalid) { *buf = taosMemoryCalloc(1, total); memcpy(*buf, p->buf, total); From 80a8858fe12a25631449ae0752d4d171e0d29255 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 17 Oct 2022 20:10:57 +0800 Subject: [PATCH 075/135] fix invalid packet --- source/dnode/vnode/src/meta/metaTable.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 385e59d152..420c0474fb 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -116,7 +116,6 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const indexMultiTermAdd(terms, term); } } - taosArrayDestroy(pTagVals); indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); indexMultiTermDestroy(terms); From 34c7cda3566ffe675d79fd38072929cee907f054 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 17 Oct 2022 20:25:11 +0800 Subject: [PATCH 076/135] test: change sleep with commit signal --- tests/system-test/7-tmq/subscribeDb4.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index 63a59c0dc3..d253c34a53 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -41,7 +41,7 @@ class TDTestCase: ifcheckdata = 0 ifManualCommit = 1 groupId = 'group.id:cgrp1' - autoCommit = 'enable.auto.commit:false' + autoCommit = 'enable.auto.commit:true' autoCommitInterval = 'auto.commit.interval.ms:1000' autoOffset = 'auto.offset.reset:earliest' @@ -87,8 +87,9 @@ class TDTestCase: tdLog.info("start consume processor") tmqCom.startTmqSimProcess(self.pollDelay,self.paraDict["dbName"],self.showMsg, self.showRow,self.cdbName) - tdLog.info("After waiting for a period of time, drop one stable") - time.sleep(3) + tdLog.info("After waiting for a commit notify, drop one stable") + #time.sleep(3) + tmqCom.getStartCommitNotifyFromTmqsim() tdSql.execute("drop table %s.%s" %(self.paraDict['dbName'], self.paraDict['stbName'])) tdLog.info("wait result from consumer, then check it") From 00080750fa99ca4a388a59ae2e2a94da0ab4ac81 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 17 Oct 2022 20:40:11 +0800 Subject: [PATCH 077/135] fix: mnode may deadlock while drop another mnode if it crashed --- source/dnode/mnode/sdb/inc/sdb.h | 4 ++ source/dnode/mnode/sdb/src/sdb.c | 20 +++++++ source/dnode/mnode/sdb/src/sdbFile.c | 7 +-- source/dnode/mnode/sdb/src/sdbHash.c | 83 +++++++++++++--------------- 4 files changed, 65 insertions(+), 49 deletions(-) diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index be93bd2f59..a2373a55ce 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -403,6 +403,10 @@ const char *sdbStatusName(ESdbStatus status); void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper); int32_t sdbGetIdFromRaw(SSdb *pSdb, SSdbRaw *pRaw); +void sdbWriteLock(SSdb *pSdb, int32_t type); +void sdbReadLock(SSdb *pSdb, int32_t type); +void sdbUnLock(SSdb *pSdb, int32_t type); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 5393c42da3..e73fd28e71 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -181,3 +181,23 @@ void sdbGetCommitInfo(SSdb *pSdb, int64_t *index, int64_t *term, int64_t *config pSdb->applyIndex, pSdb->applyTerm, pSdb->applyConfig, *index, *term, *config); #endif } + +void sdbWriteLock(SSdb *pSdb, int32_t type) { + TdThreadRwlock *pLock = &pSdb->locks[type]; + // mTrace("sdb table:%d start write lock:%p", type, pLock); + taosThreadRwlockWrlock(pLock); + // mTrace("sdb table:%d stop write lock:%p", type, pLock); +} + +void sdbReadLock(SSdb *pSdb, int32_t type) { + TdThreadRwlock *pLock = &pSdb->locks[type]; + // mTrace("sdb table:%d start read lock:%p", type, pLock); + taosThreadRwlockRdlock(pLock); + // mTrace("sdb table:%d stop read lock:%p", type, pLock); +} + +void sdbUnLock(SSdb *pSdb, int32_t type) { + TdThreadRwlock *pLock = &pSdb->locks[type]; + // mTrace("sdb table:%d unlock:%p", type, pLock); + taosThreadRwlockUnlock(pLock); +} diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 18dbea150b..5eedcc545a 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -363,9 +363,8 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { mInfo("write %s to sdb file, total %d rows", sdbTableName(i), sdbGetSize(pSdb, i)); - SHashObj *hash = pSdb->hashObjs[i]; - TdThreadRwlock *pLock = &pSdb->locks[i]; - taosThreadRwlockWrlock(pLock); + SHashObj *hash = pSdb->hashObjs[i]; + sdbWriteLock(pSdb, i); SSdbRow **ppRow = taosHashIterate(hash, NULL); while (ppRow != NULL) { @@ -410,7 +409,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { sdbFreeRaw(pRaw); ppRow = taosHashIterate(hash, ppRow); } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, i); } if (code == 0) { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 0fee0fbca1..1e56eb87bf 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -133,12 +133,12 @@ static int32_t sdbGetkeySize(SSdb *pSdb, ESdbType type, const void *pKey) { } static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pRow, int32_t keySize) { - TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockWrlock(pLock); + int32_t type = pRow->type; + sdbWriteLock(pSdb, type); SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize); if (pOldRow != NULL) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; return terrno; @@ -149,7 +149,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * sdbPrintOper(pSdb, pRow, "insert"); if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; @@ -164,12 +164,12 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosHashRemove(hash, pRow->pObj, keySize); sdbFreeRow(pSdb, pRow, false); terrno = code; - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); return terrno; } } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) { pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); @@ -183,26 +183,27 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * } static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pNewRow, int32_t keySize) { - TdThreadRwlock *pLock = &pSdb->locks[pNewRow->type]; - taosThreadRwlockWrlock(pLock); + int32_t type = pNewRow->type; + sdbWriteLock(pSdb, type); SSdbRow **ppOldRow = taosHashGet(hash, pNewRow->pObj, keySize); if (ppOldRow == NULL || *ppOldRow == NULL) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); return sdbInsertRow(pSdb, hash, pRaw, pNewRow, keySize); } SSdbRow *pOldRow = *ppOldRow; pOldRow->status = pRaw->status; sdbPrintOper(pSdb, pOldRow, "update"); + sdbUnLock(pSdb, type); int32_t code = 0; - SdbUpdateFp updateFp = pSdb->updateFps[pNewRow->type]; + SdbUpdateFp updateFp = pSdb->updateFps[type]; if (updateFp != NULL) { code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pNewRow, false); pSdb->tableVer[pOldRow->type]++; @@ -210,12 +211,12 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * } static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pRow, int32_t keySize) { - TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockWrlock(pLock); + int32_t type = pRow->type; + sdbWriteLock(pSdb, type); SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize); if (ppOldRow == NULL || *ppOldRow == NULL) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; return terrno; @@ -228,7 +229,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosHashRemove(hash, pOldRow->pObj, keySize); pSdb->tableVer[pOldRow->type]++; - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pRow, false); @@ -282,12 +283,11 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { void *pRet = NULL; int32_t keySize = sdbGetkeySize(pSdb, type, pKey); - TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); SSdbRow **ppRow = taosHashGet(hash, pKey, keySize); if (ppRow == NULL || *ppRow == NULL) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; return NULL; } @@ -310,13 +310,13 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { break; } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); return pRet; } static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { - TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockWrlock(pLock); + int32_t type = pRow->type; + sdbWriteLock(pSdb, type); int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); sdbPrintOper(pSdb, pRow, "check"); @@ -324,7 +324,7 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { sdbFreeRow(pSdb, pRow, true); } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); } void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) { @@ -333,9 +333,9 @@ void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) { SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow)); if (pRow->type >= SDB_MAX) return; - TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; + int32_t type = pRow->type; if (lock) { - taosThreadRwlockWrlock(pLock); + sdbWriteLock(pSdb, type); } int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); @@ -345,7 +345,7 @@ void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) { } if (lock) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); } } @@ -357,8 +357,7 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; - TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); SSdbRow **ppRow = taosHashIterate(hash, pIter); while (ppRow != NULL) { @@ -373,7 +372,7 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { *ppObj = pRow->pObj; break; } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); return ppRow; } @@ -384,9 +383,8 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; - TdThreadRwlock *pLock = &pSdb->locks[type]; if (lock) { - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); } SSdbRow **ppRow = taosHashIterate(hash, pIter); @@ -404,7 +402,7 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat break; } if (lock) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); } return ppRow; @@ -416,18 +414,17 @@ void sdbCancelFetch(SSdb *pSdb, void *pIter) { SHashObj *hash = sdbGetHash(pSdb, pRow->type); if (hash == NULL) return; - TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockRdlock(pLock); + int32_t type = pRow->type; + sdbReadLock(pSdb, type); taosHashCancelIterate(hash, pIter); - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); } void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2, void *p3) { SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return; - TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); SSdbRow **ppRow = taosHashIterate(hash, NULL); while (ppRow != NULL) { @@ -443,17 +440,16 @@ void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2 ppRow = taosHashIterate(hash, ppRow); } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); } int32_t sdbGetSize(SSdb *pSdb, ESdbType type) { SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return 0; - TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); int32_t size = taosHashGetSize(hash); - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); return size; } @@ -465,9 +461,7 @@ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) { if (pSdb->keyTypes[type] != SDB_KEY_INT32) return -1; int32_t maxId = 0; - - TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); SSdbRow **ppRow = taosHashIterate(hash, NULL); while (ppRow != NULL) { @@ -477,8 +471,7 @@ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) { ppRow = taosHashIterate(hash, ppRow); } - taosThreadRwlockUnlock(pLock); - + sdbUnLock(pSdb, type); maxId = TMAX(maxId, pSdb->maxId[type]); return maxId + 1; } From 2d9c9e472c9b4e918b66b400a1da105707c259ec Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 17 Oct 2022 21:38:39 +0800 Subject: [PATCH 078/135] test: comment out unstable case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 39adfc9d53..4c0ef5a527 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -248,7 +248,7 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 - python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 From c370ca45d65a3fc3c999cc7696256e0d20a52972 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 17 Oct 2022 22:10:46 +0800 Subject: [PATCH 079/135] fix: add taosadapter to macos service --- .../tools/com.taosdata.taosadapter.plist | 33 +++++++++++++++++++ packaging/tools/install.sh | 8 ++--- packaging/tools/install_client.sh | 2 +- packaging/tools/mac_before_install.txt | 1 + packaging/tools/make_install.sh | 21 +++++++----- packaging/tools/post.sh | 9 +++-- packaging/tools/preun.sh | 4 +-- packaging/tools/remove.sh | 8 +++-- packaging/tools/remove_client.sh | 2 +- 9 files changed, 67 insertions(+), 21 deletions(-) create mode 100644 packaging/tools/com.taosdata.taosadapter.plist diff --git a/packaging/tools/com.taosdata.taosadapter.plist b/packaging/tools/com.taosdata.taosadapter.plist new file mode 100644 index 0000000000..2956b1ea5a --- /dev/null +++ b/packaging/tools/com.taosdata.taosadapter.plist @@ -0,0 +1,33 @@ + + + + + Label + com.tdengine.taosadapter + ProgramArguments + + /usr/local/bin/taosadapter + + ProcessType + Interactive + Disabled + + RunAtLoad + + LaunchOnlyOnce + + SessionCreate + + ExitTimeOut + 600 + KeepAlive + + SuccessfulExit + + AfterInitialDemand + + + Program + /usr/local/bin/taosadapter + + \ No newline at end of file diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 9694a89a8f..660bfd4fbf 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -69,7 +69,7 @@ prompt_force=0 initd_mod=0 service_mod=2 -if pidof systemd &>/dev/null; then +if ps aux | grep -v grep | grep systemd &>/dev/null; then service_mod=0 elif $(which service &>/dev/null); then service_mod=1 @@ -575,11 +575,11 @@ function install_web() { function clean_service_on_sysvinit() { - if pidof ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then ${csudo}service ${serverName} stop || : fi - if pidof tarbitrator &>/dev/null; then + if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then ${csudo}service tarbitratord stop || : fi @@ -759,7 +759,7 @@ function updateProduct() { echo -e "${GREEN}Start to update ${productName}...${NC}" # Stop the service if running - if pidof ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then if ((${service_mod} == 0)); then ${csudo}systemctl stop ${serverName} || : elif ((${service_mod} == 1)); then diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 254e7212fc..f7d677c982 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -265,7 +265,7 @@ function update_TDengine() { tar -zxf ${tarName} echo -e "${GREEN}Start to update ${productName} client...${NC}" # Stop the client shell if running - if pidof ${clientName} &> /dev/null; then + if ps aux | grep -v grep | grep ${clientName} &> /dev/null; then kill_client sleep 1 fi diff --git a/packaging/tools/mac_before_install.txt b/packaging/tools/mac_before_install.txt index b33cd478b8..3b6d610e88 100644 --- a/packaging/tools/mac_before_install.txt +++ b/packaging/tools/mac_before_install.txt @@ -2,4 +2,5 @@ TDengine is a high-efficient, scalable, high-available distributed time-series d To configure TDengine : edit /etc/taos/taos.cfg To start service : launchctl start com.tdengine.taosd +To start Taos Adapter : launchctl start com.tdengine.taosadapter To access TDengine : use taos in shell \ No newline at end of file diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index d0e94e3eb3..9034fd85f5 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -77,7 +77,7 @@ os_type=0 if [ "$osType" != "Darwin" ]; then initd_mod=0 - if pidof systemd &>/dev/null; then + if ps aux | grep -v grep | grep systemd &>/dev/null; then service_mod=0 elif $(which service &>/dev/null); then service_mod=1 @@ -130,7 +130,6 @@ function kill_taosadapter() { } function kill_taosd() { - ps -ef | grep ${serverName} pid=$(ps -ef | grep -w ${serverName} | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : @@ -438,7 +437,7 @@ function install_web() { } function clean_service_on_sysvinit() { - if pidof ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then ${csudo}service ${serverName} stop || : fi @@ -535,6 +534,10 @@ function install_service_on_launchctl() { ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : ${csudo}cp ${script_dir}/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : + + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : + ${csudo}cp ${script_dir}/com.taosdata.taosadapter.plist /Library/LaunchDaemons/com.taosdata.taosadapter.plist + ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : } function install_service() { @@ -568,7 +571,7 @@ function update_TDengine() { echo -e "${GREEN}Start to update ${productName}...${NC}" # Stop the service if running - if pidof ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then if ((${service_mod} == 0)); then ${csudo}systemctl stop ${serverName} || : elif ((${service_mod} == 1)); then @@ -615,11 +618,12 @@ function update_TDengine() { else if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" + [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" else echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" fi - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" fi echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName}${NC} in shell${NC}" @@ -666,11 +670,12 @@ function install_TDengine() { else if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" + [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" else echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" fi - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" fi echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName}${NC} in shell${NC}" diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index b729513b97..708a93c4fa 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -75,7 +75,7 @@ fi initd_mod=0 service_mod=2 -if pidof systemd &> /dev/null; then +if ps aux | grep -v grep | grep systemd &> /dev/null; then service_mod=0 elif $(which service &> /dev/null); then service_mod=1 @@ -421,7 +421,7 @@ function clean_service_on_sysvinit() { #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" #${csudo}sed -i "\|${restart_config_str}|d" /etc/inittab || : - if pidof taosd &> /dev/null; then + if ps aux | grep -v grep | grep taosd &> /dev/null; then ${csudo}service taosd stop || : fi @@ -512,6 +512,11 @@ function install_service_on_launchctl() { ${csudo}cp ${install_main_dir}/service/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist || : ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist || : fi + if [ -f ${install_main_dir}/service/com.taosdata.taosadapter.plist ]; then + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : + ${csudo}cp ${install_main_dir}/service/com.taosdata.taosadapter.plist /Library/LaunchDaemons/com.taosdata.taosadapter.plist || : + ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist || : + fi } function install_taosadapter_service() { diff --git a/packaging/tools/preun.sh b/packaging/tools/preun.sh index 8dee9da988..0e96c71d5d 100755 --- a/packaging/tools/preun.sh +++ b/packaging/tools/preun.sh @@ -25,7 +25,7 @@ fi initd_mod=0 service_mod=2 -if pidof systemd &> /dev/null; then +if ps aux | grep -v grep | grep systemd &> /dev/null; then service_mod=0 elif $(which service &> /dev/null); then service_mod=1 @@ -82,7 +82,7 @@ function clean_service_on_sysvinit() { #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" #${csudo}sed -i "\|${restart_config_str}|d" /etc/inittab || : - if pidof taosd &> /dev/null; then + if ps aux | grep -v grep | grep taosd &> /dev/null; then echo "TDengine taosd is running, stopping it..." ${csudo}service taosd stop || : fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index bace0bc869..8e5630ec2f 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -55,7 +55,7 @@ fi initd_mod=0 service_mod=2 -if pidof systemd &>/dev/null; then +if ps aux | grep -v grep | grep systemd &>/dev/null; then service_mod=0 elif $(which service &>/dev/null); then service_mod=1 @@ -171,12 +171,12 @@ function clean_service_on_systemd() { } function clean_service_on_sysvinit() { - if pidof ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then echo "${productName} ${serverName} is running, stopping it..." ${csudo}service ${serverName} stop || : fi - if pidof tarbitrator &>/dev/null; then + if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then echo "${productName} tarbitrator is running, stopping it..." ${csudo}service tarbitratord stop || : fi @@ -215,6 +215,8 @@ function clean_service_on_sysvinit() { function clean_service_on_launchctl() { ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : + ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : } function clean_service() { diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 54f7a949f3..7ab7fa8599 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -28,7 +28,7 @@ if command -v sudo > /dev/null; then fi function kill_client() { - if [ -n "$(pidof ${clientName})" ]; then + if [ -n "$(ps aux | grep -v grep | grep ${clientName})" ]; then ${csudo}kill -9 $pid || : fi } From 78df93e0cae4b96ea4dea8641a23728b8723e6f0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 17 Oct 2022 22:17:47 +0800 Subject: [PATCH 080/135] fix invalid packet --- source/libs/index/src/indexTfile.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 2f879ff552..4fdf6d9e57 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -587,6 +587,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { taosMemoryFree(buf); return -1; } + buf = t; } char* p = buf; From 5fc475b9608c401d31cd8e4baf27bd8616dc38a0 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 18 Oct 2022 09:40:19 +0800 Subject: [PATCH 081/135] fix(stream): coverity scan, memory leak --- source/libs/executor/src/timewindowoperator.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index cbbd874bb9..947afd4ebc 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1480,7 +1480,7 @@ static void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, STimeWindowAgg for (int32_t i = 0; i < pBlock->info.rows; i++) { TSKEY startTs = TMAX(tsStarts[i], pTwSup->minTs); TSKEY endTs = TMIN(tsEnds[i], pTwSup->maxTs); - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTs, pInterval, TSDB_ORDER_ASC); do { @@ -1506,7 +1506,7 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* pGpDatas = (uint64_t*)pGpCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); while (win.ekey <= endTsCols[i]) { @@ -3471,6 +3471,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + initBasicInfo(&pInfo->binfo, pResBlock); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { @@ -3478,7 +3479,6 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, } initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs); - initBasicInfo(&pInfo->binfo, pResBlock); ASSERT(numOfCols > 0); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); @@ -3666,7 +3666,6 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); int32_t code = TSDB_CODE_OUT_OF_MEMORY; SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -3685,8 +3684,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh goto _error; } } - SExprSupp* pSup = &pOperator->exprSupp; - + SExprSupp* pSup = &pOperator->exprSupp; + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -5004,7 +5003,6 @@ int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFu SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode; - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; int32_t code = TSDB_CODE_OUT_OF_MEMORY; @@ -5040,6 +5038,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys }; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -5827,8 +5826,6 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->isFinal = false; - - pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); SExprSupp* pSup = &pOperator->exprSupp; @@ -5837,8 +5834,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys initStreamFunciton(pSup->pCtx, pSup->numOfExprs); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } From 951857430ef90c19d929644042a4123ad46b3805 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Oct 2022 10:21:19 +0800 Subject: [PATCH 082/135] test: comment out unstable case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 39adfc9d53..4c0ef5a527 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -248,7 +248,7 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 - python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 From bdb81e2b89c8fbfc82815d37c6c64a0f05c28770 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 18 Oct 2022 10:44:24 +0800 Subject: [PATCH 083/135] fix: fix varchar and bigint compare issue --- source/libs/catalog/src/ctgCache.c | 6 +++++ source/libs/scalar/src/sclvector.c | 38 +++++++++++++++--------------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 3a774afdcc..010ca0d81f 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1393,8 +1393,10 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); } else { + CTG_LOCK(CTG_WRITE, &pCache->metaLock); taosMemoryFree(pCache->pMeta); pCache->pMeta = meta; + CTG_UNLOCK(CTG_WRITE, &pCache->metaLock); } if (NULL == orig) { @@ -1457,6 +1459,8 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa return TSDB_CODE_SUCCESS; } + CTG_LOCK(CTG_WRITE, &pCache->indexLock); + if (pCache->pIndex) { if (0 == suid) { suid = pCache->pIndex->suid; @@ -1466,6 +1470,8 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa } pCache->pIndex = pIndex; + CTG_UNLOCK(CTG_WRITE, &pCache->indexLock); + *index = NULL; ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f467721248..22e9fb4305 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -862,25 +862,25 @@ int32_t vectorConvertSingleColImpl(const SScalarParam* pIn, SScalarParam* pOut, int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB + 1][TSDB_DATA_TYPE_BLOB + 1] = { /* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB */ - /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 7, 9, 7, 11, 12, 13, 14, 0, 7, 0, 0, - /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, - /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, - /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0, - /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 9, 7, 5, 5, 5, 7, 0, 7, 0, 0, - /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, - /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, - /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, - /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, - /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, - /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, - /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, - /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0, - /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, - /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, 7, 0, 0, + /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, + /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, + /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0, + /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, 7, 0, 0, + /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, + /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, + /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, + /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, + /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, + /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, + /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, + /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0, + /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, + /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; int32_t vectorGetConvertType(int32_t type1, int32_t type2) { if (type1 == type2) { From b5f7623219cfc9f9b3f5de3738444ca52643cf57 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 18 Oct 2022 10:53:29 +0800 Subject: [PATCH 084/135] fix invalid packet --- source/libs/index/src/indexFilter.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index a503b82d3a..075408f1b3 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -231,7 +231,9 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SIF_ERR_RET(sifGetValueFromNode(node, ¶m->condValue)); param->colId = -1; param->colValType = (uint8_t)(vn->node.resType.type); - memcpy(param->colName, vn->literal, sizeof(param->colName)); + if (strlen(vn->literal) <= sizeof(param->colName)) { + memcpy(param->colName, vn->literal, strlen(vn->literal)); + } break; } case QUERY_NODE_COLUMN: { From 44401b6b909734aecd28539d831a1904e3076a4e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 18 Oct 2022 10:58:33 +0800 Subject: [PATCH 085/135] fix: memory leak --- source/client/src/clientTmq.c | 7 ++++++- source/dnode/vnode/src/tq/tqSink.c | 2 +- tests/system-test/7-tmq/tmqAutoCreateTbl.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 8f94d88b1c..838b851d6d 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -455,7 +455,10 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT return -1; } void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); - if (buf == NULL) return -1; + if (buf == NULL) { + taosMemoryFree(pOffset); + return -1; + } ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); @@ -468,6 +471,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT // build param SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); if (pParam == NULL) { + taosMemoryFree(pOffset); taosMemoryFree(buf); return -1; } @@ -477,6 +481,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT // build send info SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { + taosMemoryFree(pOffset); taosMemoryFree(buf); taosMemoryFree(pParam); return -1; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 9bdcf04e89..ab195b1eb3 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -436,7 +436,7 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { } if (mr.me.ctbEntry.suid != suid) { tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid: %ld, actual suid %ld", - TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry); + TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid); metaReaderClear(&mr); taosMemoryFree(ctbName); continue; diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index 587472ed82..ce2cad4b14 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -160,7 +160,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 1000, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} From 3190ef65a3c43521e62cd015bd1fae6f1de9f3e5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 17 Oct 2022 21:13:52 +0800 Subject: [PATCH 086/135] fix: deal with misaligned idx entries in walCheckAndRepairIdxFile --- source/libs/wal/src/walMeta.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 85791fa682..fa22805df2 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -481,6 +481,10 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { continue; } + if (offset != (idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry)) { + continue; + } + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { wWarn("vgId:%d, failed to read log file since %s. file:%s, offset:%" PRId64 ", idx entry ver:%" PRId64 "", pWal->cfg.vgId, terrstr(), fLogNameStr, idxEntry.offset, idxEntry.ver); @@ -493,6 +497,8 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { } offset += sizeof(SWalIdxEntry); + ASSERT(offset == (idxEntry.ver - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)); + // ftruncate idx file if (offset < fileSize) { if (taosFtruncateFile(pIdxFile, offset) < 0) { From 23e18bff76781825aed80efc4cfe90450c6ab74b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 01:31:19 +0800 Subject: [PATCH 087/135] fix: fsync fatal log messages --- source/util/src/tlog.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 2e2300ba14..9d97cf7ab2 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -443,10 +443,13 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) { if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL && osLogSpaceAvailable()) { taosUpdateLogNums(level); - if (tsAsyncLog) { + if (tsAsyncLog && level != DEBUG_FATAL) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); + if (level == DEBUG_FATAL) { + taosFsyncFile(tsLogObj.logHandle->pFile); + } } if (tsLogObj.maxLines > 0) { From f0427c93223193fc03ea8261726b42a8159e863c Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 01:37:55 +0800 Subject: [PATCH 088/135] fix: ftrunc on appending WAL log failure for recovery in walWriteImpl --- source/libs/wal/src/walWrite.c | 41 ++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 1e5c63fa8d..7da88ec95d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -408,25 +408,33 @@ END: static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; - int64_t idxOffset = taosLSeekFile(pWal->pIdxFile, 0, SEEK_END); + SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); + ASSERT(pFileInfo != NULL); + ASSERT(pFileInfo->firstVer >= 0); + int64_t idxOffset = (entry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry); wDebug("vgId:%d, write index, index:%" PRId64 ", offset:%" PRId64 ", at %" PRId64, pWal->cfg.vgId, ver, offset, idxOffset); + int64_t size = taosWriteFile(pWal->pIdxFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { + wError("vgId:%d, failed to write idx entry due to %s. ver:%lld", pWal->cfg.vgId, strerror(errno), ver); terrno = TAOS_SYSTEM_ERROR(errno); - // TODO truncate return -1; } return 0; } -// TODO gurantee atomicity by truncate failed writing static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen) { int64_t code = 0; int64_t offset = walGetCurFileOffset(pWal); + SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); + ASSERT(pFileInfo != NULL); + if (pFileInfo->firstVer == -1) { + pFileInfo->firstVer = index; + } pWal->writeHead.head.version = index; pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.msgType = msgType; @@ -437,11 +445,9 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); - wDebug("vgId:%d, wal write log %ld, msgType: %s", pWal->cfg.vgId, index, TMSG_INFO(msgType)); if (taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { - // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); @@ -450,7 +456,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy } if (taosWriteFile(pWal->pLogFile, (char *)body, bodyLen) != bodyLen) { - // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); @@ -460,7 +465,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy code = walWriteIndex(pWal, index, offset); if (code < 0) { - // TODO ftruncate goto END; } @@ -468,14 +472,27 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy if (pWal->vers.firstVer == -1) pWal->vers.firstVer = index; pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalCkHead) + bodyLen; - if (walGetCurFileInfo(pWal)->firstVer == -1) { - walGetCurFileInfo(pWal)->firstVer = index; - } - walGetCurFileInfo(pWal)->lastVer = index; - walGetCurFileInfo(pWal)->fileSize += sizeof(SWalCkHead) + bodyLen; + pFileInfo->lastVer = index; + pFileInfo->fileSize += sizeof(SWalCkHead) + bodyLen; return 0; + END: + // recover in a reverse order + if (taosFtruncateFile(pWal->pLogFile, offset) < 0) { + wFatal("vgId:%d, failed to ftruncate logfile to offset:%lld during recovery due to %s", pWal->cfg.vgId, offset, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + ASSERT(0 && "failed to recover from error"); + } + + int64_t idxOffset = (index - pFileInfo->firstVer) * sizeof(SWalIdxEntry); + if (taosFtruncateFile(pWal->pIdxFile, idxOffset) < 0) { + wFatal("vgId:%d, failed to ftruncate idxfile to offset:%lld during recovery due to %s", pWal->cfg.vgId, idxOffset, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + ASSERT(0 && "failed to recover from error"); + } return -1; } From af9aa9d78331061da00d37fdd95ee776d860d1a9 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 02:12:05 +0800 Subject: [PATCH 089/135] enh: assert on alignment of idx entries in walWriteIndex --- source/libs/wal/src/walWrite.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 7da88ec95d..c088145ff6 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -421,6 +421,8 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + + ASSERT(taosLSeekFile(pWal->pIdxFile, 0, SEEK_END) == idxOffset + sizeof(SWalIdxEntry) && "Offset of idx entries misaligned"); return 0; } From bb59553e39312176f86b353a5161d006b91f380a Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Tue, 18 Oct 2022 11:23:24 +0800 Subject: [PATCH 090/135] Update 01-arch.md --- docs/en/21-tdinternal/01-arch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md index a938084eb1..697ecb98a1 100644 --- a/docs/en/21-tdinternal/01-arch.md +++ b/docs/en/21-tdinternal/01-arch.md @@ -222,12 +222,12 @@ TDengine provides a variety of query processing functions for tables and STables ### Query Process -1. TDEngine client driver `taosc` parses the SQL statement and generates an abstract syntax tree (AST), then check and verify the AST according to metadata. During this stage, the metadata management module in `taosc` (Catalog) requests the metadata of the involved database and table from mnode and vnode. +1. TDengine client driver `taosc` parses the SQL statement and generates an abstract syntax tree (AST), then checks and verifies the AST according to metadata. During this stage, the metadata management module in `taosc` (Catalog) requests the metadata of the involved database and table from mnode and vnode. 2. After the verification passes, `taosc` generates distributed query plan and optimizes the plan. 3. `taosc` schedules the tasks according to configured query policy, a query sub-task may be scheduled to a vnode or qnode according to data relative and system load. Please be noted that both vnode and qnode are logic execution unit, the physical execution node is dnode (data node). 4. When a dnode receives a query request, it identifies which vnode or qnode this query request is targeted, and forwards the request to the query execution queue of the identified vnode or qnode. 5. The query execution thread of the vnode or qnode establishes fundamental query execution context, and executes the query, and notifies the client once obtaining a part of result data. -6. TDengine client driver `taosc` will initiates next level query tasks or obtain the result simply. +6. TDengine client driver `taosc` will initiate next level query tasks or obtain the result simply. ### Aggregation by Time Axis, Downsampling, Interpolation From cf2b6ccbfb5544070f379040a3f01a3a782d7e1d Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 09:58:02 +0800 Subject: [PATCH 091/135] enh: write idx ahead of writing its log entry in WAL --- source/libs/wal/src/walWrite.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index c088145ff6..aa6d52174d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -449,6 +449,11 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); wDebug("vgId:%d, wal write log %ld, msgType: %s", pWal->cfg.vgId, index, TMSG_INFO(msgType)); + code = walWriteIndex(pWal, index, offset); + if (code < 0) { + goto END; + } + if (taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), @@ -465,11 +470,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy goto END; } - code = walWriteIndex(pWal, index, offset); - if (code < 0) { - goto END; - } - // set status if (pWal->vers.firstVer == -1) pWal->vers.firstVer = index; pWal->vers.lastVer = index; From fc9982de68454be3bf340ba928ef3ed964563255 Mon Sep 17 00:00:00 2001 From: Zhiqiang Wang <1296468573@qq.com> Date: Tue, 18 Oct 2022 11:41:59 +0800 Subject: [PATCH 092/135] docs: change macos client doc (#17420) * docs: change macos client doc * docs: change macos client doc * docs: make taosadapter/taosbenchmark description align with prev * fix: chinese colon * fix: chinese colon Co-authored-by: Shuduo Sang --- docs/en/05-get-started/03-package.md | 18 +++++++++++++----- docs/en/07-develop/01-connect/index.md | 16 ++++++++++++---- .../03-connector/_macos_install.mdx | 19 +++++++++++++++++++ .../03-connector/_verify_macos.mdx | 15 +++++++++++++++ .../03-connector/_windows_install.mdx | 8 +++++--- docs/en/14-reference/03-connector/index.mdx | 16 ++++++++++++---- docs/zh/05-get-started/03-package.md | 18 +++++++++++++----- docs/zh/07-develop/01-connect/index.md | 16 ++++++++++++---- docs/zh/08-connector/_macos_install.mdx | 19 +++++++++++++++++++ docs/zh/08-connector/_verify_macos.mdx | 15 +++++++++++++++ docs/zh/08-connector/_windows_install.mdx | 2 ++ docs/zh/08-connector/index.md | 16 ++++++++++++---- tools/taosws-rs | 2 +- 13 files changed, 150 insertions(+), 30 deletions(-) create mode 100644 docs/en/14-reference/03-connector/_macos_install.mdx create mode 100644 docs/en/14-reference/03-connector/_verify_macos.mdx create mode 100644 docs/zh/08-connector/_macos_install.mdx create mode 100644 docs/zh/08-connector/_verify_macos.mdx diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index 7800fb1c88..54d2e046c2 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -116,7 +116,7 @@ Note: TDengine only supports Windows Server 2016/2019 and Windows 10/11 on the W 1. Download the macOS installation package. -2. Run the downloaded package to install TDengine. +2. Run the downloaded package to install TDengine. If the installation is blocked, you can right-click or ctrl-click on the installation package and select `Open`. @@ -179,12 +179,20 @@ The following `systemctl` commands can help you manage TDengine service: ::: +## Command Line Interface (CLI) + +You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in terminal. + After the installation is complete, run `C:\TDengine\taosd.exe` to start TDengine Server. +## Command Line Interface (CLI) + +You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can run `taos.exe` in the `C:\TDengine` directory of the Windows terminal to start the TDengine command line. + @@ -206,12 +214,12 @@ The following `launchctl` commands can help you manage TDengine service: ::: - - - ## Command Line Interface (CLI) -You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in the Linux/macOS terminal where TDengine is installed, or you can run `taos.exe` in the `C:\TDengine` directory of the Windows terminal where TDengine is installed to start the TDengine command line. +You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in terminal. + + + ```bash taos diff --git a/docs/en/07-develop/01-connect/index.md b/docs/en/07-develop/01-connect/index.md index 5ccddb06b0..45bbaa2751 100644 --- a/docs/en/07-develop/01-connect/index.md +++ b/docs/en/07-develop/01-connect/index.md @@ -15,10 +15,12 @@ import ConnCSNative from "./_connect_cs.mdx"; import ConnC from "./_connect_c.mdx"; import ConnR from "./_connect_r.mdx"; import ConnPHP from "./_connect_php.mdx"; -import InstallOnWindows from "../../14-reference/03-connector/_linux_install.mdx"; -import InstallOnLinux from "../../14-reference/03-connector/_windows_install.mdx"; +import InstallOnLinux from "../../14-reference/03-connector/_linux_install.mdx"; +import InstallOnWindows from "../../14-reference/03-connector/_windows_install.mdx"; +import InstallOnMacOS from "../../14-reference/03-connector/_macos_install.mdx"; import VerifyLinux from "../../14-reference/03-connector/_verify_linux.mdx"; import VerifyWindows from "../../14-reference/03-connector/_verify_windows.mdx"; +import VerifyMacOS from "../../14-reference/03-connector/_verify_macos.mdx"; Any application running on any platform can access TDengine through the REST API provided by TDengine. For information, see [REST API](/reference/rest-api/). Applications can also use the connectors for various programming languages, including C/C++, Java, Python, Go, Node.js, C#, and Rust, to access TDengine. These connectors support connecting to TDengine clusters using both native interfaces (taosc). Some connectors also support connecting over a REST interface. Community developers have also contributed several unofficial connectors, such as the ADO.NET connector, the Lua connector, and the PHP connector. @@ -44,10 +46,13 @@ If you are choosing to use the native connection and the the application is not - + - + + + + @@ -62,6 +67,9 @@ After the above installation and configuration are done and making sure TDengine + + + ## Install Connectors diff --git a/docs/en/14-reference/03-connector/_macos_install.mdx b/docs/en/14-reference/03-connector/_macos_install.mdx new file mode 100644 index 0000000000..8813d65869 --- /dev/null +++ b/docs/en/14-reference/03-connector/_macos_install.mdx @@ -0,0 +1,19 @@ +import PkgListV3 from "/components/PkgListV3"; + +1. Download the client installation package + + + + [All Downloads](../../releases/tdengine) + +2. Execute the installer, select the default value as prompted, and complete the installation. If the installation is blocked, you can right-click or ctrl-click on the installation package and select `Open`. +3. configure taos.cfg + + Edit `taos.cfg` file (full path is `/etc/taos/taos.cfg` by default), modify `firstEP` with actual TDengine server's End Point, for example `h1.tdengine.com:6030` + +:::tip + +1. If the computer does not run the TDengine service but installs the TDengine client driver, then you need to config `firstEP` in `taos.cfg` only, and there is no need to configure `FQDN`; +2. If you encounter the "Unable to resolve FQDN" error, please make sure the FQDN in the `/etc/hosts` file of the current computer is correctly configured, or the DNS service is correctly configured. + +::: diff --git a/docs/en/14-reference/03-connector/_verify_macos.mdx b/docs/en/14-reference/03-connector/_verify_macos.mdx new file mode 100644 index 0000000000..bac5f2f3e1 --- /dev/null +++ b/docs/en/14-reference/03-connector/_verify_macos.mdx @@ -0,0 +1,15 @@ +Execute TDengine CLI program `taos` directly from the macOS shell to connect to the TDengine service and enter the TDengine CLI interface, as shown in the following example. + +```text +$ taos + +taos> show databases; + name | +================================= + information_schema | + performance_schema | + db | +Query OK, 3 rows in database (0.019154s) + +taos> +``` diff --git a/docs/en/14-reference/03-connector/_windows_install.mdx b/docs/en/14-reference/03-connector/_windows_install.mdx index 81fc5573bb..a96e02feea 100644 --- a/docs/en/14-reference/03-connector/_windows_install.mdx +++ b/docs/en/14-reference/03-connector/_windows_install.mdx @@ -10,12 +10,14 @@ import PkgListV3 from "/components/PkgListV3"; The default installation path is C:\TDengine, including the following files (directories). - - _taos.exe_ : TDengine CLI command-line program - - _cfg_ : configuration file directory + - _taos.exe_: TDengine CLI command-line program + - _taosadapter.exe_: server-side executable that provides RESTful services and accepts writing requests from a variety of other softwares + - _taosBenchmark.exe_: TDengine testing tool + - _cfg_: configuration file directory - _driver_: client driver dynamic link library - _examples_: sample programs bash/C/C#/go/JDBC/Python/Node.js - _include_: header files - - _log_ : log file + - _log_: log file - _unins000.exe_: uninstaller 4. configure taos.cfg diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx index 675a019cc1..4fd9c452d8 100644 --- a/docs/en/14-reference/03-connector/index.mdx +++ b/docs/en/14-reference/03-connector/index.mdx @@ -74,10 +74,12 @@ The different database framework specifications for various programming language import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; -import InstallOnWindows from "./_linux_install.mdx"; -import InstallOnLinux from "./_windows_install.mdx"; +import InstallOnLinux from "./_linux_install.mdx"; +import InstallOnWindows from "./_windows_install.mdx"; +import InstallOnMacOS from "./_macos_install.mdx"; import VerifyWindows from "./_verify_windows.mdx"; import VerifyLinux from "./_verify_linux.mdx"; +import VerifyMacOS from "./_verify_macos.mdx"; ## Install Client Driver @@ -90,10 +92,13 @@ The client driver needs to be installed if you use the native interface connecto - + - + + + + @@ -108,5 +113,8 @@ After completing the above installation and configuration and you have confirmed + + + diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 167a68d3c8..781a65402c 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -115,7 +115,7 @@ apt-get 方式只适用于 Debian 或 Ubuntu 系统。 1. 从列表中下载获得 pkg 安装程序； -2. 运行可执行程序来安装 TDengine。 +2. 运行可执行程序来安装 TDengine。如果安装被阻止，可以右键或者按 Ctrl 点击安装包，选择 `打开`。 @@ -178,12 +178,20 @@ Active: inactive (dead) ::: +## TDengine 命令行（CLI） + +为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在终端执行 `taos` 即可。 + 安装后，在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。 +## TDengine 命令行（CLI） + +为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在终端执行 `taos` 即可。 + @@ -205,12 +213,12 @@ Active: inactive (dead) ::: - - - ## TDengine 命令行（CLI） -为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux、macOS 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。 +为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。 + + + ```bash taos diff --git a/docs/zh/07-develop/01-connect/index.md b/docs/zh/07-develop/01-connect/index.md index d3ffe7eba2..da31dec4c1 100644 --- a/docs/zh/07-develop/01-connect/index.md +++ b/docs/zh/07-develop/01-connect/index.md @@ -14,9 +14,11 @@ import ConnCSNative from "./_connect_cs.mdx"; import ConnC from "./_connect_c.mdx"; import ConnR from "./_connect_r.mdx"; import ConnPHP from "./_connect_php.mdx"; -import InstallOnWindows from "../../08-connector/_linux_install.mdx"; -import InstallOnLinux from "../../08-connector/_windows_install.mdx"; +import InstallOnLinux from "../../08-connector/_linux_install.mdx"; +import InstallOnWindows from "../../08-connector/_windows_install.mdx"; +import InstallOnMacOS from "../../08-connector/_macos_install.mdx"; import VerifyLinux from "../../08-connector/_verify_linux.mdx"; +import VerifyMacOS from "../../08-connector/_verify_macos.mdx"; import VerifyWindows from "../../08-connector/_verify_windows.mdx"; TDengine 提供了丰富的应用程序开发接口，为了便于用户快速开发自己的应用，TDengine 支持了多种编程语言的连接器，其中官方连接器包括支持 C/C++、Java、Python、Go、Node.js、C#、Rust、Lua（社区贡献）和 PHP （社区贡献）的连接器。这些连接器支持使用原生接口（taosc）和 REST 接口（部分语言暂不支持）连接 TDengine 集群。社区开发者也贡献了多个非官方连接器，例如 ADO.NET 连接器、Lua 连接器和 PHP 连接器。 @@ -43,10 +45,13 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 - + - + + + + @@ -61,6 +66,9 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 + + + ## 安装连接器 diff --git a/docs/zh/08-connector/_macos_install.mdx b/docs/zh/08-connector/_macos_install.mdx new file mode 100644 index 0000000000..4925ffabcd --- /dev/null +++ b/docs/zh/08-connector/_macos_install.mdx @@ -0,0 +1,19 @@ +import PkgListV3 from "/components/PkgListV3"; + +1. 下载客户端安装包 + + + + [所有下载](../../releases/tdengine) + +2. 执行安装程序，按提示选择默认值，完成安装。如果安装被阻止，可以右键或者按 Ctrl 点击安装包，选择 `打开`。 +3. 配置 taos.cfg + + 编辑 `taos.cfg` 文件（默认路径/etc/taos/taos.cfg），将 `firstEP` 修改为 TDengine 服务器的 End Point，例如：`h1.tdengine.com:6030` + +:::tip + +1. 如本机没有部署 TDengine 服务，仅安装了应用驱动，则 `taos.cfg` 中仅需配置 `firstEP`，无需在本机配置 `FQDN`。 +2. 为防止与服务器端连接时出现“Unable to resolve FQDN”错误，建议确认本机的 `/etc/hosts` 文件已经配置了服务器正确的 FQDN 值，或配置好了 DNS 服务。 + +::: diff --git a/docs/zh/08-connector/_verify_macos.mdx b/docs/zh/08-connector/_verify_macos.mdx new file mode 100644 index 0000000000..276a541210 --- /dev/null +++ b/docs/zh/08-connector/_verify_macos.mdx @@ -0,0 +1,15 @@ +在 macOS shell 下直接执行 `taos` 连接到 TDengine 服务，进入到 TDengine CLI 界面，示例如下： + +```text +$ taos + +taos> show databases; + name | +================================= + information_schema | + performance_schema | + db | +Query OK, 3 rows in database (0.019154s) + +taos> +``` diff --git a/docs/zh/08-connector/_windows_install.mdx b/docs/zh/08-connector/_windows_install.mdx index 3cd688e615..04874c8a86 100644 --- a/docs/zh/08-connector/_windows_install.mdx +++ b/docs/zh/08-connector/_windows_install.mdx @@ -12,6 +12,8 @@ import PkgListV3 from "/components/PkgListV3"; 默认安装路径为：C:\TDengine，其中包括以下文件(目录)： - _taos.exe_：TDengine CLI 命令行程序 + - _taosadapter.exe_：提供 RESTful 服务和接受其他多种软件写入请求的服务端可执行文件 + - _taosBenchmark.exe_：TDengine 测试程序 - _cfg_ : 配置文件目录 - _driver_: 应用驱动动态链接库 - _examples_: 示例程序 bash/C/C#/go/JDBC/Python/Node.js diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md index f54470f742..e00e0b2fa2 100644 --- a/docs/zh/08-connector/index.md +++ b/docs/zh/08-connector/index.md @@ -74,10 +74,12 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; -import InstallOnWindows from "./_linux_install.mdx"; -import InstallOnLinux from "./_windows_install.mdx"; +import InstallOnLinux from "./_linux_install.mdx"; +import InstallOnWindows from "./_windows_install.mdx"; +import InstallOnMacOS from "./_macos_install.mdx"; import VerifyWindows from "./_verify_windows.mdx"; import VerifyLinux from "./_verify_linux.mdx"; +import VerifyMacOS from "./_verify_macos.mdx"; ## 安装客户端驱动 @@ -90,10 +92,13 @@ import VerifyLinux from "./_verify_linux.mdx"; - + - + + + + @@ -108,5 +113,8 @@ import VerifyLinux from "./_verify_linux.mdx"; + + + diff --git a/tools/taosws-rs b/tools/taosws-rs index 7a94ffab45..1bdfca396c 160000 --- a/tools/taosws-rs +++ b/tools/taosws-rs @@ -1 +1 @@ -Subproject commit 7a94ffab45f08e16f09b3f430fe75d717054adb6 +Subproject commit 1bdfca396cd6730cdc334e06fc7b2156dd1239a0 From 95dd30be57bc32aa1e3d8203353008f62bf02533 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Oct 2022 11:43:58 +0800 Subject: [PATCH 093/135] fix(query): fix coverity issues. --- source/client/src/clientEnv.c | 9 +- source/client/src/clientImpl.c | 3 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 194 +++++++++++++----- source/libs/executor/src/executil.c | 3 +- source/libs/executor/src/executor.c | 1 + source/libs/executor/src/executorimpl.c | 49 +++-- source/libs/executor/src/projectoperator.c | 2 +- source/libs/executor/src/scanoperator.c | 8 +- source/libs/executor/src/sortoperator.c | 18 +- source/libs/executor/src/timewindowoperator.c | 46 +++-- source/libs/executor/src/tlinearhash.c | 5 +- source/libs/executor/src/tsort.c | 3 +- 12 files changed, 228 insertions(+), 113 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index fc489f08af..b3c5565cfb 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -438,21 +438,18 @@ int taos_init() { } int taos_options_imp(TSDB_OPTION option, const char *str) { - if (option != TSDB_OPTION_CONFIGDIR) { - taos_init(); // initialize global config - } else { + if (option == TSDB_OPTION_CONFIGDIR) { tstrncpy(configDir, str, PATH_MAX); tscInfo("set cfg:%s to %s", configDir, str); return 0; + } else { + taos_init(); // initialize global config } SConfig *pCfg = taosGetCfg(); SConfigItem *pItem = NULL; switch (option) { - case TSDB_OPTION_CONFIGDIR: - pItem = cfgGetItem(pCfg, "configDir"); - break; case TSDB_OPTION_SHELL_ACTIVITY_TIMER: pItem = cfgGetItem(pCfg, "shellActivityTimer"); break; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b494140da5..df717bda23 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1130,7 +1130,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM break; } - if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { + // TODO weired responding code? + if (TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 1da5297396..3f88d478e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -645,11 +645,11 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks; double el = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug( - "load block of %"PRIzu" tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " - "time:%.2f ms %s", - numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, - pReader->idStr); + tsdbDebug("load block of %" PRIzu + " tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " + "time:%.2f ms %s", + numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, + pReader->idStr); pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; @@ -1579,7 +1579,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -1589,7 +1592,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow1); } else { init = true; - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } @@ -1600,16 +1606,29 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + int32_t code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; } - doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } } else { if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); - doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + int32_t code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (minKey == tsLast) { @@ -1618,7 +1637,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow1); } else { init = true; - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } @@ -1628,7 +1650,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow); } else { init = true; - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -1662,13 +1687,16 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) { return TSDB_CODE_SUCCESS; } else { - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + code = tRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1679,7 +1707,11 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, tRowMergerClear(&merge); } } else { // not merge block data - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); ASSERT(mergeBlockData); @@ -1688,7 +1720,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + code = tRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1724,7 +1756,11 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader STSRow* pTSRow = NULL; SRowMerger merge = {0}; - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); @@ -1732,7 +1768,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge); - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + code = tRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1756,9 +1792,9 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { - SRowMerger merge = {0}; - STSRow* pTSRow = NULL; - + SRowMerger merge = {0}; + STSRow* pTSRow = NULL; + int32_t code = TSDB_CODE_SUCCESS; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SArray* pDelList = pBlockScanInfo->delSkyline; @@ -1850,12 +1886,17 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, piRow, pSchema); + code = tRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); + + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, + &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (minKey == k.ts) { @@ -1863,23 +1904,31 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, pRow); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + code = tRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, + pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } else { if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + code = tRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, + pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (minKey == ik.ts) { @@ -1888,12 +1937,16 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, piRow, pSchema); + code = tRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, + pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (minKey == tsLast) { @@ -1924,7 +1977,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } } - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + code = tRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2085,9 +2138,13 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc STSRow* pTSRow = NULL; SRowMerger merge = {0}; - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + code = tRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2925,6 +2982,10 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe } STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } + tRowMergerAdd(pMerger, pRow, pTSchema); } @@ -3094,13 +3155,17 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, } STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); - if(pTSchema1 == NULL) { + if (pTSchema1 == NULL) { return terrno; } tRowMergerAdd(&merge, pNextRow, pTSchema1); - doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); + code = doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + code = tRowMergerGetRow(&merge, pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3121,19 +3186,44 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pSchema); - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); + int32_t code = tRowMergerInit(&merge, piRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, + pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } tRowMerge(&merge, pRow); - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + int32_t code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } tRowMerge(&merge, piRow); - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, + pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } int32_t code = tRowMergerGetRow(&merge, pTSRow); @@ -3457,18 +3547,18 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl STsdbReader* pPrevReader = pReader->innerReader[0]; STsdbReader* pNextReader = pReader->innerReader[1]; - // we need only one row - pPrevReader->capacity = 1; - pPrevReader->status.pTableMap = pReader->status.pTableMap; - pPrevReader->pSchema = pReader->pSchema; - pPrevReader->pMemSchema = pReader->pMemSchema; - pPrevReader->pReadSnap = pReader->pReadSnap; + // we need only one row + pPrevReader->capacity = 1; + pPrevReader->status.pTableMap = pReader->status.pTableMap; + pPrevReader->pSchema = pReader->pSchema; + pPrevReader->pMemSchema = pReader->pMemSchema; + pPrevReader->pReadSnap = pReader->pReadSnap; - pNextReader->capacity = 1; - pNextReader->status.pTableMap = pReader->status.pTableMap; - pNextReader->pSchema = pReader->pSchema; - pNextReader->pMemSchema = pReader->pMemSchema; - pNextReader->pReadSnap = pReader->pReadSnap; + pNextReader->capacity = 1; + pNextReader->status.pTableMap = pReader->status.pTableMap; + pNextReader->pSchema = pReader->pSchema; + pNextReader->pMemSchema = pReader->pMemSchema; + pNextReader->pReadSnap = pReader->pReadSnap; code = doOpenReaderImpl(pPrevReader); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index f99c832381..79f6166079 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -122,7 +122,8 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in if (order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC) { __compar_fn_t fn = (order == TSDB_ORDER_ASC) ? resultrowComparAsc : resultrowComparDesc; - taosSort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn); + int32_t size = POINTER_BYTES; + taosSort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), size, fn); } pGroupResInfo->index = 0; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 0f4118257b..0d861c1fb9 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -276,6 +276,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, &keyInfo.groupId); if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(keyBuf); return code; } } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index aaef411d2d..a7e98527f4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -97,11 +97,10 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator); void doSetOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; + ASSERT(pOperator->pTaskInfo != NULL); pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start * 1000) / 1000.0; - if (pOperator->pTaskInfo != NULL) { - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - } + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { @@ -1092,13 +1091,13 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM } SFilterInfo* filter = pFilterInfo; - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); -// pError("start filter"); + // pError("start filter"); // todo move to the initialization function int32_t code = 0; - bool needFree = false; + bool needFree = false; if (filter == NULL) { needFree = true; code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); @@ -1848,6 +1847,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { taosMemoryFreeClear(pMsg); + taosMemoryFree(pWrapper); qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return pTaskInfo->code; @@ -3094,8 +3094,12 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* } return pOperator; + _error: - destroyAggOperatorInfo(pInfo); + if (pInfo != NULL) { + destroyAggOperatorInfo(pInfo); + } + taosMemoryFreeClear(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -3242,13 +3246,16 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - SSDataBlock* pResBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc); - SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr); + pInfo->pRes = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc); + SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr); + pOperator->exprSupp.pExprInfo = pExprInfo; + pInfo->pNotFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pInfo->numOfNotFillExpr); int32_t code = createWStartTsAsNotFillExpr(pInfo, pPhyFillNode); if (code != TSDB_CODE_SUCCESS) { goto _error; } + SInterval* pInterval = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval @@ -3258,16 +3265,20 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* int32_t type = convertFillType(pPhyFillNode->mode); SResultInfo* pResultInfo = &pOperator->resultInfo; + initResultSizeInfo(&pOperator->resultInfo, 4096); - blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity); - initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; int32_t numOfOutputCols = 0; - SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, - &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); + pInfo->pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, + &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr, (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity, @@ -3276,17 +3287,14 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - pInfo->pRes = pResBlock; - pInfo->pFinalRes = createOneDataBlock(pResBlock, false); + pInfo->pFinalRes = createOneDataBlock(pInfo->pRes, false); blockDataEnsureCapacity(pInfo->pFinalRes, pOperator->resultInfo.capacity); pInfo->pCondition = pPhyFillNode->node.pConditions; - pInfo->pColMatchColInfo = pColMatchColInfo; pOperator->name = "FillOperator"; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; - pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; @@ -3298,8 +3306,11 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* return pOperator; _error: + if (pInfo != NULL) { + destroyFillOperatorInfo(pInfo); + } + taosMemoryFreeClear(pOperator); - taosMemoryFreeClear(pInfo); return NULL; } @@ -3388,7 +3399,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { pSchema->colId = pColNode->colId; pSchema->type = pColNode->node.resType.type; pSchema->bytes = pColNode->node.resType.bytes; - strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); + tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index fe6ee4129a..ddb6d73aab 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -384,6 +384,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy numOfRows = TWOMB / pResBlock->info.rowSize; } + initBasicInfo(&pInfo->binfo, pResBlock); initResultSizeInfo(&pOperator->resultInfo, numOfRows); int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str); @@ -391,7 +392,6 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy goto _error; } - initBasicInfo(&pInfo->binfo, pResBlock); setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr); pInfo->binfo.pRes = pResBlock; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c6bd3532ef..8e1858d8c0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1770,6 +1770,7 @@ FETCH_NEXT_BLOCK: pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; printDataBlock(pDelBlock, "stream scan delete result"); if (pInfo->pDeleteDataRes->info.rows > 0) { + blockDataDestroy(pDelBlock); return pInfo->pDeleteDataRes; } else { goto FETCH_NEXT_BLOCK; @@ -3001,7 +3002,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { while (1) { int64_t startTs = taosGetTimestampUs(); tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); - strcpy(pInfo->req.user, pInfo->pUser); + tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); char* buf1 = taosMemoryCalloc(1, contLen); @@ -3332,6 +3333,11 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags const char* idStr) { int64_t st = taosGetTimestampUs(); + if (pHandle == NULL) { + qError("invalid handle, in creating operator tree", idStr); + return TSDB_CODE_INVALID_PARA; + } + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { qError("failed to getTableList, code: %s", tstrerror(code)); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 0d5f521034..7ca3de5214 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -398,15 +398,16 @@ int32_t finishSortGroup(SOperatorInfo* pOperator) { SGroupSortOperatorInfo* pInfo = pOperator->info; SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pCurrSortHandle); + pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod; pInfo->sortExecInfo.sortBuffer = sortExecInfo.sortBuffer; pInfo->sortExecInfo.loops += sortExecInfo.loops; pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes; pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes; - if (pInfo->pCurrSortHandle != NULL) { - tsortDestroySortHandle(pInfo->pCurrSortHandle); - } + + tsortDestroySortHandle(pInfo->pCurrSortHandle); pInfo->pCurrSortHandle = NULL; + return TSDB_CODE_SUCCESS; } @@ -717,10 +718,9 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size SMultiwayMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwayMergeOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; - SSDataBlock* pResBlock = createResDataBlock(pDescNode); - - int32_t rowSize = pResBlock->info.rowSize; + pInfo->binfo.pRes = createResDataBlock(pDescNode); + int32_t rowSize = pInfo->binfo.pRes->info.rowSize; if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { goto _error; } @@ -734,7 +734,6 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size initResultSizeInfo(&pOperator->resultInfo, 1024); pInfo->groupSort = pMergePhyNode->groupSort; - pInfo->binfo.pRes = pResBlock; pInfo->pSortInfo = pSortInfo; pInfo->pColMatchInfo = pColMatchColInfo; pInfo->pInputBlock = pInputBlock; @@ -761,7 +760,10 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size _error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pInfo); + if (pInfo != NULL) { + destroyMultiwayMergeOperatorInfo(pInfo); + } + taosMemoryFree(pOperator); return NULL; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index cbbd874bb9..199306ed36 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1480,7 +1480,7 @@ static void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, STimeWindowAgg for (int32_t i = 0; i < pBlock->info.rows; i++) { TSKEY startTs = TMAX(tsStarts[i], pTwSup->minTs); TSKEY endTs = TMIN(tsEnds[i], pTwSup->maxTs); - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTs, pInterval, TSDB_ORDER_ASC); do { @@ -1506,8 +1506,9 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* pGpDatas = (uint64_t*)pGpCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); while (win.ekey <= endTsCols[i]) { uint64_t winGpId = pGpDatas[i]; @@ -2748,11 +2749,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi goto _error; } - int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); - SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc); int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; - SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; if (pStateNode->window.pExprs != NULL) { @@ -2775,6 +2772,10 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); + SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc); + initResultSizeInfo(&pOperator->resultInfo, 4096); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { @@ -3618,14 +3619,13 @@ void destroyStreamSessionAggOperatorInfo(void* param) { int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock) { + initBasicInfo(pBasicInfo, pResultBlock); int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); if (code != TSDB_CODE_SUCCESS) { return code; } + initStreamFunciton(pSup->pCtx, pSup->numOfExprs); - - initBasicInfo(pBasicInfo, pResultBlock); - for (int32_t i = 0; i < numOfCols; ++i) { pSup->pCtx[i].saveHandle.pBuf = NULL; } @@ -3665,9 +3665,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh SExecTaskInfo* pTaskInfo) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); int32_t code = TSDB_CODE_OUT_OF_MEMORY; + SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3685,8 +3684,11 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh goto _error; } } + SExprSupp* pSup = &pOperator->exprSupp; + SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -3701,8 +3703,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh if (pInfo->pDummyCtx == NULL) { goto _error; } - initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols); + initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols); pInfo->twAggSup = (STimeWindowAggSupp){ .waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType, @@ -5004,14 +5006,14 @@ int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFu SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode; - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = TSDB_CODE_SUCCESS; SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } @@ -5025,7 +5027,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys if (pStateNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); - int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -5040,6 +5042,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys }; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -5615,7 +5618,6 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); - SSDataBlock* pResBlock = createResDataBlock(pIntervalPhyNode->window.node.pOutputDataBlockDesc); SInterval interval = {.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, @@ -5644,6 +5646,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge goto _error; } + SSDataBlock* pResBlock = createResDataBlock(pIntervalPhyNode->window.node.pOutputDataBlockDesc); initBasicInfo(&pIntervalInfo->binfo, pResBlock); initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win); @@ -5675,7 +5678,10 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge return pOperator; _error: - destroyMergeIntervalOperatorInfo(pMergeIntervalInfo); + if (pMergeIntervalInfo != NULL) { + destroyMergeIntervalOperatorInfo(pMergeIntervalInfo); + } + taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; @@ -5827,8 +5833,6 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->isFinal = false; - - pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); SExprSupp* pSup = &pOperator->exprSupp; @@ -5837,8 +5841,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys initStreamFunciton(pSup->pCtx, pSup->numOfExprs); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 824e988c2f..b133041fdc 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -220,7 +220,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) { pHashObj->pBucket[pHashObj->numOfBuckets] = pBucket; pBucket->pPageIdList = taosArrayInit(2, sizeof(int32_t)); - if (pBucket->pPageIdList == NULL || pBucket == NULL) { + if (pBucket->pPageIdList == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -256,6 +256,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ int32_t code = createDiskbasedBuf(&pHashObj->pBuf, pageSize, inMemPages * pageSize, "", tsTempDir); if (code != 0) { + taosMemoryFree(pHashObj); terrno = code; return NULL; } @@ -351,7 +352,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data char* pStart = p->data; while (pStart - ((char*)p) < p->num) { SLHashNode* pNode = (SLHashNode*)pStart; - ASSERT(pNode->keyLen > 0 && pNode->dataLen >= 0); + ASSERT(pNode->keyLen > 0); char* k = GET_LHASH_NODE_KEY(pNode); int32_t hashv = pHashObj->hashFn(k, pNode->keyLen); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 74da5c45ee..85582cbd39 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -548,7 +548,8 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { SSDataBlock* pBlock = createOneDataBlock(pHandle->pDataBlock, false); code = doAddNewExternalMemSource(pHandle->pBuf, pResList, pBlock, &pHandle->sourceId, pPageIdList); - if (code != 0) { + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pResList); return code; } } From 9fe6d23462a41390493bf3bb81181a2302cc38d6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 18 Oct 2022 11:49:03 +0800 Subject: [PATCH 094/135] feat:add new interface for schemaless --- source/client/src/clientSml.c | 16 +++++-- source/client/test/smlTest.cpp | 28 +++++------ utils/test/c/sml_test.c | 85 ++++++++++++++++++++++------------ 3 files changed, 82 insertions(+), 47 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index fe1946d77d..9241b3c5d4 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -79,7 +79,7 @@ #define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " #define MAX_RETRY_TIMES 5 -#define LINE_BATCH 2000 +#define LINE_BATCH 2 //================================================================================================= typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; @@ -2199,7 +2199,7 @@ static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { } if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ret = smlParseTelnetString(info, (const char *)data, POINTER_SHIFT(data, len), tinfo, cols); + ret = smlParseTelnetString(info, (const char *)data, (char*)data + len, tinfo, cols); } else if (info->protocol == TSDB_SML_JSON_PROTOCOL) { ret = smlParseJSONString(info, (cJSON *)data, tinfo, cols); } else { @@ -2408,6 +2408,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char* rawLine, char } len++; } + if(info->protocol == TSDB_SML_LINE_PROTOCOL && tmp[0] == '#'){ // this line is comment + continue; + } } if (info->protocol == TSDB_SML_LINE_PROTOCOL) { @@ -2418,7 +2421,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char* rawLine, char ASSERT(0); } if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, lines[i]); + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); return code; } } @@ -2692,11 +2695,16 @@ TAOS_RES *taos_schemaless_insert_raw(TAOS* taos, char* lines, int len, int32_t * } int numLines = 0; + *totalRows = 0; + char *tmp = lines; for(int i = 0; i < len; i++){ if(lines[i] == '\n' || i == len - 1){ numLines++; + if(tmp[0] != '#' || protocol != TSDB_SML_LINE_PROTOCOL){ //ignore comment + (*totalRows)++; + } + tmp = lines + i + 1; } } - *totalRows = numLines; return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision); } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index ab4442eca7..54778e87a7 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -44,7 +44,7 @@ TEST(testCase, smlParseInfluxString_Test) { char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, tmp, strlen(tmp) + 1); - int ret = smlParseInfluxString(sql, &elements, &msgBuf); + int ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen(",st")); @@ -63,14 +63,14 @@ TEST(testCase, smlParseInfluxString_Test) { tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_NE(ret, 0); // case 3 false tmp = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1); ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); @@ -79,7 +79,7 @@ TEST(testCase, smlParseInfluxString_Test) { tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -98,7 +98,7 @@ TEST(testCase, smlParseInfluxString_Test) { tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql + 1); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -116,21 +116,21 @@ TEST(testCase, smlParseInfluxString_Test) { tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); // case 7 tmp = " st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); // case 8 false tmp = ", st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_NE(ret, 0); taosMemoryFree(sql); } @@ -542,7 +542,7 @@ TEST(testCase, smlParseTelnetLine_error_Test) { "sys.procs.running 1479496100 42 host= web01", }; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - int ret = smlParseTelnetLine(info, (void *)sql[i]); + int ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); ASSERT_NE(ret, 0); } @@ -561,7 +561,7 @@ TEST(testCase, smlParseTelnetLine_diff_type_Test) { int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i]); + ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); if (ret != TSDB_CODE_SUCCESS) break; } ASSERT_NE(ret, 0); @@ -617,7 +617,7 @@ TEST(testCase, smlParseTelnetLine_json_error_Test) { int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i]); + ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); ASSERT_NE(ret, 0); } @@ -653,7 +653,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i]); + ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); if (ret != TSDB_CODE_SUCCESS) break; } ASSERT_NE(ret, 0); @@ -688,7 +688,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { }; int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i]); + ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); if (ret != TSDB_CODE_SUCCESS) break; } ASSERT_NE(ret, 0); @@ -1002,7 +1002,7 @@ TEST(testCase, sml_col_4096_Test) { int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseInfluxLine(info, sql[i]); + ret = smlParseInfluxLine(info, sql[i], strlen(sql[i])); if (ret != TSDB_CODE_SUCCESS) break; } ASSERT_NE(ret, 0); diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index b6d8d75ba0..09079d3a26 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1155,37 +1155,64 @@ int smlProcess_18784_Test() { return code; } +int sml_19221_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000", + }; + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + void* tmp = taosMemoryCalloc(256, 1); + memcpy(tmp, sql[0], strlen(sql[0])); + *(char*)(tmp+44) = 0; + int32_t totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + int main(int argc, char *argv[]) { int ret = 0; - ret = smlProcess_influx_Test(); - ASSERT(!ret); - ret = smlProcess_telnet_Test(); - ASSERT(!ret); - ret = smlProcess_json1_Test(); - ASSERT(!ret); - ret = smlProcess_json2_Test(); - ASSERT(!ret); - ret = smlProcess_json3_Test(); - ASSERT(!ret); - ret = smlProcess_json4_Test(); - ASSERT(!ret); - ret = sml_TD15662_Test(); - ASSERT(!ret); - ret = sml_TD15742_Test(); - ASSERT(!ret); - ret = sml_16384_Test(); - ASSERT(!ret); - ret = sml_oom_Test(); - ASSERT(!ret); - ret = sml_16368_Test(); - ASSERT(!ret); - ret = sml_dup_time_Test(); - ASSERT(!ret); - ret = sml_16960_Test(); - ASSERT(!ret); - ret = sml_add_tag_col_Test(); - ASSERT(!ret); - ret = smlProcess_18784_Test(); +// ret = smlProcess_influx_Test(); +// ASSERT(!ret); +// ret = smlProcess_telnet_Test(); +// ASSERT(!ret); +// ret = smlProcess_json1_Test(); +// ASSERT(!ret); +// ret = smlProcess_json2_Test(); +// ASSERT(!ret); +// ret = smlProcess_json3_Test(); +// ASSERT(!ret); +// ret = smlProcess_json4_Test(); +// ASSERT(!ret); +// ret = sml_TD15662_Test(); +// ASSERT(!ret); +// ret = sml_TD15742_Test(); +// ASSERT(!ret); +// ret = sml_16384_Test(); +// ASSERT(!ret); +// ret = sml_oom_Test(); +// ASSERT(!ret); +// ret = sml_16368_Test(); +// ASSERT(!ret); +// ret = sml_dup_time_Test(); +// ASSERT(!ret); +// ret = sml_16960_Test(); +// ASSERT(!ret); +// ret = sml_add_tag_col_Test(); +// ASSERT(!ret); +// ret = smlProcess_18784_Test(); +// ASSERT(!ret); + ret = sml_19221_Test(); ASSERT(!ret); return ret; } From 2e4dca1171176b7f8f2da9adfd18df76d754c611 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Oct 2022 11:53:22 +0800 Subject: [PATCH 095/135] fix: remove a typo --- source/libs/executor/src/sortoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 1707bf9480..7ca3de5214 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -758,7 +758,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size } return pOperator; -_error: 3 +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; if (pInfo != NULL) { destroyMultiwayMergeOperatorInfo(pInfo); From ab7a00abbe5d7833bc2891b1bb1aec0ffe36806d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 18 Oct 2022 11:54:30 +0800 Subject: [PATCH 096/135] fix: fix case when crash issue --- source/libs/executor/src/timewindowoperator.c | 3 +++ source/libs/scalar/src/scalar.c | 14 ++++++++++++++ source/libs/scalar/src/sclvector.c | 2 +- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index cbbd874bb9..55a8f16698 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1724,6 +1724,9 @@ static void destroyStateWindowOperatorInfo(void* param) { cleanupBasicInfo(&pInfo->binfo); taosMemoryFreeClear(pInfo->stateKey.pData); cleanupExprSupp(&pInfo->scalarSup); + colDataDestroy(&pInfo->twAggSup.timeWindowData); + cleanupAggSup(&pInfo->aggSup); + cleanupGroupResInfo(&pInfo->groupResInfo); taosMemoryFreeClear(param); } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 1f09fd4799..cc1949f17d 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -629,6 +629,8 @@ _return: sclFreeParam(pWhen); sclFreeParam(pThen); + taosMemoryFree(pWhen); + taosMemoryFree(pThen); SCL_RET(code); } @@ -664,6 +666,8 @@ int32_t sclWalkWhenList(SScalarCtx *ctx, SNodeList* pList, struct SListCell* pCe sclFreeParam(pWhen); sclFreeParam(pThen); + taosMemoryFreeClear(pWhen); + taosMemoryFreeClear(pThen); } if (pElse) { @@ -688,6 +692,8 @@ _return: sclFreeParam(pWhen); sclFreeParam(pThen); + taosMemoryFree(pWhen); + taosMemoryFree(pThen); SCL_RET(code); } @@ -929,6 +935,10 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp sclFreeParam(&comp); sclFreeParam(pWhen); sclFreeParam(pThen); + taosMemoryFree(pCase); + taosMemoryFree(pElse); + taosMemoryFree(pWhen); + taosMemoryFree(pThen); return TSDB_CODE_SUCCESS; @@ -940,6 +950,10 @@ _return: sclFreeParam(pWhen); sclFreeParam(pThen); sclFreeParam(output); + taosMemoryFree(pCase); + taosMemoryFree(pElse); + taosMemoryFree(pWhen); + taosMemoryFree(pThen); SCL_RET(code); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f467721248..91039ed577 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -654,7 +654,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam* pIn, SScalarParam* pOut, return TSDB_CODE_APP_ERROR; } - int32_t rstart = startIndex >= 0 ? startIndex : 0; + int32_t rstart = (startIndex >= 0 && startIndex < pIn->numOfRows) ? startIndex : 0; int32_t rend = numOfRows > 0 ? rstart + numOfRows - 1 : rstart + pIn->numOfRows - 1; SSclVectorConvCtx cCtx = {pIn, pOut, rstart, rend, pInputCol->info.type, pOutputCol->info.type}; From f76261dcb567fcdd89e561a9a62fde0a5c3f5666 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Oct 2022 12:31:15 +0800 Subject: [PATCH 097/135] fix: become leader if replica num is 1 --- source/dnode/mnode/impl/src/mndMain.c | 19 ++++++++++++------- source/dnode/mnode/impl/src/mndMnode.c | 6 ++++++ source/libs/sync/src/syncElection.c | 13 +++++++++++++ source/libs/sync/src/syncMain.c | 2 +- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 5fb23b045c..8314601df3 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -603,22 +603,27 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { return 0; } if (mndAcquireRpc(pMsg->info.node) == 0) return 0; + + SMnode *pMnode = pMsg->info.node; + const char *role = syncGetMyRoleStr(pMnode->syncMgmt.sync); + bool restored = syncIsRestoreFinish(pMnode->syncMgmt.sync); if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { + mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, + pMnode->stopped, restored, role); return -1; } - SEpSet epSet = {0}; - SMnode *pMnode = pMsg->info.node; + const STraceId *trace = &pMsg->info.traceId; + SEpSet epSet = {0}; mndGetMnodeEpSet(pMnode, &epSet); - const STraceId *trace = &pMsg->info.traceId; mDebug( - "msg:%p, failed to check mnode state since %s, mnode restored:%d stopped:%d, sync restored:%d role:%s type:%s " - "numOfEps:%d inUse:%d", - pMsg, terrstr(), pMnode->restored, pMnode->stopped, syncIsRestoreFinish(pMnode->syncMgmt.sync), - syncGetMyRoleStr(pMnode->syncMgmt.sync), TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); + "msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d " + "role:%s, redirect numOfEps:%d inUse:%d", + pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, restored, role, epSet.numOfEps, + epSet.inUse); if (epSet.numOfEps > 0) { for (int32_t i = 0; i < epSet.numOfEps; ++i) { diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 63ce538536..f25d436aee 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -747,7 +747,13 @@ static void mndReloadSyncConfig(SMnode *pMnode) { } if (cfg.myIndex == -1) { +#if 1 mInfo("vgId:1, mnode sync not reconfig since selfIndex is -1"); +#else + // cannot reconfig because the leader may fail to elect after reboot + mInfo("vgId:1, mnode sync not reconfig since selfIndex is -1, do sync stop oper"); + syncStop(pMnode->syncMgmt.sync); +#endif return; } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 375f2e5730..3f13249ce6 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -96,6 +96,19 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { syncNodeCandidate2Leader(pSyncNode); pSyncNode->pVotesGranted->toLeader = true; return ret; + } + + if (pSyncNode->replicaNum == 1) { + // only myself, to leader + voteGrantedUpdate(pSyncNode->pVotesGranted, pSyncNode); + votesRespondUpdate(pSyncNode->pVotesRespond, pSyncNode); + + pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum); + + syncNodeCandidate2Leader(pSyncNode); + pSyncNode->pVotesGranted->toLeader = true; + return ret; + } switch (pSyncNode->pRaftCfg->snapshotStrategy) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b255d83933..83bd7a5f38 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2282,7 +2282,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); - ASSERT(voteGrantedMajority(pSyncNode->pVotesGranted)); + //ASSERT(voteGrantedMajority(pSyncNode->pVotesGranted)); syncNodeBecomeLeader(pSyncNode, "candidate to leader"); syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode); From a43201d2c27b8dee52514457736cd788558f06ac Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Tue, 18 Oct 2022 13:09:47 +0800 Subject: [PATCH 098/135] Update tmqAutoCreateTbl.py --- tests/system-test/7-tmq/tmqAutoCreateTbl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index 587472ed82..ce2cad4b14 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -160,7 +160,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 1000, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} From 7b84b217c2cba993bead5af81c658162c51cda95 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 18 Oct 2022 13:46:42 +0800 Subject: [PATCH 099/135] opti:add auto compile --- source/client/src/clientSml.c | 2 +- utils/test/c/sml_test.c | 65 ++++++++++++++++++----------------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 9241b3c5d4..f2739c9ab8 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -79,7 +79,7 @@ #define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " #define MAX_RETRY_TIMES 5 -#define LINE_BATCH 2 +#define LINE_BATCH 2000 //================================================================================================= typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 09079d3a26..1066054b7d 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1162,18 +1162,19 @@ int sml_19221_Test() { taos_free_result(pRes); const char *sql[] = { - "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000", + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000", }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - void* tmp = taosMemoryCalloc(256, 1); + char* tmp = (char*)taosMemoryCalloc(1024, 1); memcpy(tmp, sql[0], strlen(sql[0])); *(char*)(tmp+44) = 0; int32_t totalRows = 0; pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + ASSERT(totalRows == 3); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); @@ -1182,36 +1183,36 @@ int sml_19221_Test() { int main(int argc, char *argv[]) { int ret = 0; -// ret = smlProcess_influx_Test(); -// ASSERT(!ret); -// ret = smlProcess_telnet_Test(); -// ASSERT(!ret); -// ret = smlProcess_json1_Test(); -// ASSERT(!ret); -// ret = smlProcess_json2_Test(); -// ASSERT(!ret); -// ret = smlProcess_json3_Test(); -// ASSERT(!ret); -// ret = smlProcess_json4_Test(); -// ASSERT(!ret); -// ret = sml_TD15662_Test(); -// ASSERT(!ret); -// ret = sml_TD15742_Test(); -// ASSERT(!ret); -// ret = sml_16384_Test(); -// ASSERT(!ret); -// ret = sml_oom_Test(); -// ASSERT(!ret); -// ret = sml_16368_Test(); -// ASSERT(!ret); -// ret = sml_dup_time_Test(); -// ASSERT(!ret); -// ret = sml_16960_Test(); -// ASSERT(!ret); -// ret = sml_add_tag_col_Test(); -// ASSERT(!ret); -// ret = smlProcess_18784_Test(); -// ASSERT(!ret); + ret = smlProcess_influx_Test(); + ASSERT(!ret); + ret = smlProcess_telnet_Test(); + ASSERT(!ret); + ret = smlProcess_json1_Test(); + ASSERT(!ret); + ret = smlProcess_json2_Test(); + ASSERT(!ret); + ret = smlProcess_json3_Test(); + ASSERT(!ret); + ret = smlProcess_json4_Test(); + ASSERT(!ret); + ret = sml_TD15662_Test(); + ASSERT(!ret); + ret = sml_TD15742_Test(); + ASSERT(!ret); + ret = sml_16384_Test(); + ASSERT(!ret); + ret = sml_oom_Test(); + ASSERT(!ret); + ret = sml_16368_Test(); + ASSERT(!ret); + ret = sml_dup_time_Test(); + ASSERT(!ret); + ret = sml_16960_Test(); + ASSERT(!ret); + ret = sml_add_tag_col_Test(); + ASSERT(!ret); + ret = smlProcess_18784_Test(); + ASSERT(!ret); ret = sml_19221_Test(); ASSERT(!ret); return ret; From aa3551887c04158c5681de0521f230247b36866e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 18 Oct 2022 14:03:42 +0800 Subject: [PATCH 100/135] fix: taosbenchmark coverity scan c64858f (#17427) * fix: taosbenchmark coverity scan issues * fix: taosbenchmark default json case * fix: update taos-tools 217a267 * fix: taosbenchmark coverity scan issues 2849aa4 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 73409a950a..ea4d48a2b1 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 4d02980 + GIT_TAG c64858f SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 029abe73963bd672b97f6218c641bb1cfce713b7 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 18 Oct 2022 14:32:18 +0800 Subject: [PATCH 101/135] fix: coverity issues --- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/libs/tdb/src/db/tdbBtree.c | 1 + source/libs/tdb/src/db/tdbDb.c | 6 +++--- source/libs/tdb/src/db/tdbPCache.c | 2 +- source/libs/tdb/src/db/tdbPager.c | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 2b7982d381..c9a277ae1f 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -51,7 +51,7 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema return -1; } - strncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN); + tstrncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN); pMetaRsp->numOfColumns = pSchema->nCols; pMetaRsp->tableType = TSDB_NORMAL_TABLE; pMetaRsp->sversion = pSchema->version; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 9e17f50dce..8f11fb88a2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1086,6 +1086,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const // fetch next ofp, a new ofp and make it dirty ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt); if (ret < 0) { + tdbFree(pBuf); return -1; } } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 6c01348bc2..a2803d3ecf 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -106,7 +106,7 @@ int32_t tdbBegin(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerBegin(pPager, pTxn); if (ret < 0) { - tdbError("failed to begin pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); + tdbError("failed to begin pager since %s. dbName:%s, txnId:%ld", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -121,7 +121,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerCommit(pPager, pTxn); if (ret < 0) { - tdbError("failed to commit pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); + tdbError("failed to commit pager since %s. dbName:%s, txnId:%ld", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -136,7 +136,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerAbort(pPager, pTxn); if (ret < 0) { - tdbError("failed to abort pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); + tdbError("failed to abort pager since %s. dbName:%s, txnId:%ld", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index c73ddce74c..732a57639f 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -268,7 +268,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) // 4. Try a create new page if (!pPage) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); - if (ret < 0 && pPage != NULL) { + if (ret < 0 || pPage == NULL) { // TODO ASSERT(0); return NULL; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 57ad2a783a..90332617cd 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -541,7 +541,7 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { - tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, + tdbError("failed to write page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->jFileName, pPage->pageSize); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -570,7 +570,7 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize); if (ret < 0) { - tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName, + tdbError("failed to write page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, pPage->pageSize); terrno = TAOS_SYSTEM_ERROR(errno); return -1; From bce4fc9cf86a89afd5242e862690fc7a7e4956c7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 18 Oct 2022 14:43:06 +0800 Subject: [PATCH 102/135] feat:add new interface for schemaless --- utils/test/c/sml_test.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 1066054b7d..f7fd59cd7b 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -63,6 +63,7 @@ int smlProcess_influx_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); return code; } @@ -86,6 +87,8 @@ int smlProcess_telnet_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -125,6 +128,8 @@ int smlProcess_json1_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -165,6 +170,8 @@ int smlProcess_json2_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -233,6 +240,8 @@ int smlProcess_json3_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -292,6 +301,8 @@ int smlProcess_json4_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -313,6 +324,8 @@ int sml_TD15662_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -333,6 +346,8 @@ int sml_TD15742_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -362,6 +377,8 @@ int sml_16384_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -781,6 +798,8 @@ int sml_oom_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -825,6 +844,8 @@ int sml_16368_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -862,6 +883,8 @@ int sml_dup_time_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -1068,6 +1091,8 @@ int sml_16960_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -1097,6 +1122,7 @@ int sml_add_tag_col_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); return code; } @@ -1151,6 +1177,7 @@ int smlProcess_18784_Test() { rowIndex++; } taos_free_result(pRes); + taos_close(taos); return code; } @@ -1178,6 +1205,9 @@ int sml_19221_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + taosMemoryFree(tmp); + return code; } From 98306426f0ed6c3b51173baa43339843143fcabb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 18 Oct 2022 14:47:02 +0800 Subject: [PATCH 103/135] fix:assert core if db is empty --- source/client/src/clientMsgHandler.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index cdf977bea3..8680f93f8c 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -194,6 +194,10 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { SUseDbRsp usedbRsp = {0}; tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); + if(strlen(usedbRsp.db) == 0){ + return TSDB_CODE_MND_DB_NOT_EXIST; + } + SName name = {0}; tNameFromString(&name, usedbRsp.db, T_NAME_ACCT | T_NAME_DB); From 0f6ba60da4dcae5ee23e082a68f94b72cd9c52ea Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Oct 2022 14:49:43 +0800 Subject: [PATCH 104/135] fix: wrong unlock commands --- source/dnode/mnode/sdb/src/sdbHash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 1e56eb87bf..85e937fcc3 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -203,7 +203,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); } - sdbUnLock(pSdb, type); + // sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pNewRow, false); pSdb->tableVer[pOldRow->type]++; From 0cbaa03fcc4cc6cc3ac836f89215d3ad4b929db0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 14:51:19 +0800 Subject: [PATCH 105/135] fix: osDefaultInit on all platform in testing Init at sut.cpp --- source/dnode/mgmt/test/sut/src/sut.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index 699203e8c1..a4d2e46881 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -43,9 +43,7 @@ void Testbase::InitLog(const char* path) { } void Testbase::Init(const char* path, int16_t port) { -#ifdef _TD_DARWIN_64 osDefaultInit(); -#endif tsServerPort = port; strcpy(tsLocalFqdn, "localhost"); snprintf(tsLocalEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); From eaa2a344e1d78fbcb69a64f1710881cfaa989f99 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 18 Oct 2022 15:01:52 +0800 Subject: [PATCH 106/135] fix: plan problem with 'partition by tag group by tag' statement --- source/libs/planner/src/planOptimizer.c | 31 +++++++++++++++++---- source/libs/planner/test/planPartByTest.cpp | 2 ++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index d6110d9fcd..ccae1f54be 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -124,8 +124,8 @@ static void optSetParentOrder(SLogicNode* pNode, EOrder order) { EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { - *((bool*)pContext) = - (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); + *((bool*)pContext) = + (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; @@ -1520,11 +1520,15 @@ static bool partTagsHasIndefRowsSelectFunc(SNodeList* pFuncs) { return false; } -static int32_t partTagsRewriteGroupTagsToFuncs(SNodeList* pGroupTags, SNodeList* pAggFuncs) { +static int32_t partTagsRewriteGroupTagsToFuncs(SNodeList* pGroupTags, int32_t start, SNodeList* pAggFuncs) { bool hasIndefRowsSelectFunc = partTagsHasIndefRowsSelectFunc(pAggFuncs); int32_t code = TSDB_CODE_SUCCESS; + int32_t index = 0; SNode* pNode = NULL; FOREACH(pNode, pGroupTags) { + if (index++ < start) { + continue; + } if (hasIndefRowsSelectFunc) { code = nodesListStrictAppend(pAggFuncs, partTagsCreateWrapperFunc("_select_value", pNode)); } else { @@ -1559,20 +1563,35 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub } } else { SAggLogicNode* pAgg = (SAggLogicNode*)pNode; + int32_t start = -1; SNode* pGroupKey = NULL; FOREACH(pGroupKey, pAgg->pGroupKeys) { - code = nodesListMakeStrictAppend( - &pScan->pGroupTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0))); + SNode* pGroupExpr = nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0); + if (NULL != pScan->pGroupTags) { + SNode* pGroupTag = NULL; + FOREACH(pGroupTag, pScan->pGroupTags) { + if (nodesEqualNode(pGroupTag, pGroupExpr)) { + continue; + } + } + } + if (start < 0) { + start = LIST_LENGTH(pScan->pGroupTags); + } + code = nodesListMakeStrictAppend(&pScan->pGroupTags, nodesCloneNode(pGroupExpr)); if (TSDB_CODE_SUCCESS != code) { break; } } NODES_DESTORY_LIST(pAgg->pGroupKeys); - code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, pAgg->pAggFuncs); + if (TSDB_CODE_SUCCESS == code && start >= 0) { + code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, start, pAgg->pAggFuncs); + } } if (TSDB_CODE_SUCCESS == code) { code = partTagsOptRebuildTbanme(pScan->pGroupTags); } + pCxt->optimized = true; return code; } diff --git a/source/libs/planner/test/planPartByTest.cpp b/source/libs/planner/test/planPartByTest.cpp index 48ab1e1ac2..3d0467dffe 100644 --- a/source/libs/planner/test/planPartByTest.cpp +++ b/source/libs/planner/test/planPartByTest.cpp @@ -61,6 +61,8 @@ TEST_F(PlanPartitionByTest, withGroupBy) { run("SELECT COUNT(*) FROM t1 PARTITION BY c1 GROUP BY c2"); run("SELECT TBNAME, c1 FROM st1 PARTITION BY TBNAME GROUP BY c1"); + + run("SELECT COUNT(*) FROM t1 PARTITION BY TBNAME GROUP BY TBNAME"); } TEST_F(PlanPartitionByTest, withTimeLineFunc) { From 41866ed397e6e6c3096cf1be3099585964f56469 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 18 Oct 2022 15:13:52 +0800 Subject: [PATCH 107/135] fix:defeats in converity scan --- source/client/src/clientSml.c | 10 ++- utils/test/c/sml_test.c | 141 ++-------------------------------- utils/test/c/tmqDemo.c | 10 +-- utils/test/c/tmqSim.c | 2 +- 4 files changed, 21 insertions(+), 142 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5c37822222..904ad32d0a 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1371,8 +1371,14 @@ static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) { static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { SHashObj *s1 = *(SHashObj **)key1; SHashObj *s2 = *(SHashObj **)key2; - SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN); - SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN); + SSmlKv **kv1pp = (SSmlKv **)taosHashGet(s1, TS, TS_LEN); + SSmlKv **kv2pp = (SSmlKv **)taosHashGet(s2, TS, TS_LEN); + if(!kv1pp || !kv2pp){ + uError("smlKvTimeHashCompare kv is null"); + return -1; + } + SSmlKv *kv1 = *kv1pp; + SSmlKv *kv2 = *kv2pp; if(!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP){ uError("smlKvTimeHashCompare kv1"); return -1; diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index b6d8d75ba0..a9dbce62e5 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -99,27 +99,8 @@ int smlProcess_json1_Test() { taos_free_result(pRes); const char *sql[] = { - "[" - " {" - " \"metric\": \"sys.cpu.nice\"," - " \"timestamp\": 0," - " \"value\": 18," - " \"tags\": {" - " \"host\": \"web01\"," - " \"id\": \"t1\"," - " \"dc\": \"lga\"" - " }" - " }," - " {" - " \"metric\": \"sys.cpu.nice\"," - " \"timestamp\": 1662344042," - " \"value\": 9," - " \"tags\": {" - " \"host\": \"web02\"," - " \"dc\": \"lga\"" - " }" - " }" - "]",}; + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}}]" + }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -138,28 +119,8 @@ int smlProcess_json2_Test() { taos_free_result(pRes); const char *sql[] = { - "{" - " \"metric\": \"meter_current0\"," - " \"timestamp\": {" - " \"value\" : 1662344042," - " \"type\" : \"s\"" - " }," - " \"value\": {" - " \"value\" : 10.3," - " \"type\" : \"i64\"" - " }," - " \"tags\": {" - " \"groupid\": { " - " \"value\" : 2," - " \"type\" : \"bigint\"" - " }," - " \"location\": { " - " \"value\" : \"北京\"," - " \"type\" : \"binary\"" - " }," - " \"id\": \"d1001\"" - " }" - "}",}; + "{\"metric\":\"meter_current0\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"groupid\":{\"value\":2,\"type\":\"bigint\"},\"location\":{\"value\":\"北京\",\"type\":\"binary\"},\"id\":\"d1001\"}}" + }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -178,56 +139,7 @@ int smlProcess_json3_Test() { taos_free_result(pRes); const char *sql[] = { - "{" - " \"metric\": \"meter_current1\"," - " \"timestamp\": {" - " \"value\" : 1662344042," - " \"type\" : \"s\"" - " }," - " \"value\": {" - " \"value\" : 10.3," - " \"type\" : \"i64\"" - " }," - " \"tags\": {" - " \"t1\": { " - " \"value\" : 2," - " \"type\" : \"bigint\"" - " }," - " \"t2\": { " - " \"value\" : 2," - " \"type\" : \"int\"" - " }," - " \"t3\": { " - " \"value\" : 2," - " \"type\" : \"i16\"" - " }," - " \"t4\": { " - " \"value\" : 2," - " \"type\" : \"i8\"" - " }," - " \"t5\": { " - " \"value\" : 2," - " \"type\" : \"f32\"" - " }," - " \"t6\": { " - " \"value\" : 2," - " \"type\" : \"double\"" - " }," - " \"t7\": { " - " \"value\" : \"8323\"," - " \"type\" : \"binary\"" - " }," - " \"t8\": { " - " \"value\" : \"北京\"," - " \"type\" : \"nchar\"" - " }," - " \"t9\": { " - " \"value\" : true," - " \"type\" : \"bool\"" - " }," - " \"id\": \"d1001\"" - " }" - "}",}; + "{\"metric\":\"meter_current1\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"t1\":{\"value\":2,\"type\":\"bigint\"},\"t2\":{\"value\":2,\"type\":\"int\"},\"t3\":{\"value\":2,\"type\":\"i16\"},\"t4\":{\"value\":2,\"type\":\"i8\"},\"t5\":{\"value\":2,\"type\":\"f32\"},\"t6\":{\"value\":2,\"type\":\"double\"},\"t7\":{\"value\":\"8323\",\"type\":\"binary\"},\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":{\"value\":true,\"type\":\"bool\"},\"id\":\"d1001\"}}"}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -246,47 +158,8 @@ int smlProcess_json4_Test() { taos_free_result(pRes); const char *sql[] = { - "{" - " \"metric\": \"meter_current2\"," - " \"timestamp\": {" - " \"value\" : 1662344042000," - " \"type\" : \"ms\"" - " }," - " \"value\": \"ni\"," - " \"tags\": {" - " \"t1\": { " - " \"value\" : 20," - " \"type\" : \"i64\"" - " }," - " \"t2\": { " - " \"value\" : 25," - " \"type\" : \"i32\"" - " }," - " \"t3\": { " - " \"value\" : 2," - " \"type\" : \"smallint\"" - " }," - " \"t4\": { " - " \"value\" : 2," - " \"type\" : \"tinyint\"" - " }," - " \"t5\": { " - " \"value\" : 2," - " \"type\" : \"float\"" - " }," - " \"t6\": { " - " \"value\" : 0.2," - " \"type\" : \"f64\"" - " }," - " \"t7\": \"nsj\"," - " \"t8\": { " - " \"value\" : \"北京\"," - " \"type\" : \"nchar\"" - " }," - " \"t9\": false," - " \"id\": \"d1001\"" - " }" - "}",}; + "{\"metric\":\"meter_current2\",\"timestamp\":{\"value\":1662344042000,\"type\":\"ms\"},\"value\":\"ni\",\"tags\":{\"t1\":{\"value\":20,\"type\":\"i64\"},\"t2\":{\"value\":25,\"type\":\"i32\"},\"t3\":{\"value\":2,\"type\":\"smallint\"},\"t4\":{\"value\":2,\"type\":\"tinyint\"},\"t5\":{\"value\":2,\"type\":\"float\"},\"t6\":{\"value\":0.2,\"type\":\"f64\"},\"t7\":\"nsj\",\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":false,\"id\":\"d1001\"}}" + }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); diff --git a/utils/test/c/tmqDemo.c b/utils/test/c/tmqDemo.c index 784b45c92b..d105b50579 100644 --- a/utils/test/c/tmqDemo.c +++ b/utils/test/c/tmqDemo.c @@ -130,15 +130,15 @@ void parseArgument(int32_t argc, char* argv[]) { printHelp(); exit(0); } else if (strcmp(argv[i], "-d") == 0) { - strcpy(g_stConfInfo.dbName, argv[++i]); + tstrncpy(g_stConfInfo.dbName, argv[++i], sizeof(g_stConfInfo.dbName)); } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); + tstrncpy(configDir, argv[++i], PATH_MAX); } else if (strcmp(argv[i], "-s") == 0) { - strcpy(g_stConfInfo.stbName, argv[++i]); + tstrncpy(g_stConfInfo.stbName, argv[++i], sizeof(g_stConfInfo.stbName)); } else if (strcmp(argv[i], "-w") == 0) { - strcpy(g_stConfInfo.vnodeWalPath, argv[++i]); + tstrncpy(g_stConfInfo.vnodeWalPath, argv[++i], sizeof(g_stConfInfo.vnodeWalPath)); } else if (strcmp(argv[i], "-f") == 0) { - strcpy(g_stConfInfo.resultFileName, argv[++i]); + tstrncpy(g_stConfInfo.resultFileName, argv[++i], sizeof(g_stConfInfo.resultFileName)); } else if (strcmp(argv[i], "-t") == 0) { g_stConfInfo.numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index d36fe0855a..25e83363ee 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -949,7 +949,7 @@ void parseConsumeInfo() { token = strtok(g_stConfInfo.stThreads[i].topicString, delim); while (token != NULL) { // printf("%s\n", token ); - strcpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token); + tstrncpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token, sizeof(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic])); ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); g_stConfInfo.stThreads[i].numOfTopic++; From 9078f6157c763e6976ce836229efd450edb593a4 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 18 Oct 2022 15:37:00 +0800 Subject: [PATCH 108/135] fix: shell input Chinese char error --- tools/shell/src/shellCommand.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 0b26d685fd..c34b22b691 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -101,11 +101,8 @@ void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) { /* update the values */ cmd->commandSize += size; cmd->cursorOffset += size; - for (int i = 0; i < size; i++) { - taosMbToWchar(&wc, c + i, size); - cmd->screenOffset += taosWcharWidth(wc); - cmd->endOffset += taosWcharWidth(wc); - } + cmd->screenOffset += taosWcharWidth(wc); + cmd->endOffset += taosWcharWidth(wc); #ifdef WINDOWS #else shellShowOnScreen(cmd); From 0f5f0d7cfbe8c68e18fa1a7dc946d6539330b21d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 18 Oct 2022 15:42:28 +0800 Subject: [PATCH 109/135] chore: remove codes unused currently for sma --- source/dnode/vnode/src/inc/sma.h | 2 + source/dnode/vnode/src/sma/smaCommit.c | 8 + source/dnode/vnode/src/sma/smaFS2.c | 720 +++++++++++++++++++++++++ source/dnode/vnode/src/sma/smaRollup.c | 420 +-------------- source/dnode/vnode/src/sma/smaUtil.c | 7 +- 5 files changed, 740 insertions(+), 417 deletions(-) create mode 100644 source/dnode/vnode/src/sma/smaFS2.c diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index dade85b12d..50e3c3a9c1 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -258,6 +258,7 @@ enum { TD_FTYPE_RSMA_QTASKINFO = 0, }; +#if 0 struct STFile { uint8_t state; STFInfo info; @@ -287,6 +288,7 @@ int32_t tdUpdateTFileHeader(STFile *pTFile); void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); void tdCloseTFile(STFile *pTFile); void tdDestroyTFile(STFile *pTFile); +#endif void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index db5f4c55b9..6168a00815 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -17,14 +17,17 @@ extern SSmaMgmt smaMgmt; +#if 0 static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma); static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma); static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma); +#endif static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); +#if 0 /** * @brief Only applicable to Rollup SMA * @@ -48,6 +51,7 @@ int32_t smaSyncCommit(SSma *pSma) { return tdProcessRSmaSyncCommitImpl(pSma); } * @return int32_t */ int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaSyncPostCommitImpl(pSma); } +#endif /** * @brief Only applicable to Rollup SMA @@ -108,6 +112,7 @@ int32_t smaBegin(SSma *pSma) { return TSDB_CODE_SUCCESS; } +#if 0 /** * @brief pre-commit for rollup sma(sync commit). * 1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED. @@ -169,6 +174,7 @@ static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma) { #endif return TSDB_CODE_SUCCESS; } +#endif // SQTaskFile ====================================================== @@ -230,6 +236,7 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) { return TSDB_CODE_SUCCESS; } +#if 0 /** * @brief post-commit for rollup sma * 1) clean up the outdated qtaskinfo files @@ -249,6 +256,7 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } +#endif /** * @brief Rsma async commit implementation(only do some necessary light weighted task) diff --git a/source/dnode/vnode/src/sma/smaFS2.c b/source/dnode/vnode/src/sma/smaFS2.c new file mode 100644 index 0000000000..9dced9c23b --- /dev/null +++ b/source/dnode/vnode/src/sma/smaFS2.c @@ -0,0 +1,720 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" + +// ================================================================================================= +static int32_t tdRSmaEncodeFS(uint8_t *p, SRSmaFS *pFS); +static int32_t tPutQTaskFile(uint8_t *p, SQTaskFile *pFile); +static int32_t tPutQTaskFileSet(uint8_t *p, SQTaskFileSet *pSet); + +static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output); +static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2); +static int32_t tdQTaskInfCmprFn2(const void *p1, const void *p2); + +// ================================================================================================= +static int32_t tdRSmaEncodeFS(uint8_t *p, SRSmaFS *pFS) { + int32_t n = 0; + uint32_t nSet = taosArrayGetSize(pFS->aQTaskFSet); + + // version + n += tPutU8(p ? p + n : p, 0); + + // SArray + n += tPutU32v(p ? p + n : p, nSet); + for (uint32_t i = 0; i < nSet; ++i) { + n += tPutQTaskFileSet(p ? p + n : p, (SQTaskFileSet *)taosArrayGet(pFS->aQTaskFSet, i)); + } + + return n; +} + +static int32_t tPutQTaskFileSet(uint8_t *p, SQTaskFileSet *pSet) { + int32_t n = 0; + int32_t nQTaskF = taosArrayGetSize(pSet->aQTaskF); + + n += tPutI64v(p ? p + n : p, pSet->suid); + n += tPutI32v(p ? p + n : p, nQTaskF); + + // SQTaskFile + for (int32_t i = 0; i < nQTaskF; ++i) { + n += tPutQTaskFile(p ? p + n : p, taosArrayGet(pSet->aQTaskF, i)); + } + + return n; +} + +static int32_t tPutQTaskFile(uint8_t *p, SQTaskFile *pFile) { + int32_t n = 0; + + n += tPutI8(p ? p + n : p, pFile->level); + n += tPutI64v(p ? p + n : p, pFile->version); + n += tPutI64v(p ? p + n : p, pFile->mtime); + + return n; +} + +static int32_t tGetQTaskFileSet(uint8_t *p, SQTaskFileSet *pSet) { + int32_t n = 0; + int32_t nQTaskF = 0; + + n += tGetI64v(p + n, &pSet->suid); + n += tGetI32v(p + n, &nQTaskF); + + ASSERT(nQTaskF >= 0); + + pSet->aQTaskF = taosArrayInit(nQTaskF, sizeof(SQTaskFile)); + if (!pSet->aQTaskF) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < nQTaskF; ++i) { + SQTaskFile *pFile = taosArrayGet(pSet->aQTaskF, i); + pFile->nRef = 1; + n += tGetQTaskFile(p + n, pFile); + } + + return n; +} + +static int32_t tGetQTaskFile(uint8_t *p, SQTaskFile *pFile) { + int32_t n = 0; + + n += tGetI8(p + n, &pFile->level); + n += tGetI64v(p + n, &pFile->version); + n += tGetI64v(p + n, &pFile->mtime); + + return n; +} + +static int32_t tdRSmaGnrtFS(SSma *pSma, SRSmaFS *pFS, char *fname) { + int32_t code = 0; + int64_t n; + int64_t size; + uint8_t *pData = NULL; + TdFilePtr pFD = NULL; + + // to binary + size = tdRSmaEncodeFS(NULL, pFS) + sizeof(TSCKSUM); + pData = taosMemoryMalloc(size); + if (pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + n = tdRSmaEncodeFS(pData, pFS); + ASSERT(n + sizeof(TSCKSUM) == size); + taosCalcChecksumAppend(0, pData, size); + + // create and write + pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + n = taosWriteFile(pFD, pData, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosFsyncFile(pFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + taosCloseFile(&pFD); + + if (pData) taosMemoryFree(pData); + return code; + +_err: + smaError("vgId:%d, rsma gnrt fs failed since %s", SMA_VID(pSma), tstrerror(code)); + if (pData) taosMemoryFree(pData); + return code; +} + +int32_t tdRSmaFSCommit1(SSma *pSma, SRSmaFS *pFSNew) { + int32_t code = 0; + char tfname[TSDB_FILENAME_LEN]; + char fname[TSDB_FILENAME_LEN]; + + snprintf(tfname, TSDB_FILENAME_LEN - 1, "%s%s%s%sRSMAFS.t", tfsGetPrimaryPath(pSma->pVnode->pTfs), TD_DIRSEP, + VNODE_RSMA_DIR, TD_DIRSEP); + snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sRSMAFS", tfsGetPrimaryPath(pSma->pVnode->pTfs), TD_DIRSEP, + VNODE_RSMA_DIR, TD_DIRSEP); + + // gnrt CURRENT.t + code = tdRSmaGnrtFS(pSma, pFSNew, tfname); + if (code) goto _err; + + // rename + code = taosRenameFile(tfname, fname); + if (code) { + code = TAOS_SYSTEM_ERROR(code); + goto _err; + } + + return code; + +_err: + smaError("vgId:%d, rsma fs commit phase 1 failed since %s", SMA_VID(pSma), tstrerror(code)); + return code; +} + +int32_t tdRSmaFSMergeQTaskFSet(SQTaskFileSet *pSetOld, SQTaskFileSet *pSetNew) { + SQTaskFileSet * + +} + +int32_t tdRSmaFSCommit2(SSma *pSma, SRSmaFS *pFSNew) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = NULL; + SRSmaInfo *pRSmaInfo = NULL; + SRSmaFS *pRSmaFS = NULL; + int32_t code = 0; + int32_t nRef; + char fname[TSDB_FILENAME_LEN]; + + if (!pEnv) { + terrno = TSDB_CODE_RSMA_INVALID_ENV; + return TSDB_CODE_FAILED; + } + + pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + if (!pStat || !(pRSmaFS = RSMA_FS(pStat))) { + terrno = TSDB_CODE_RSMA_INVALID_STAT; + return TSDB_CODE_FAILED; + } + + // data + int32_t iOld = 0; + int32_t iNew = 0; + while (true) { + int32_t nOld = taosArrayGetSize(pRSmaFS->aQTaskFSet); + int32_t nNew = taosArrayGetSize(pFSNew->aQTaskFSet); + SQTaskFileSet fSet; + + if (iOld >= nOld && iNew >= nNew) break; + + SQTaskFileSet *pSetOld = (iOld < nOld) ? taosArrayGet(pRSmaFS->aQTaskFSet, iOld) : NULL; + SQTaskFileSet *pSetNew = (iNew < nNew) ? taosArrayGet(pFSNew->aQTaskFSet, iNew) : NULL; + + if (pSetOld && pSetNew) { + if (pSetOld->suid == pSetNew->suid) { + goto _merge_old_and_new; + } else if (pSetOld->suid < pSetNew->suid) { + goto _remove_old; + } else { + goto _add_new; + } + } else if (pSetOld) { + goto _remove_old; + } else { + goto _add_new; + } + + _merge_old_and_new: + + + + // head + fSet.pHeadF = pSetOld->pHeadF; + if ((!sameDisk) || (pSetOld->pHeadF->commitID != pSetNew->pHeadF->commitID)) { + pSetOld->pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); + if (pSetOld->pHeadF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->pHeadF = *pSetNew->pHeadF; + pSetOld->pHeadF->nRef = 1; + + nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1); + if (nRef == 0) { + tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname); + taosRemoveFile(fname); + taosMemoryFree(fSet.pHeadF); + } + } else { + ASSERT(fSet.pHeadF->size == pSetNew->pHeadF->size); + ASSERT(fSet.pHeadF->offset == pSetNew->pHeadF->offset); + } + + // data + fSet.pDataF = pSetOld->pDataF; + if ((!sameDisk) || (pSetOld->pDataF->commitID != pSetNew->pDataF->commitID)) { + pSetOld->pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); + if (pSetOld->pDataF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->pDataF = *pSetNew->pDataF; + pSetOld->pDataF->nRef = 1; + + nRef = atomic_sub_fetch_32(&fSet.pDataF->nRef, 1); + if (nRef == 0) { + tsdbDataFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pDataF, fname); + taosRemoveFile(fname); + taosMemoryFree(fSet.pDataF); + } + } else { + ASSERT(pSetOld->pDataF->size <= pSetNew->pDataF->size); + pSetOld->pDataF->size = pSetNew->pDataF->size; + } + + // sma + fSet.pSmaF = pSetOld->pSmaF; + if ((!sameDisk) || (pSetOld->pSmaF->commitID != pSetNew->pSmaF->commitID)) { + pSetOld->pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); + if (pSetOld->pSmaF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->pSmaF = *pSetNew->pSmaF; + pSetOld->pSmaF->nRef = 1; + + nRef = atomic_sub_fetch_32(&fSet.pSmaF->nRef, 1); + if (nRef == 0) { + tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pSmaF, fname); + taosRemoveFile(fname); + taosMemoryFree(fSet.pSmaF); + } + } else { + ASSERT(pSetOld->pSmaF->size <= pSetNew->pSmaF->size); + pSetOld->pSmaF->size = pSetNew->pSmaF->size; + } + + // stt + if (sameDisk) { + if (pSetNew->nSttF > pSetOld->nSttF) { + ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1); + pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[pSetOld->nSttF] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[pSetOld->nSttF] = *pSetNew->aSttF[pSetOld->nSttF]; + pSetOld->aSttF[pSetOld->nSttF]->nRef = 1; + pSetOld->nSttF++; + } else if (pSetNew->nSttF < pSetOld->nSttF) { + ASSERT(pSetNew->nSttF == 1); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pSttFile); + } + pSetOld->aSttF[iStt] = NULL; + } + + pSetOld->nSttF = 1; + pSetOld->aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[0] = *pSetNew->aSttF[0]; + pSetOld->aSttF[0]->nRef = 1; + } else { + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + if (pSetOld->aSttF[iStt]->commitID != pSetNew->aSttF[iStt]->commitID) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pSttFile); + } + + pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[iStt] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; + pSetOld->aSttF[iStt]->nRef = 1; + } else { + ASSERT(pSetOld->aSttF[iStt]->size == pSetOld->aSttF[iStt]->size); + ASSERT(pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset); + } + } + } + } else { + ASSERT(pSetOld->nSttF == pSetNew->nSttF); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pSttFile); + } + + pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[iStt] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; + pSetOld->aSttF[iStt]->nRef = 1; + } + } + + if (!sameDisk) { + pSetOld->diskId = pSetNew->diskId; + } + + iOld++; + iNew++; + continue; + + _remove_old: + nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1); + if (nRef == 0) { + tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname); + taosRemoveFile(fname); + taosMemoryFree(pSetOld->pHeadF); + } + + nRef = atomic_sub_fetch_32(&pSetOld->pDataF->nRef, 1); + if (nRef == 0) { + tsdbDataFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pDataF, fname); + taosRemoveFile(fname); + taosMemoryFree(pSetOld->pDataF); + } + + nRef = atomic_sub_fetch_32(&pSetOld->pSmaF->nRef, 1); + if (nRef == 0) { + tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pSmaF, fname); + taosRemoveFile(fname); + taosMemoryFree(pSetOld->pSmaF); + } + + for (int8_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + nRef = atomic_sub_fetch_32(&pSetOld->aSttF[iStt]->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSttF[iStt], fname); + taosRemoveFile(fname); + taosMemoryFree(pSetOld->aSttF[iStt]); + } + } + + taosArrayRemove(pTsdb->fs.aDFileSet, iOld); + continue; + + _add_new: + fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSttF = 1}; + + // head + fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); + if (fSet.pHeadF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *fSet.pHeadF = *pSetNew->pHeadF; + fSet.pHeadF->nRef = 1; + + // data + fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); + if (fSet.pDataF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *fSet.pDataF = *pSetNew->pDataF; + fSet.pDataF->nRef = 1; + + // sma + fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); + if (fSet.pSmaF == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *fSet.pSmaF = *pSetNew->pSmaF; + fSet.pSmaF->nRef = 1; + + // stt + ASSERT(pSetNew->nSttF == 1); + fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (fSet.aSttF[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *fSet.aSttF[0] = *pSetNew->aSttF[0]; + fSet.aSttF[0]->nRef = 1; + + if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + iOld++; + iNew++; + continue; + } + + return code; + +_err: + tsdbError("vgId:%d, tsdb fs commit phase 2 failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +/** + * @brief Open RSma FS from qTaskInfo files + * + * @param pSma + * @param version + * @return int32_t + */ +int32_t tdRSmaFSOpen(SSma *pSma, int64_t version) { + SVnode *pVnode = pSma->pVnode; + int64_t commitID = pVnode->state.commitID; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = NULL; + SArray *output = NULL; + + terrno = TSDB_CODE_SUCCESS; + + if (!pEnv) { + return TSDB_CODE_SUCCESS; + } + + if (tdFetchQTaskInfoFiles(pSma, version, &output) < 0) { + goto _end; + } + + pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + + for (int32_t i = 0; i < taosArrayGetSize(output); ++i) { + int32_t vid = 0; + int64_t version = -1; + sscanf((const char *)taosArrayGetP(output, i), "v%dqinf.v%" PRIi64, &vid, &version); + SQTaskFile qTaskFile = {.version = version, .nRef = 1}; + if ((terrno = tdRSmaFSUpsertQTaskFile(RSMA_FS(pStat), &qTaskFile)) < 0) { + goto _end; + } + smaInfo("vgId:%d, open fs, version:%" PRIi64 ", ref:%d", TD_VID(pVnode), qTaskFile.version, qTaskFile.nRef); + } + +_end: + for (int32_t i = 0; i < taosArrayGetSize(output); ++i) { + void *ptr = taosArrayGetP(output, i); + taosMemoryFreeClear(ptr); + } + taosArrayDestroy(output); + + if (terrno != TSDB_CODE_SUCCESS) { + smaError("vgId:%d, open rsma fs failed since %s", TD_VID(pVnode), terrstr()); + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +void tdRSmaFSClose(SRSmaFS *fs) { taosArrayDestroy(fs->aQTaskFSet); } + +static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2) { + if (*(int64_t *)p1 < ((SQTaskFile *)p2)->version) { + return -1; + } else if (*(int64_t *)p1 > ((SQTaskFile *)p2)->version) { + return 1; + } + return 0; +} + +int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { + SArray *aQTaskFSet = RSMA_FS(pStat)->aQTaskFSet; + SQTaskFile *pTaskF = NULL; + int32_t oldVal = 0; + + taosRLockLatch(RSMA_FS_LOCK(pStat)); + if ((pTaskF = taosArraySearch(aQTaskFSet, &version, tdQTaskInfCmprFn1, TD_EQ))) { + oldVal = atomic_fetch_add_32(&pTaskF->nRef, 1); + ASSERT(oldVal > 0); + } + taosRUnLockLatch(RSMA_FS_LOCK(pStat)); + return oldVal; +} + +int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat) { + SArray *aQTaskFSet = RSMA_FS(pStat)->aQTaskFSet; + int64_t version = -1; + + taosRLockLatch(RSMA_FS_LOCK(pStat)); + if (taosArrayGetSize(aQTaskFSet) > 0) { + version = ((SQTaskFile *)taosArrayGetLast(aQTaskFSet))->version; + } + taosRUnLockLatch(RSMA_FS_LOCK(pStat)); + return version; +} + +void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { + SVnode *pVnode = pSma->pVnode; + SArray *aQTaskFSet = RSMA_FS(pStat)->aQTaskFSet; + char qTaskFullName[TSDB_FILENAME_LEN]; + SQTaskFile *pTaskF = NULL; + int32_t idx = -1; + + taosWLockLatch(RSMA_FS_LOCK(pStat)); + if ((idx = taosArraySearchIdx(aQTaskFSet, &version, tdQTaskInfCmprFn1, TD_EQ)) >= 0) { + ASSERT(idx < taosArrayGetSize(aQTaskFSet)); + pTaskF = taosArrayGet(aQTaskFSet, idx); + if (atomic_sub_fetch_32(&pTaskF->nRef, 1) <= 0) { + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskFullName); + if (taosRemoveFile(qTaskFullName) < 0) { + smaWarn("vgId:%d, failed to remove %s since %s", TD_VID(pVnode), qTaskFullName, + tstrerror(TAOS_SYSTEM_ERROR(errno))); + } else { + smaDebug("vgId:%d, success to remove %s", TD_VID(pVnode), qTaskFullName); + } + taosArrayRemove(aQTaskFSet, idx); + } + } + taosWUnLockLatch(RSMA_FS_LOCK(pStat)); +} + +/** + * @brief Fetch qtaskfiles LE than version + * + * @param pSma + * @param version + * @param output + * @return int32_t + */ +static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output) { + SVnode *pVnode = pSma->pVnode; + TdDirPtr pDir = NULL; + TdDirEntryPtr pDirEntry = NULL; + char dir[TSDB_FILENAME_LEN]; + const char *pattern = "v[0-9]+qinf\\.v([0-9]+)?$"; + regex_t regex; + int code = 0; + + terrno = TSDB_CODE_SUCCESS; + + tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir); + + if (!taosCheckExistFile(dir)) { + smaDebug("vgId:%d, fetch qtask files, no need as dir %s not exist", TD_VID(pVnode), dir); + return TSDB_CODE_SUCCESS; + } + + // Resource allocation and init + if ((code = regcomp(®ex, pattern, REG_EXTENDED)) != 0) { + terrno = TSDB_CODE_RSMA_REGEX_MATCH; + char errbuf[128]; + regerror(code, ®ex, errbuf, sizeof(errbuf)); + smaWarn("vgId:%d, fetch qtask files, regcomp for %s failed since %s", TD_VID(pVnode), dir, errbuf); + return TSDB_CODE_FAILED; + } + + if (!(pDir = taosOpenDir(dir))) { + regfree(®ex); + terrno = TAOS_SYSTEM_ERROR(errno); + smaError("vgId:%d, fetch qtask files, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); + return TSDB_CODE_FAILED; + } + + int32_t dirLen = strlen(dir); + char *dirEnd = POINTER_SHIFT(dir, dirLen); + regmatch_t regMatch[2]; + while ((pDirEntry = taosReadDir(pDir))) { + char *entryName = taosGetDirEntryName(pDirEntry); + if (!entryName) { + continue; + } + + code = regexec(®ex, entryName, 2, regMatch, 0); + + if (code == 0) { + // match + smaInfo("vgId:%d, fetch qtask files, max ver:%" PRIi64 ", %s found", TD_VID(pVnode), version, entryName); + + int64_t ver = -1; + sscanf((const char *)POINTER_SHIFT(entryName, regMatch[1].rm_so), "%" PRIi64, &ver); + if ((ver <= version) && (ver > -1)) { + if (!(*output)) { + if (!(*output = taosArrayInit(1, POINTER_BYTES))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + } + char *entryDup = strdup(entryName); + if (!entryDup) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + if (!taosArrayPush(*output, &entryDup)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + } else { + } + } else if (code == REG_NOMATCH) { + // not match + smaTrace("vgId:%d, fetch qtask files, not match %s", TD_VID(pVnode), entryName); + continue; + } else { + // has other error + char errbuf[128]; + regerror(code, ®ex, errbuf, sizeof(errbuf)); + smaWarn("vgId:%d, fetch qtask files, regexec failed since %s", TD_VID(pVnode), errbuf); + terrno = TSDB_CODE_RSMA_REGEX_MATCH; + goto _end; + } + } +_end: + taosCloseDir(&pDir); + regfree(®ex); + return terrno == 0 ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; +} + +static int32_t tdQTaskFileCmprFn2(const void *p1, const void *p2) { + if (((SQTaskFile *)p1)->version < ((SQTaskFile *)p2)->version) { + return -1; + } else if (((SQTaskFile *)p1)->version > ((SQTaskFile *)p2)->version) { + return 1; + } + + return 0; +} + +int32_t tdRSmaFSUpsertQTaskFile(SRSmaFS *pFS, SQTaskFile *qTaskFile) { + int32_t code = 0; + int32_t idx = taosArraySearchIdx(pFS->aQTaskFSet, qTaskFile, tdQTaskFileCmprFn2, TD_GE); + + if (idx < 0) { + idx = taosArrayGetSize(pFS->aQTaskFSet); + } else { + SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskFSet, idx); + int32_t c = tdQTaskFileCmprFn2(pTaskF, qTaskFile); + if (c == 0) { + pTaskF->nRef = qTaskFile->nRef; + pTaskF->level = qTaskFile->level; + pTaskF->version = qTaskFile->version; + pTaskF->mtime = qTaskFile->mtime; + goto _exit; + } + } + + if (taosArrayInsert(pFS->aQTaskFSet, idx, qTaskFile) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + +_exit: + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 722a3f479e..77c5955098 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -15,8 +15,6 @@ #include "sma.h" -#define RSMA_QTASKINFO_BUFSIZE (32768) // size -#define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid #define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt #define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt #define RSMA_FETCH_DELAY_MAX (120000) // ms @@ -48,23 +46,10 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SR static void tdRSmaFetchTrigger(void *param, void *tmrId); static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo); static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); -static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); -static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); -static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, int8_t type, SRSmaQTaskInfoIter *pIter); -static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem); static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables); static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int8_t type, int64_t qTaskFileVer); static int32_t tdRSmaRestoreTSDataReload(SSma *pSma); -static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) { - // adapt accordingly if definition of SRSmaInfo update - SRSmaInfo *pResult = NULL; - ASSERT(pItem->level == TSDB_RETENTION_L1 || pItem->level == TSDB_RETENTION_L2); - pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + RSMA_INFO_HEAD_LEN)); - ASSERT(pResult->pTSchema->numOfCols > 1); - return pResult; -} - struct SRSmaQTaskInfoItem { int32_t len; int8_t type; @@ -104,12 +89,6 @@ void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, con snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi64 "%s%" PRIi8, suid, TD_DIRSEP, level); } -static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { - return lenWithHead - RSMA_QTASKINFO_HEAD_LEN; -} - -static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskInfoIter *pIter) { taosMemoryFreeClear(pIter->pBuf); } - static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC if (!taskHandle || !(*taskHandle)) return; @@ -803,6 +782,7 @@ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inpu return TSDB_CODE_SUCCESS; } +#if 0 static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { SSubmitMsgIter msgIter = {0}; SSubmitBlkIter blkIter = {0}; @@ -820,6 +800,7 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { } return 0; } +#endif /** * @brief sync mode @@ -1189,65 +1170,6 @@ _err: return TSDB_CODE_FAILED; } -static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int8_t type, int64_t qTaskFileVer) { - SVnode *pVnode = pSma->pVnode; - STFile tFile = {0}; - char qTaskInfoFName[TSDB_FILENAME_LEN] = {0}; - - tdRSmaQTaskInfoGetFileName(TD_VID(pVnode), qTaskFileVer, qTaskInfoFName); - if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { - goto _err; - } - - if (!taosCheckExistFile(TD_TFILE_FULL_NAME(&tFile))) { - if (qTaskFileVer > 0) { - smaWarn("vgId:%d, restore rsma task %" PRIi8 " for version %" PRIi64 ", not start as %s not exist", - TD_VID(pVnode), type, qTaskFileVer, TD_TFILE_FULL_NAME(&tFile)); - } else { - smaDebug("vgId:%d, restore rsma task %" PRIi8 " for version %" PRIi64 ", no need as %s not exist", TD_VID(pVnode), - type, qTaskFileVer, TD_TFILE_FULL_NAME(&tFile)); - } - return TSDB_CODE_SUCCESS; - } - - if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) { - goto _err; - } - - STFInfo tFileInfo = {0}; - if (tdLoadTFileHeader(&tFile, &tFileInfo) < 0) { - goto _err; - } - - SRSmaQTaskInfoIter fIter = {0}; - if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) { - tdRSmaQTaskInfoIterDestroy(&fIter); - tdCloseTFile(&tFile); - tdDestroyTFile(&tFile); - goto _err; - } - - if (tdRSmaQTaskInfoRestore(pSma, type, &fIter) < 0) { - tdRSmaQTaskInfoIterDestroy(&fIter); - tdCloseTFile(&tFile); - tdDestroyTFile(&tFile); - goto _err; - } - - tdRSmaQTaskInfoIterDestroy(&fIter); - tdCloseTFile(&tFile); - tdDestroyTFile(&tFile); - - // restored successfully from committed or sync - smaInfo("vgId:%d, restore rsma task %" PRIi8 " for version %" PRIi64 ", qtaskinfo reload succeed", TD_VID(pVnode), - type, qTaskFileVer); - return TSDB_CODE_SUCCESS; -_err: - smaError("vgId:%d, restore rsma task %" PRIi8 " for version %" PRIi64 ", qtaskinfo reload failed since %s", - TD_VID(pVnode), type, qTaskFileVer, terrstr()); - return TSDB_CODE_FAILED; -} - /** * @brief reload ts data from checkpoint * @@ -1270,19 +1192,12 @@ int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer) return TSDB_CODE_SUCCESS; } -#if 0 - // step 2: retrieve qtaskinfo items from the persistence file(rsma/qtaskinfo) and restore - if (tdRSmaRestoreQTaskInfoReload(pSma, type, qtaskFileVer) < 0) { - goto _err; - } -#endif - - // step 3: reload ts data from checkpoint + // step 2: reload ts data from checkpoint if (tdRSmaRestoreTSDataReload(pSma) < 0) { goto _err; } - // step 4: open SRSmaFS for qTaskFiles + // step 3: open SRSmaFS for qTaskFiles if (tdRSmaFSOpen(pSma, qtaskFileVer) < 0) { goto _err; } @@ -1295,191 +1210,6 @@ _err: return TSDB_CODE_FAILED; } -/** - * @brief Restore from SRSmaQTaskInfoItem - * - * @param pSma - * @param pItem - * @return int32_t - */ -static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) { - SRSmaInfo *pRSmaInfo = NULL; - void *qTaskInfo = NULL; - - pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pItem->suid); - if (!pRSmaInfo) { - smaDebug("vgId:%d, no restore as no rsma info for table:%" PRIu64, SMA_VID(pSma), pItem->suid); - return TSDB_CODE_SUCCESS; - } - - if (pItem->type == TSDB_RETENTION_L1) { - qTaskInfo = RSMA_INFO_QTASK(pRSmaInfo, 0); - } else if (pItem->type == TSDB_RETENTION_L2) { - qTaskInfo = RSMA_INFO_QTASK(pRSmaInfo, 1); - } else { - ASSERT(0); - } - - if (!qTaskInfo) { - tdReleaseRSmaInfo(pSma, pRSmaInfo); - smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for table:%" PRIu64, SMA_VID(pSma), pItem->suid); - return TSDB_CODE_SUCCESS; - } - - if (qDeserializeTaskStatus(qTaskInfo, pItem->qTaskInfo, pItem->len) < 0) { - tdReleaseRSmaInfo(pSma, pRSmaInfo); - smaError("vgId:%d, restore rsma task failed for table:%" PRIi64 " level %d since %s", SMA_VID(pSma), pItem->suid, - pItem->type, terrstr()); - return TSDB_CODE_FAILED; - } - smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, - pItem->type); - - tdReleaseRSmaInfo(pSma, pRSmaInfo); - return TSDB_CODE_SUCCESS; -} - -static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile) { - memset(pIter, 0, sizeof(*pIter)); - pIter->pTFile = pTFile; - pIter->offset = TD_FILE_HEAD_SIZE; - - if (tdGetTFileSize(pTFile, &pIter->fsize) < 0) { - return TSDB_CODE_FAILED; - } - - if ((pIter->fsize - TD_FILE_HEAD_SIZE) < RSMA_QTASKINFO_BUFSIZE) { - pIter->nAlloc = pIter->fsize - TD_FILE_HEAD_SIZE; - } else { - pIter->nAlloc = RSMA_QTASKINFO_BUFSIZE; - } - - if (pIter->nAlloc < TD_FILE_HEAD_SIZE) { - pIter->nAlloc = TD_FILE_HEAD_SIZE; - } - - pIter->pBuf = taosMemoryMalloc(pIter->nAlloc); - if (!pIter->pBuf) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - pIter->qBuf = pIter->pBuf; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish) { - STFile *pTFile = pIter->pTFile; - int64_t nBytes = RSMA_QTASKINFO_BUFSIZE; - - if (pIter->offset >= pIter->fsize) { - *isFinish = true; - return TSDB_CODE_SUCCESS; - } - - if ((pIter->fsize - pIter->offset) < RSMA_QTASKINFO_BUFSIZE) { - nBytes = pIter->fsize - pIter->offset; - } - - if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { - return TSDB_CODE_FAILED; - } - - if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) { - return TSDB_CODE_FAILED; - } - - int32_t infoLen = 0; - taosDecodeFixedI32(pIter->pBuf, &infoLen); - if (infoLen > nBytes) { - if (infoLen <= RSMA_QTASKINFO_BUFSIZE) { - terrno = TSDB_CODE_RSMA_FILE_CORRUPTED; - smaError("iterate rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr()); - return TSDB_CODE_FAILED; - } - if (pIter->nAlloc < infoLen) { - pIter->nAlloc = infoLen; - void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen); - if (!pBuf) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - pIter->pBuf = pBuf; - } - - nBytes = infoLen; - - if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { - return TSDB_CODE_FAILED; - } - - if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) { - return TSDB_CODE_FAILED; - } - } - - pIter->qBuf = pIter->pBuf; - pIter->offset += nBytes; - pIter->nBytes = nBytes; - pIter->nBufPos = 0; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, int8_t type, SRSmaQTaskInfoIter *pIter) { - while (1) { - // block iter - bool isFinish = false; - if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) { - return TSDB_CODE_FAILED; - } - if (isFinish) { - return TSDB_CODE_SUCCESS; - } - - // consume the block - int32_t qTaskInfoLenWithHead = 0; - pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead); - if (qTaskInfoLenWithHead < RSMA_QTASKINFO_HEAD_LEN) { - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - smaError("vgId:%d, restore rsma task %" PRIi8 " from qtaskinfo file %s failed since %s", SMA_VID(pSma), type, - TD_TFILE_FULL_NAME(pIter->pTFile), terrstr()); - return TSDB_CODE_FAILED; - } - - while (1) { - if ((pIter->nBufPos + qTaskInfoLenWithHead) <= pIter->nBytes) { - SRSmaQTaskInfoItem infoItem = {0}; - pIter->qBuf = taosDecodeFixedI8(pIter->qBuf, &infoItem.type); - pIter->qBuf = taosDecodeFixedI64(pIter->qBuf, &infoItem.suid); - infoItem.qTaskInfo = pIter->qBuf; - infoItem.len = tdRSmaQTaskInfoContLen(qTaskInfoLenWithHead); - // do the restore job - smaDebug("vgId:%d, restore rsma task %" PRIi8 " from qtaskinfo file %s offset:%" PRIi64 "\n", SMA_VID(pSma), - type, TD_TFILE_FULL_NAME(pIter->pTFile), pIter->offset - pIter->nBytes + pIter->nBufPos); - tdRSmaQTaskInfoItemRestore(pSma, &infoItem); - - pIter->qBuf = POINTER_SHIFT(pIter->qBuf, infoItem.len); - pIter->nBufPos += qTaskInfoLenWithHead; - - if ((pIter->nBufPos + RSMA_QTASKINFO_HEAD_LEN) >= pIter->nBytes) { - // prepare and load next block in the file - pIter->offset -= (pIter->nBytes - pIter->nBufPos); - break; - } - - pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead); - continue; - } - // prepare and load next block in the file - pIter->offset -= (pIter->nBytes - pIter->nBufPos); - break; - } - } - - return TSDB_CODE_SUCCESS; -} - int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { SSma *pSma = pRSmaStat->pSma; SVnode *pVnode = pSma->pVnode; @@ -1523,148 +1253,6 @@ _err: return TSDB_CODE_FAILED; } -#if 0 -int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { - SSma *pSma = pRSmaStat->pSma; - SVnode *pVnode = pSma->pVnode; - int32_t vid = SMA_VID(pSma); - int64_t toffset = 0; - bool isFileCreated = false; - - if (taosHashGetSize(pInfoHash) <= 0) { - return TSDB_CODE_SUCCESS; - } - - void *infoHash = taosHashIterate(pInfoHash, NULL); - if (!infoHash) { - return TSDB_CODE_SUCCESS; - } - - int64_t fsMaxVer = tdRSmaFSMaxVer(pSma, pRSmaStat); - if (pRSmaStat->commitAppliedVer <= fsMaxVer) { - smaDebug("vgId:%d, rsma persist, no need as applied %" PRIi64 " not larger than fsMaxVer %" PRIi64, vid, - pRSmaStat->commitAppliedVer, fsMaxVer); - return TSDB_CODE_SUCCESS; - } - - STFile tFile = {0}; -#if 0 - if (pRSmaStat->commitAppliedVer > 0) { - char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); - if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { - smaError("vgId:%d, rsma persist, init %s failed since %s", vid, qTaskInfoFName, terrstr()); - goto _err; - } - if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) { - smaError("vgId:%d, rsma persist, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); - goto _err; - } - smaDebug("vgId:%d, rsma, serialize qTaskInfo, file %s created", vid, TD_TFILE_FULL_NAME(&tFile)); - - isFileCreated = true; - } -#endif - - while (infoHash) { - SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; - - if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - infoHash = taosHashIterate(pInfoHash, infoHash); - continue; - } - - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { -#if 0 - qTaskInfo_t taskInfo = RSMA_INFO_IQTASK(pRSmaInfo, i); -#endif - qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pRSmaInfo, i); - if (!taskInfo) { - smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); - continue; - } - - char *pOutput = NULL; - int32_t len = 0; - int8_t type = (int8_t)(i + 1); - if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) { - smaError("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo failed since %s", vid, pRSmaInfo->suid, - i + 1, terrstr()); - goto _err; - } - if (!pOutput || len <= 0) { - smaDebug("vgId:%d, rsma, table %" PRIi64 - " level %d serialize qTaskInfo success but no output(len %d), not persist", - vid, pRSmaInfo->suid, i + 1, len); - taosMemoryFreeClear(pOutput); - continue; - } - - smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo success with len %d, need persist", vid, - pRSmaInfo->suid, i + 1, len); - - if (!isFileCreated) { - char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); - if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { - smaError("vgId:%d, rsma persist, init %s failed since %s", vid, qTaskInfoFName, terrstr()); - goto _err; - } - if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) { - smaError("vgId:%d, rsma persist, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); - goto _err; - } - smaDebug("vgId:%d, rsma, table %" PRIi64 " serialize qTaskInfo, file %s created", vid, pRSmaInfo->suid, - TD_TFILE_FULL_NAME(&tFile)); - - isFileCreated = true; - } - - char tmpBuf[RSMA_QTASKINFO_HEAD_LEN] = {0}; - void *pTmpBuf = &tmpBuf; - int32_t headLen = 0; - headLen += taosEncodeFixedI32(&pTmpBuf, len + RSMA_QTASKINFO_HEAD_LEN); - headLen += taosEncodeFixedI8(&pTmpBuf, type); - headLen += taosEncodeFixedI64(&pTmpBuf, pRSmaInfo->suid); - - ASSERT(headLen <= RSMA_QTASKINFO_HEAD_LEN); - tdAppendTFile(&tFile, (void *)&tmpBuf, headLen, &toffset); - smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d head part(len:%d) appended to offset:%" PRIi64, vid, - pRSmaInfo->suid, i + 1, headLen, toffset); - tdAppendTFile(&tFile, pOutput, len, &toffset); - smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d body part len:%d appended to offset:%" PRIi64, vid, - pRSmaInfo->suid, i + 1, len, toffset); - - taosMemoryFree(pOutput); - } - - infoHash = taosHashIterate(pInfoHash, infoHash); - } - - if (isFileCreated) { - if (tdUpdateTFileHeader(&tFile) < 0) { - smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile), - tstrerror(terrno)); - goto _err; - } else { - smaDebug("vgId:%d, rsma, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile)); - } - - tdCloseTFile(&tFile); - tdDestroyTFile(&tFile); - } - return TSDB_CODE_SUCCESS; -_err: - smaError("vgId:%d, rsma persist failed since %s", vid, terrstr()); - if (isFileCreated) { - tdRemoveTFile(&tFile); - tdDestroyTFile(&tFile); - } - return TSDB_CODE_FAILED; -} - -#endif - /** * @brief trigger to get rsma result in async mode * diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 6d7b7df1ee..4d09d690d6 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -16,7 +16,7 @@ #include "sma.h" // smaFileUtil ================ - +#if 0 #define TD_FILE_STATE_OK 0 #define TD_FILE_STATE_BAD 1 @@ -182,6 +182,8 @@ void tdCloseTFile(STFile *pTFile) { void tdDestroyTFile(STFile *pTFile) { taosMemoryFreeClear(TD_TFILE_FULL_NAME(pTFile)); } +#endif + void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName) { if (version < 0) { @@ -221,6 +223,7 @@ void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool e } } +#if 0 int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname) { TD_TFILE_SET_STATE(pTFile, TD_FILE_STATE_OK); TD_TFILE_SET_CLOSED(pTFile); @@ -286,6 +289,8 @@ int32_t tdRemoveTFile(STFile *pTFile) { return 0; } +#endif + // smaXXXUtil ================ void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { void *pResult = taosAcquireRef(rsetId, refId); From 4c6623d3ddbea2ed6a2e5e776a3a49861ef6e038 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 18 Oct 2022 15:47:36 +0800 Subject: [PATCH 110/135] chore: remove obsolete file --- source/dnode/vnode/src/sma/smaFS2.c | 720 ---------------------------- 1 file changed, 720 deletions(-) delete mode 100644 source/dnode/vnode/src/sma/smaFS2.c diff --git a/source/dnode/vnode/src/sma/smaFS2.c b/source/dnode/vnode/src/sma/smaFS2.c deleted file mode 100644 index 9dced9c23b..0000000000 --- a/source/dnode/vnode/src/sma/smaFS2.c +++ /dev/null @@ -1,720 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sma.h" - -// ================================================================================================= -static int32_t tdRSmaEncodeFS(uint8_t *p, SRSmaFS *pFS); -static int32_t tPutQTaskFile(uint8_t *p, SQTaskFile *pFile); -static int32_t tPutQTaskFileSet(uint8_t *p, SQTaskFileSet *pSet); - -static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output); -static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2); -static int32_t tdQTaskInfCmprFn2(const void *p1, const void *p2); - -// ================================================================================================= -static int32_t tdRSmaEncodeFS(uint8_t *p, SRSmaFS *pFS) { - int32_t n = 0; - uint32_t nSet = taosArrayGetSize(pFS->aQTaskFSet); - - // version - n += tPutU8(p ? p + n : p, 0); - - // SArray - n += tPutU32v(p ? p + n : p, nSet); - for (uint32_t i = 0; i < nSet; ++i) { - n += tPutQTaskFileSet(p ? p + n : p, (SQTaskFileSet *)taosArrayGet(pFS->aQTaskFSet, i)); - } - - return n; -} - -static int32_t tPutQTaskFileSet(uint8_t *p, SQTaskFileSet *pSet) { - int32_t n = 0; - int32_t nQTaskF = taosArrayGetSize(pSet->aQTaskF); - - n += tPutI64v(p ? p + n : p, pSet->suid); - n += tPutI32v(p ? p + n : p, nQTaskF); - - // SQTaskFile - for (int32_t i = 0; i < nQTaskF; ++i) { - n += tPutQTaskFile(p ? p + n : p, taosArrayGet(pSet->aQTaskF, i)); - } - - return n; -} - -static int32_t tPutQTaskFile(uint8_t *p, SQTaskFile *pFile) { - int32_t n = 0; - - n += tPutI8(p ? p + n : p, pFile->level); - n += tPutI64v(p ? p + n : p, pFile->version); - n += tPutI64v(p ? p + n : p, pFile->mtime); - - return n; -} - -static int32_t tGetQTaskFileSet(uint8_t *p, SQTaskFileSet *pSet) { - int32_t n = 0; - int32_t nQTaskF = 0; - - n += tGetI64v(p + n, &pSet->suid); - n += tGetI32v(p + n, &nQTaskF); - - ASSERT(nQTaskF >= 0); - - pSet->aQTaskF = taosArrayInit(nQTaskF, sizeof(SQTaskFile)); - if (!pSet->aQTaskF) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - for (int32_t i = 0; i < nQTaskF; ++i) { - SQTaskFile *pFile = taosArrayGet(pSet->aQTaskF, i); - pFile->nRef = 1; - n += tGetQTaskFile(p + n, pFile); - } - - return n; -} - -static int32_t tGetQTaskFile(uint8_t *p, SQTaskFile *pFile) { - int32_t n = 0; - - n += tGetI8(p + n, &pFile->level); - n += tGetI64v(p + n, &pFile->version); - n += tGetI64v(p + n, &pFile->mtime); - - return n; -} - -static int32_t tdRSmaGnrtFS(SSma *pSma, SRSmaFS *pFS, char *fname) { - int32_t code = 0; - int64_t n; - int64_t size; - uint8_t *pData = NULL; - TdFilePtr pFD = NULL; - - // to binary - size = tdRSmaEncodeFS(NULL, pFS) + sizeof(TSCKSUM); - pData = taosMemoryMalloc(size); - if (pData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - n = tdRSmaEncodeFS(pData, pFS); - ASSERT(n + sizeof(TSCKSUM) == size); - taosCalcChecksumAppend(0, pData, size); - - // create and write - pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pFD, pData, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosFsyncFile(pFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - taosCloseFile(&pFD); - - if (pData) taosMemoryFree(pData); - return code; - -_err: - smaError("vgId:%d, rsma gnrt fs failed since %s", SMA_VID(pSma), tstrerror(code)); - if (pData) taosMemoryFree(pData); - return code; -} - -int32_t tdRSmaFSCommit1(SSma *pSma, SRSmaFS *pFSNew) { - int32_t code = 0; - char tfname[TSDB_FILENAME_LEN]; - char fname[TSDB_FILENAME_LEN]; - - snprintf(tfname, TSDB_FILENAME_LEN - 1, "%s%s%s%sRSMAFS.t", tfsGetPrimaryPath(pSma->pVnode->pTfs), TD_DIRSEP, - VNODE_RSMA_DIR, TD_DIRSEP); - snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sRSMAFS", tfsGetPrimaryPath(pSma->pVnode->pTfs), TD_DIRSEP, - VNODE_RSMA_DIR, TD_DIRSEP); - - // gnrt CURRENT.t - code = tdRSmaGnrtFS(pSma, pFSNew, tfname); - if (code) goto _err; - - // rename - code = taosRenameFile(tfname, fname); - if (code) { - code = TAOS_SYSTEM_ERROR(code); - goto _err; - } - - return code; - -_err: - smaError("vgId:%d, rsma fs commit phase 1 failed since %s", SMA_VID(pSma), tstrerror(code)); - return code; -} - -int32_t tdRSmaFSMergeQTaskFSet(SQTaskFileSet *pSetOld, SQTaskFileSet *pSetNew) { - SQTaskFileSet * - -} - -int32_t tdRSmaFSCommit2(SSma *pSma, SRSmaFS *pFSNew) { - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = NULL; - SRSmaInfo *pRSmaInfo = NULL; - SRSmaFS *pRSmaFS = NULL; - int32_t code = 0; - int32_t nRef; - char fname[TSDB_FILENAME_LEN]; - - if (!pEnv) { - terrno = TSDB_CODE_RSMA_INVALID_ENV; - return TSDB_CODE_FAILED; - } - - pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - if (!pStat || !(pRSmaFS = RSMA_FS(pStat))) { - terrno = TSDB_CODE_RSMA_INVALID_STAT; - return TSDB_CODE_FAILED; - } - - // data - int32_t iOld = 0; - int32_t iNew = 0; - while (true) { - int32_t nOld = taosArrayGetSize(pRSmaFS->aQTaskFSet); - int32_t nNew = taosArrayGetSize(pFSNew->aQTaskFSet); - SQTaskFileSet fSet; - - if (iOld >= nOld && iNew >= nNew) break; - - SQTaskFileSet *pSetOld = (iOld < nOld) ? taosArrayGet(pRSmaFS->aQTaskFSet, iOld) : NULL; - SQTaskFileSet *pSetNew = (iNew < nNew) ? taosArrayGet(pFSNew->aQTaskFSet, iNew) : NULL; - - if (pSetOld && pSetNew) { - if (pSetOld->suid == pSetNew->suid) { - goto _merge_old_and_new; - } else if (pSetOld->suid < pSetNew->suid) { - goto _remove_old; - } else { - goto _add_new; - } - } else if (pSetOld) { - goto _remove_old; - } else { - goto _add_new; - } - - _merge_old_and_new: - - - - // head - fSet.pHeadF = pSetOld->pHeadF; - if ((!sameDisk) || (pSetOld->pHeadF->commitID != pSetNew->pHeadF->commitID)) { - pSetOld->pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); - if (pSetOld->pHeadF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->pHeadF = *pSetNew->pHeadF; - pSetOld->pHeadF->nRef = 1; - - nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1); - if (nRef == 0) { - tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname); - taosRemoveFile(fname); - taosMemoryFree(fSet.pHeadF); - } - } else { - ASSERT(fSet.pHeadF->size == pSetNew->pHeadF->size); - ASSERT(fSet.pHeadF->offset == pSetNew->pHeadF->offset); - } - - // data - fSet.pDataF = pSetOld->pDataF; - if ((!sameDisk) || (pSetOld->pDataF->commitID != pSetNew->pDataF->commitID)) { - pSetOld->pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); - if (pSetOld->pDataF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->pDataF = *pSetNew->pDataF; - pSetOld->pDataF->nRef = 1; - - nRef = atomic_sub_fetch_32(&fSet.pDataF->nRef, 1); - if (nRef == 0) { - tsdbDataFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pDataF, fname); - taosRemoveFile(fname); - taosMemoryFree(fSet.pDataF); - } - } else { - ASSERT(pSetOld->pDataF->size <= pSetNew->pDataF->size); - pSetOld->pDataF->size = pSetNew->pDataF->size; - } - - // sma - fSet.pSmaF = pSetOld->pSmaF; - if ((!sameDisk) || (pSetOld->pSmaF->commitID != pSetNew->pSmaF->commitID)) { - pSetOld->pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); - if (pSetOld->pSmaF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->pSmaF = *pSetNew->pSmaF; - pSetOld->pSmaF->nRef = 1; - - nRef = atomic_sub_fetch_32(&fSet.pSmaF->nRef, 1); - if (nRef == 0) { - tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pSmaF, fname); - taosRemoveFile(fname); - taosMemoryFree(fSet.pSmaF); - } - } else { - ASSERT(pSetOld->pSmaF->size <= pSetNew->pSmaF->size); - pSetOld->pSmaF->size = pSetNew->pSmaF->size; - } - - // stt - if (sameDisk) { - if (pSetNew->nSttF > pSetOld->nSttF) { - ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1); - pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (pSetOld->aSttF[pSetOld->nSttF] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->aSttF[pSetOld->nSttF] = *pSetNew->aSttF[pSetOld->nSttF]; - pSetOld->aSttF[pSetOld->nSttF]->nRef = 1; - pSetOld->nSttF++; - } else if (pSetNew->nSttF < pSetOld->nSttF) { - ASSERT(pSetNew->nSttF == 1); - for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { - SSttFile *pSttFile = pSetOld->aSttF[iStt]; - nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); - if (nRef == 0) { - tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); - taosRemoveFile(fname); - taosMemoryFree(pSttFile); - } - pSetOld->aSttF[iStt] = NULL; - } - - pSetOld->nSttF = 1; - pSetOld->aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (pSetOld->aSttF[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->aSttF[0] = *pSetNew->aSttF[0]; - pSetOld->aSttF[0]->nRef = 1; - } else { - for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { - if (pSetOld->aSttF[iStt]->commitID != pSetNew->aSttF[iStt]->commitID) { - SSttFile *pSttFile = pSetOld->aSttF[iStt]; - nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); - if (nRef == 0) { - tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); - taosRemoveFile(fname); - taosMemoryFree(pSttFile); - } - - pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (pSetOld->aSttF[iStt] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; - pSetOld->aSttF[iStt]->nRef = 1; - } else { - ASSERT(pSetOld->aSttF[iStt]->size == pSetOld->aSttF[iStt]->size); - ASSERT(pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset); - } - } - } - } else { - ASSERT(pSetOld->nSttF == pSetNew->nSttF); - for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { - SSttFile *pSttFile = pSetOld->aSttF[iStt]; - nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); - if (nRef == 0) { - tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); - taosRemoveFile(fname); - taosMemoryFree(pSttFile); - } - - pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (pSetOld->aSttF[iStt] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; - pSetOld->aSttF[iStt]->nRef = 1; - } - } - - if (!sameDisk) { - pSetOld->diskId = pSetNew->diskId; - } - - iOld++; - iNew++; - continue; - - _remove_old: - nRef = atomic_sub_fetch_32(&pSetOld->pHeadF->nRef, 1); - if (nRef == 0) { - tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pHeadF, fname); - taosRemoveFile(fname); - taosMemoryFree(pSetOld->pHeadF); - } - - nRef = atomic_sub_fetch_32(&pSetOld->pDataF->nRef, 1); - if (nRef == 0) { - tsdbDataFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pDataF, fname); - taosRemoveFile(fname); - taosMemoryFree(pSetOld->pDataF); - } - - nRef = atomic_sub_fetch_32(&pSetOld->pSmaF->nRef, 1); - if (nRef == 0) { - tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pSmaF, fname); - taosRemoveFile(fname); - taosMemoryFree(pSetOld->pSmaF); - } - - for (int8_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { - nRef = atomic_sub_fetch_32(&pSetOld->aSttF[iStt]->nRef, 1); - if (nRef == 0) { - tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSttF[iStt], fname); - taosRemoveFile(fname); - taosMemoryFree(pSetOld->aSttF[iStt]); - } - } - - taosArrayRemove(pTsdb->fs.aDFileSet, iOld); - continue; - - _add_new: - fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSttF = 1}; - - // head - fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); - if (fSet.pHeadF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *fSet.pHeadF = *pSetNew->pHeadF; - fSet.pHeadF->nRef = 1; - - // data - fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); - if (fSet.pDataF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *fSet.pDataF = *pSetNew->pDataF; - fSet.pDataF->nRef = 1; - - // sma - fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); - if (fSet.pSmaF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *fSet.pSmaF = *pSetNew->pSmaF; - fSet.pSmaF->nRef = 1; - - // stt - ASSERT(pSetNew->nSttF == 1); - fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (fSet.aSttF[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *fSet.aSttF[0] = *pSetNew->aSttF[0]; - fSet.aSttF[0]->nRef = 1; - - if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - iOld++; - iNew++; - continue; - } - - return code; - -_err: - tsdbError("vgId:%d, tsdb fs commit phase 2 failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -/** - * @brief Open RSma FS from qTaskInfo files - * - * @param pSma - * @param version - * @return int32_t - */ -int32_t tdRSmaFSOpen(SSma *pSma, int64_t version) { - SVnode *pVnode = pSma->pVnode; - int64_t commitID = pVnode->state.commitID; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = NULL; - SArray *output = NULL; - - terrno = TSDB_CODE_SUCCESS; - - if (!pEnv) { - return TSDB_CODE_SUCCESS; - } - - if (tdFetchQTaskInfoFiles(pSma, version, &output) < 0) { - goto _end; - } - - pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - - for (int32_t i = 0; i < taosArrayGetSize(output); ++i) { - int32_t vid = 0; - int64_t version = -1; - sscanf((const char *)taosArrayGetP(output, i), "v%dqinf.v%" PRIi64, &vid, &version); - SQTaskFile qTaskFile = {.version = version, .nRef = 1}; - if ((terrno = tdRSmaFSUpsertQTaskFile(RSMA_FS(pStat), &qTaskFile)) < 0) { - goto _end; - } - smaInfo("vgId:%d, open fs, version:%" PRIi64 ", ref:%d", TD_VID(pVnode), qTaskFile.version, qTaskFile.nRef); - } - -_end: - for (int32_t i = 0; i < taosArrayGetSize(output); ++i) { - void *ptr = taosArrayGetP(output, i); - taosMemoryFreeClear(ptr); - } - taosArrayDestroy(output); - - if (terrno != TSDB_CODE_SUCCESS) { - smaError("vgId:%d, open rsma fs failed since %s", TD_VID(pVnode), terrstr()); - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; -} - -void tdRSmaFSClose(SRSmaFS *fs) { taosArrayDestroy(fs->aQTaskFSet); } - -static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2) { - if (*(int64_t *)p1 < ((SQTaskFile *)p2)->version) { - return -1; - } else if (*(int64_t *)p1 > ((SQTaskFile *)p2)->version) { - return 1; - } - return 0; -} - -int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { - SArray *aQTaskFSet = RSMA_FS(pStat)->aQTaskFSet; - SQTaskFile *pTaskF = NULL; - int32_t oldVal = 0; - - taosRLockLatch(RSMA_FS_LOCK(pStat)); - if ((pTaskF = taosArraySearch(aQTaskFSet, &version, tdQTaskInfCmprFn1, TD_EQ))) { - oldVal = atomic_fetch_add_32(&pTaskF->nRef, 1); - ASSERT(oldVal > 0); - } - taosRUnLockLatch(RSMA_FS_LOCK(pStat)); - return oldVal; -} - -int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat) { - SArray *aQTaskFSet = RSMA_FS(pStat)->aQTaskFSet; - int64_t version = -1; - - taosRLockLatch(RSMA_FS_LOCK(pStat)); - if (taosArrayGetSize(aQTaskFSet) > 0) { - version = ((SQTaskFile *)taosArrayGetLast(aQTaskFSet))->version; - } - taosRUnLockLatch(RSMA_FS_LOCK(pStat)); - return version; -} - -void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { - SVnode *pVnode = pSma->pVnode; - SArray *aQTaskFSet = RSMA_FS(pStat)->aQTaskFSet; - char qTaskFullName[TSDB_FILENAME_LEN]; - SQTaskFile *pTaskF = NULL; - int32_t idx = -1; - - taosWLockLatch(RSMA_FS_LOCK(pStat)); - if ((idx = taosArraySearchIdx(aQTaskFSet, &version, tdQTaskInfCmprFn1, TD_EQ)) >= 0) { - ASSERT(idx < taosArrayGetSize(aQTaskFSet)); - pTaskF = taosArrayGet(aQTaskFSet, idx); - if (atomic_sub_fetch_32(&pTaskF->nRef, 1) <= 0) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskFullName); - if (taosRemoveFile(qTaskFullName) < 0) { - smaWarn("vgId:%d, failed to remove %s since %s", TD_VID(pVnode), qTaskFullName, - tstrerror(TAOS_SYSTEM_ERROR(errno))); - } else { - smaDebug("vgId:%d, success to remove %s", TD_VID(pVnode), qTaskFullName); - } - taosArrayRemove(aQTaskFSet, idx); - } - } - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); -} - -/** - * @brief Fetch qtaskfiles LE than version - * - * @param pSma - * @param version - * @param output - * @return int32_t - */ -static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output) { - SVnode *pVnode = pSma->pVnode; - TdDirPtr pDir = NULL; - TdDirEntryPtr pDirEntry = NULL; - char dir[TSDB_FILENAME_LEN]; - const char *pattern = "v[0-9]+qinf\\.v([0-9]+)?$"; - regex_t regex; - int code = 0; - - terrno = TSDB_CODE_SUCCESS; - - tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir); - - if (!taosCheckExistFile(dir)) { - smaDebug("vgId:%d, fetch qtask files, no need as dir %s not exist", TD_VID(pVnode), dir); - return TSDB_CODE_SUCCESS; - } - - // Resource allocation and init - if ((code = regcomp(®ex, pattern, REG_EXTENDED)) != 0) { - terrno = TSDB_CODE_RSMA_REGEX_MATCH; - char errbuf[128]; - regerror(code, ®ex, errbuf, sizeof(errbuf)); - smaWarn("vgId:%d, fetch qtask files, regcomp for %s failed since %s", TD_VID(pVnode), dir, errbuf); - return TSDB_CODE_FAILED; - } - - if (!(pDir = taosOpenDir(dir))) { - regfree(®ex); - terrno = TAOS_SYSTEM_ERROR(errno); - smaError("vgId:%d, fetch qtask files, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); - return TSDB_CODE_FAILED; - } - - int32_t dirLen = strlen(dir); - char *dirEnd = POINTER_SHIFT(dir, dirLen); - regmatch_t regMatch[2]; - while ((pDirEntry = taosReadDir(pDir))) { - char *entryName = taosGetDirEntryName(pDirEntry); - if (!entryName) { - continue; - } - - code = regexec(®ex, entryName, 2, regMatch, 0); - - if (code == 0) { - // match - smaInfo("vgId:%d, fetch qtask files, max ver:%" PRIi64 ", %s found", TD_VID(pVnode), version, entryName); - - int64_t ver = -1; - sscanf((const char *)POINTER_SHIFT(entryName, regMatch[1].rm_so), "%" PRIi64, &ver); - if ((ver <= version) && (ver > -1)) { - if (!(*output)) { - if (!(*output = taosArrayInit(1, POINTER_BYTES))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - } - char *entryDup = strdup(entryName); - if (!entryDup) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - if (!taosArrayPush(*output, &entryDup)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - } else { - } - } else if (code == REG_NOMATCH) { - // not match - smaTrace("vgId:%d, fetch qtask files, not match %s", TD_VID(pVnode), entryName); - continue; - } else { - // has other error - char errbuf[128]; - regerror(code, ®ex, errbuf, sizeof(errbuf)); - smaWarn("vgId:%d, fetch qtask files, regexec failed since %s", TD_VID(pVnode), errbuf); - terrno = TSDB_CODE_RSMA_REGEX_MATCH; - goto _end; - } - } -_end: - taosCloseDir(&pDir); - regfree(®ex); - return terrno == 0 ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; -} - -static int32_t tdQTaskFileCmprFn2(const void *p1, const void *p2) { - if (((SQTaskFile *)p1)->version < ((SQTaskFile *)p2)->version) { - return -1; - } else if (((SQTaskFile *)p1)->version > ((SQTaskFile *)p2)->version) { - return 1; - } - - return 0; -} - -int32_t tdRSmaFSUpsertQTaskFile(SRSmaFS *pFS, SQTaskFile *qTaskFile) { - int32_t code = 0; - int32_t idx = taosArraySearchIdx(pFS->aQTaskFSet, qTaskFile, tdQTaskFileCmprFn2, TD_GE); - - if (idx < 0) { - idx = taosArrayGetSize(pFS->aQTaskFSet); - } else { - SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskFSet, idx); - int32_t c = tdQTaskFileCmprFn2(pTaskF, qTaskFile); - if (c == 0) { - pTaskF->nRef = qTaskFile->nRef; - pTaskF->level = qTaskFile->level; - pTaskF->version = qTaskFile->version; - pTaskF->mtime = qTaskFile->mtime; - goto _exit; - } - } - - if (taosArrayInsert(pFS->aQTaskFSet, idx, qTaskFile) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - -_exit: - return code; -} \ No newline at end of file From 2525a5cf1fb6567f5508a18f0bb7bc7d488fbe42 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 16:06:52 +0800 Subject: [PATCH 111/135] enh: report proper errorcode on disk unavailability --- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index becefcd6a0..5a7f149bc6 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -65,14 +65,17 @@ static bool dmCheckDiskSpace() { bool ret = true; if (!osDataSpaceAvailable()) { dError("data disk space unavailable, i.e. %s", tsDataDir); + terrno = TSDB_CODE_VND_NO_DISKSPACE; ret = false; } if (!osLogSpaceAvailable()) { dError("log disk space unavailable, i.e. %s", tsLogDir); + terrno = TSDB_CODE_VND_NO_DISKSPACE; ret = false; } if (!osTempSpaceAvailable()) { dError("temp disk space unavailable, i.e. %s", tsTempDir); + terrno = TSDB_CODE_VND_NO_DISKSPACE; ret = false; } return ret; From 0fe14f0b167d09bbb2526430212c9e53422a8b5d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 18 Oct 2022 16:33:27 +0800 Subject: [PATCH 112/135] fix: fix coverity issues --- include/util/tdef.h | 1 + source/client/src/clientImpl.c | 16 ++++++--- source/client/src/clientMain.c | 1 - source/client/src/clientMsgHandler.c | 7 ++-- source/client/src/clientStmt.c | 2 +- source/dnode/mnode/impl/src/mndQuery.c | 25 +++++++++++++ source/dnode/mnode/impl/src/mndStb.c | 9 +++-- source/dnode/vnode/src/vnd/vnodeQuery.c | 5 +++ source/libs/catalog/src/catalog.c | 8 +++-- source/libs/catalog/src/ctgAsync.c | 4 +-- source/libs/catalog/src/ctgCache.c | 43 ++++++++++++----------- source/libs/command/src/command.c | 7 +++- source/libs/executor/src/dataDispatcher.c | 8 ++++- source/libs/executor/src/dataInserter.c | 2 +- source/libs/qcom/src/queryUtil.c | 3 +- source/libs/qcom/src/querymsg.c | 12 +++---- source/libs/qworker/src/qwUtil.c | 6 +++- source/libs/qworker/src/qworker.c | 1 - source/libs/scheduler/src/schTask.c | 3 +- 19 files changed, 113 insertions(+), 50 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index d9bb558b74..2103dc928e 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -498,6 +498,7 @@ enum { #define MAX_NUM_STR_SIZE 40 #define MAX_META_MSG_IN_BATCH 1048576 +#define MAX_META_BATCH_RSP_SIZE (1 * 1048576 * 1024) #ifdef __cplusplus } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b494140da5..272f78a980 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -199,7 +199,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) { if (TSDB_CODE_SUCCESS != nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) { - tscError("%d failed to create node allocator, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, + tscError("%d failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); @@ -955,7 +955,12 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: if (!pRequest->validateOnly) { - code = execLocalCmd(pRequest, pQuery); + if (NULL == pQuery->pRoot) { + terrno = TSDB_CODE_INVALID_PARA; + code = terrno; + } else { + code = execLocalCmd(pRequest, pQuery); + } } break; case QUERY_EXEC_MODE_RPC: @@ -997,7 +1002,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue handleQueryExecRsp(pRequest); - if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } @@ -2253,7 +2258,10 @@ void syncQueryFn(void* param, void* res, int32_t code) { void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) { if (sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; - fp(param, NULL, terrno); + if (fp) { + fp(param, NULL, terrno); + } + return; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 6126817ece..286fb85373 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -944,7 +944,6 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { if (pResultInfo->completed) { // it is a local executed query, no need to do async fetch if (QUERY_EXEC_MODE_LOCAL == pRequest->body.execMode) { - ASSERT(pResultInfo->numOfRows >= 0); if (pResultInfo->localResultFetched) { pResultInfo->numOfRows = 0; pResultInfo->current = 0; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index cdf977bea3..0a1135886e 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -288,8 +288,10 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tDeserializeSDropDbRsp(pMsg->pData, pMsg->len, &dropdbRsp); struct SCatalog* pCatalog = NULL; - catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); - catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); + int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (TSDB_CODE_SUCCESS == code) { + catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); + } } taosMemoryFree(pMsg->pData); @@ -393,6 +395,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { + blockDataDestroy(pBlock); return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index bf3fd00f14..81d0d616c9 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -152,7 +152,7 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, pStmt->bInfo.tbType = pTableMeta->tableType; pStmt->bInfo.boundTags = tags; pStmt->bInfo.tagsCached = false; - strcpy(pStmt->bInfo.stbFName, sTableName); + tstrncpy(pStmt->bInfo.stbFName, sTableName, sizeof(pStmt->bInfo.stbFName)); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 2e01fadbae..3e4c8005b4 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -90,14 +90,39 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { } for (int32_t i = 0; i < msgNum; ++i) { + if (offset >= pMsg->contLen) { + mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } + req.msgIdx = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgIdx); + if (offset >= pMsg->contLen) { + mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } req.msgType = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgType); + if (offset >= pMsg->contLen) { + mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } req.msgLen = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgLen); + if (offset >= pMsg->contLen) { + mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } req.msg = (char *)pMsg->pCont + offset; offset += req.msgLen; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 10cfb0a660..8a3179b2a9 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2553,12 +2553,17 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc char rollup[160 + VARSTR_HEADER_SIZE] = {0}; int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); + char *sep = ", "; + int32_t sepLen = strlen(sep); + int32_t rollupLen = sizeof(rollup) - 2; for (int32_t i = 0; i < rollupNum; ++i) { char *funcName = taosArrayGet(pStb->pFuncs, i); if (i) { - strcat(varDataVal(rollup), ", "); + strncat(varDataVal(rollup), sep, rollupLen); + rollupLen -= sepLen; } - strcat(varDataVal(rollup), funcName); + strncat(varDataVal(rollup), funcName, rollupLen); + rollupLen -= strlen(funcName); } varDataSetLen(rollup, strlen(varDataVal(rollup))); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 8c1f858cbb..64df3aa1eb 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -330,6 +330,11 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { rspSize += sizeof(int32_t); offset = 0; + if (rspSize > MAX_META_BATCH_RSP_SIZE) { + code = TSDB_CODE_INVALID_MSG_LEN; + goto _exit; + } + pRsp = rpcMallocCont(rspSize); if (pRsp == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b960103d94..218a86ed5c 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -302,9 +302,11 @@ int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp* rspMsg, bool syncOp) { _return: - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); - + if (output) { + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); + } + CTG_RET(code); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 1d2e3640a1..93d36bc4b3 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -252,7 +252,7 @@ int32_t ctgInitGetIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgIndexCtx* ctx = task.taskCtx; - strcpy(ctx->indexFName, name); + tstrncpy(ctx->indexFName, name, sizeof(ctx->indexFName)); taosArrayPush(pJob->pTasks, &task); @@ -277,7 +277,7 @@ int32_t ctgInitGetUdfTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgUdfCtx* ctx = task.taskCtx; - strcpy(ctx->udfName, name); + tstrncpy(ctx->udfName, name, sizeof(ctx->udfName)); taosArrayPush(pJob->pTasks, &task); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 89571ebf55..fe9a131ab8 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -660,7 +660,7 @@ int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) } msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->dbId = dbId; op->data = msg; @@ -693,7 +693,7 @@ int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) } msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); op->data = msg; @@ -721,8 +721,8 @@ int32_t ctgDropStbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, } msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - strncpy(msg->stbName, stbName, sizeof(msg->stbName)); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + tstrncpy(msg->stbName, stbName, sizeof(msg->stbName)); msg->dbId = dbId; msg->suid = suid; @@ -751,8 +751,8 @@ int32_t ctgDropTbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, } msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - strncpy(msg->tbName, tbName, sizeof(msg->tbName)); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + tstrncpy(msg->tbName, tbName, sizeof(msg->tbName)); msg->dbId = dbId; op->data = msg; @@ -785,7 +785,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId dbFName = p + 1; } - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->pCtg = pCtg; msg->dbId = dbId; msg->dbInfo = dbInfo; @@ -817,7 +817,8 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool sy char *p = strchr(output->dbFName, '.'); if (p && IS_SYS_DBNAME(p + 1)) { - memmove(output->dbFName, p + 1, strlen(p + 1)); + int32_t len = strlen(p + 1); + memmove(output->dbFName, p + 1, len >= TSDB_DB_FNAME_LEN ? TSDB_DB_FNAME_LEN - 1 : len); } msg->pCtg = pCtg; @@ -852,7 +853,7 @@ int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEp } msg->pCtg = pCtg; - strcpy(msg->dbFName, dbFName); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->vgId = vgId; msg->epSet = *pEpSet; @@ -1215,7 +1216,7 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_CACHE_STAT_INC(numOfDb, 1); SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); ctgDebug("db added to cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); @@ -1331,8 +1332,8 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uin metaRent.smaVer = pCache->pIndex->version; } - strcpy(metaRent.dbFName, dbFName); - strcpy(metaRent.stbName, tbName); + tstrncpy(metaRent.dbFName, dbFName, sizeof(metaRent.dbFName)); + tstrncpy(metaRent.stbName, tbName, sizeof(metaRent.stbName)); CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableVersion), ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); @@ -1416,8 +1417,10 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); - + if (pCache) { + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); + } + return TSDB_CODE_SUCCESS; } @@ -1584,7 +1587,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { dbCache = NULL; - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); @@ -1674,9 +1677,9 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { if (CTG_IS_META_TABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { int32_t metaSize = CTG_META_SIZE(pMeta->tbMeta); - CTG_ERR_JRET( - ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->tbName, pMeta->tbMeta, metaSize)); + code = ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->tbName, pMeta->tbMeta, metaSize); pMeta->tbMeta = NULL; + CTG_ERR_JRET(code); } if (CTG_IS_META_CTABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { @@ -1691,10 +1694,8 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { _return: - if (pMeta) { - taosMemoryFreeClear(pMeta->tbMeta); - taosMemoryFreeClear(pMeta); - } + taosMemoryFreeClear(pMeta->tbMeta); + taosMemoryFreeClear(pMeta); taosMemoryFreeClear(msg); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index a9134de537..b4229880b6 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -317,7 +317,12 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { SArray* pTagVals = NULL; STag* pTag = (STag*)pCfg->pTags; - if (pCfg->pTags && tTagIsJson(pTag)) { + if (NULL == pCfg->pTags || pCfg->numOfTags <= 0) { + qError("tag missed in table cfg, pointer:%p, numOfTags:%d", pCfg->pTags, pCfg->numOfTags); + return TSDB_CODE_APP_ERROR; + } + + if (tTagIsJson(pTag)) { char* pJson = parseTagDatatoJson(pTag); if (pJson) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 8f8e486b3a..9a27110345 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -143,9 +143,15 @@ static int32_t getStatus(SDataDispatchHandle* pDispatcher) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM); - if (NULL == pBuf || !allocBuf(pDispatcher, pInput, pBuf)) { + if (NULL == pBuf) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } + + if (!allocBuf(pDispatcher, pInput, pBuf)) { + taosFreeQitem(pBuf); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + toDataCacheEntry(pDispatcher, pInput, pBuf); taosWriteQitem(pDispatcher->pDataBlocks, pBuf); *pContinue = (DS_BUF_LOW == updateStatus(pDispatcher) ? true : false); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 4c4ba59fa9..ed455e5e75 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -323,7 +323,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat int32_t code = tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); if (code) { - destroyDataSinker((SDataSinkHandle*)pInserterNode); + destroyDataSinker((SDataSinkHandle*)inserter); return code; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index a917397d02..618f597d72 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -357,8 +357,7 @@ char* parseTagDatatoJson(void* p) { for (int j = 0; j < nCols; ++j) { STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j); // json key encode by binary - memset(tagJsonKey, 0, sizeof(tagJsonKey)); - memcpy(tagJsonKey, pTagVal->pKey, strlen(pTagVal->pKey)); + tstrncpy(tagJsonKey, pTagVal->pKey, sizeof(tagJsonKey)); // json value char type = pTagVal->type; if (type == TSDB_DATA_TYPE_NULL) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index a6f26088de..fadd07a9f3 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -173,7 +173,7 @@ int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t } SDbCfgReq dbCfgReq = {0}; - strcpy(dbCfgReq.db, input); + strncpy(dbCfgReq.db, input, sizeof(dbCfgReq.db) - 1); int32_t bufLen = tSerializeSDbCfgReq(NULL, 0, &dbCfgReq); void *pBuf = (*mallcFp)(bufLen); @@ -191,7 +191,7 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t } SUserIndexReq indexReq = {0}; - strcpy(indexReq.indexFName, input); + strncpy(indexReq.indexFName, input, sizeof(indexReq.indexFName) - 1); int32_t bufLen = tSerializeSUserIndexReq(NULL, 0, &indexReq); void *pBuf = (*mallcFp)(bufLen); @@ -233,7 +233,7 @@ int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32 } SGetUserAuthReq req = {0}; - strncpy(req.user, input, sizeof(req.user)); + strncpy(req.user, input, sizeof(req.user) - 1); int32_t bufLen = tSerializeSGetUserAuthReq(NULL, 0, &req); void *pBuf = (*mallcFp)(bufLen); @@ -251,7 +251,7 @@ int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_ } STableIndexReq indexReq = {0}; - strcpy(indexReq.tbFName, input); + strncpy(indexReq.tbFName, input, sizeof(indexReq.tbFName) - 1); int32_t bufLen = tSerializeSTableIndexReq(NULL, 0, &indexReq); void *pBuf = (*mallcFp)(bufLen); @@ -271,8 +271,8 @@ int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t SBuildTableInput *pInput = input; STableCfgReq cfgReq = {0}; cfgReq.header.vgId = pInput->vgId; - strncpy(cfgReq.dbFName, pInput->dbFName, sizeof(cfgReq.dbFName)); - strncpy(cfgReq.tbName, pInput->tbName, sizeof(cfgReq.tbName)); + strncpy(cfgReq.dbFName, pInput->dbFName, sizeof(cfgReq.dbFName) - 1); + strncpy(cfgReq.tbName, pInput->tbName, sizeof(cfgReq.tbName) - 1); int32_t bufLen = tSerializeSTableCfgReq(NULL, 0, &cfgReq); void *pBuf = (*mallcFp)(bufLen); diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 3038b87930..e9ded9b269 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -412,7 +412,7 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { while (true) { paramIdx = atomic_load_32(&gQwMgmt.paramIdx); if (paramIdx == tListLen(gQwMgmt.param)) { - newParamIdx = 0; + newParamIdx = 1; } else { newParamIdx = paramIdx + 1; } @@ -422,6 +422,10 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { } } + if (paramIdx == tListLen(gQwMgmt.param)) { + paramIdx = 0; + } + gQwMgmt.param[paramIdx].qwrId = gQwMgmt.qwRef; gQwMgmt.param[paramIdx].refId = refId; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 0ff40f3b9a..3df16563e2 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -398,7 +398,6 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu if (QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { QW_TASK_ELOG("task already dropped at wrong phase %s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_STATUS_ERROR); - break; } if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index c641b88152..b585373b0a 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -430,7 +430,8 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 if (SCH_IS_DATA_BIND_TASK(pTask)) { if (NULL == pData->pEpSet) { SCH_TASK_ELOG("no epset updated while got error %s", tstrerror(rspCode)); - SCH_ERR_JRET(rspCode); + code = rspCode; + goto _return; } } From ca2012a521ca4bab5c1253e586e2001efb559da8 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 18 Oct 2022 16:56:07 +0800 Subject: [PATCH 113/135] refactor: tmq commit --- source/client/src/clientTmq.c | 51 ++++++++++++++++-------------- source/dnode/vnode/src/inc/tq.h | 6 ++-- source/dnode/vnode/src/tq/tq.c | 4 +-- source/dnode/vnode/src/tq/tqExec.c | 2 +- source/dnode/vnode/src/tq/tqSink.c | 4 +-- 5 files changed, 36 insertions(+), 31 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 838b851d6d..0bc33739c6 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -210,6 +210,8 @@ typedef struct { typedef struct { SMqCommitCbParamSet* params; STqOffset* pOffset; + /*char topicName[TSDB_TOPIC_FNAME_LEN];*/ + /*int32_t vgId;*/ } SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { @@ -407,6 +409,14 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { return 0; } +static void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet) { + int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); + ASSERT(waitingRspNum >= 0); + if (waitingRspNum == 0) { + tmqCommitDone(pParamSet); + } +} + int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; @@ -420,18 +430,13 @@ int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { #endif taosMemoryFree(pParam->pOffset); - if (pBuf->pData) taosMemoryFree(pBuf->pData); + taosMemoryFree(pBuf->pData); /*tscDebug("receive offset commit cb of %s on vgId:%d, offset is %" PRId64, pParam->pOffset->subKey, pParam->->vgId, * pOffset->version);*/ - // count down waiting rsp - int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - ASSERT(waitingRspNum >= 0); + tmqCommitRspCountDown(pParamSet); - if (waitingRspNum == 0) { - tmqCommitDone(pParamSet); - } return 0; } @@ -591,14 +596,10 @@ FAIL: return 0; } -int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, - void* userParam) { +static int32_t tmqCommitConsumerImpl(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, + void* userParam) { int32_t code = -1; - if (msg != NULL) { - return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam); - } - SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -646,33 +647,37 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t } } + // no request is sent if (pParamSet->totalRspNum == 0) { tsem_destroy(&pParamSet->rspSem); taosMemoryFree(pParamSet); return 0; } - int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - ASSERT(waitingRspNum >= 0); - if (waitingRspNum == 0) { - tmqCommitDone(pParamSet); - } + // count down since waiting rsp num init as 1 + tmqCommitRspCountDown(pParamSet); if (!async) { tsem_wait(&pParamSet->rspSem); code = pParamSet->rspErr; tsem_destroy(&pParamSet->rspSem); taosMemoryFree(pParamSet); - } - #if 0 - if (!async) { taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); - } #endif + } - return 0; + return code; +} + +int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, + void* userParam) { + if (msg) { + return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam); + } else { + return tmqCommitConsumerImpl(tmq, automatic, async, userCb, userParam); + } } void tmqAssignAskEpTask(void* param, void* tmrId) { diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 69e6cdce9f..7f418439a8 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -149,7 +149,7 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); // tqRead -int32_t tqScan(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset); int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); @@ -181,8 +181,8 @@ int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink -void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); -void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t ver); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5e1cc15063..6d71496008 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -595,7 +595,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqInitTaosxRsp(&taosxRsp, pReq); if (fetchOffsetNew.type != TMQ_OFFSET__LOG) { - tqScan(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew); + tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew); if (metaRsp.metaRspLen > 0) { if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { @@ -924,7 +924,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqTableSink1; + pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline; ASSERT(pTask->tbSink.pSchemaWrapper); ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 58d051bec1..305ee82982 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -123,7 +123,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs return 0; } -int32_t tqScan(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) { +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) { const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index ab195b1eb3..79d7e27642 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -284,7 +284,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem return ret; } -void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { +void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pBlocks = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; int64_t suid = pTask->tbSink.stbUid; @@ -528,7 +528,7 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { taosArrayDestroy(tagArray); } -void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { +void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pRes = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; SBatchDeleteReq deleteReq = {0}; From 9e0feee081e1c6e77c877cd8926cf007d1c4bae0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Oct 2022 17:06:03 +0800 Subject: [PATCH 114/135] enh: sync propose return at once if replica is 1 --- source/dnode/mnode/impl/src/mndMnode.c | 1 + source/dnode/mnode/impl/src/mndSync.c | 8 ++++++++ source/libs/sync/src/syncMain.c | 6 ++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index f25d436aee..e23b895ebd 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -745,6 +745,7 @@ static void mndReloadSyncConfig(SMnode *pMnode) { mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes); return; } + // ASSERT(0); if (cfg.myIndex == -1) { #if 1 diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 866a3fa8b9..ac95dd2795 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -293,6 +293,14 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { if (code == 0) { tsem_wait(&pMgmt->syncSem); + } else if (code > 0) { + mInfo("trans:%d, confirm at once since replica is 1, continue execute", transId); + taosWLockLatch(&pMgmt->lock); + pMgmt->transId = 0; + taosWUnLockLatch(&pMgmt->lock); + sdbWriteWithoutFree(pMnode->pSdb, pRaw); + sdbSetApplyInfo(pMnode->pSdb, req.info.conn.applyIndex, req.info.conn.applyTerm, SYNC_INDEX_INVALID); + code = 0; } else if (code == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { terrno = TSDB_CODE_APP_NOT_READY; } else if (code == -1 && terrno == TSDB_CODE_SYN_INTERNAL_ERROR) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 83bd7a5f38..6e4eaa0aaf 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -3032,7 +3032,8 @@ static int32_t syncNodeProposeConfigChangeFinish(SSyncNode* ths, SyncReconfigFin } bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg) { - return (ths->replicaNum == 1 && syncUtilUserCommit(pMsg->msgType) && ths->vgId != 1); + return (ths->replicaNum == 1 && syncUtilUserCommit(pMsg->msgType)); + // return (ths->replicaNum == 1 && syncUtilUserCommit(pMsg->msgType) && ths->vgId != 1); } int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { @@ -3084,7 +3085,8 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, // user commit if ((ths->pFsm->FpCommitCb != NULL) && syncUtilUserCommit(pEntry->originalRpcType)) { bool internalExecute = true; - if ((ths->replicaNum == 1) && ths->restoreFinish && ths->vgId != 1) { + if ((ths->replicaNum == 1) && ths->restoreFinish) { + // if ((ths->replicaNum == 1) && ths->restoreFinish && ths->vgId != 1) { internalExecute = false; } From e2637ab6a78992457e4bc98efe2c3614a14b7651 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 18 Oct 2022 17:22:41 +0800 Subject: [PATCH 115/135] fix(query): improve performance for first function x10 quickly --- source/libs/function/src/builtins.c | 2 ++ source/libs/function/src/builtinsimpl.c | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 2308aaf214..49facfdd14 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2431,6 +2431,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_FIRST, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLast, + .dynDataRequiredFunc = firstDynDataReq, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = firstFunction, @@ -2445,6 +2446,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_FIRST_PARTIAL, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLastPartial, + .dynDataRequiredFunc = firstDynDataReq, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = firstFunction, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 7077a9b780..a756c27bb5 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2717,6 +2717,22 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) return TSDB_CODE_SUCCESS; } +EFuncDataRequired firstDynDataReq(void* pRes, STimeWindow* pTimeWindow) { + SResultRowEntryInfo* pEntry = (SResultRowEntryInfo*)pRes; + + // not initialized yet, data is required + if (pEntry == NULL) { + return FUNC_DATA_REQUIRED_DATA_LOAD; + } + + SFirstLastRes* pResult = GET_ROWCELL_INTERBUF(pEntry); + if (pResult->hasResult && pResult->ts <= pTimeWindow->skey) { + return FUNC_DATA_REQUIRED_NOT_LOAD; + } else { + return FUNC_DATA_REQUIRED_DATA_LOAD; + } +} + EFuncDataRequired lastDynDataReq(void* pRes, STimeWindow* pTimeWindow) { SResultRowEntryInfo* pEntry = (SResultRowEntryInfo*)pRes; From 85ec70d099cc44e3218f74de6183cd141bf0d5cd Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 18 Oct 2022 17:26:51 +0800 Subject: [PATCH 116/135] fix(query): improve performance for first function x10 quickly1 --- source/libs/function/inc/builtinsimpl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index b5b7453e7a..dfdf03a5a8 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -118,6 +118,7 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pB int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getFirstLastInfoSize(int32_t resBytes); +EFuncDataRequired firstDynDataReq(void* pRes, STimeWindow* pTimeWindow); EFuncDataRequired lastDynDataReq(void* pRes, STimeWindow* pTimeWindow); int32_t lastRowFunction(SqlFunctionCtx* pCtx); From 3a7d89a6bbf3940427c9280fa8d2378b6614f5b9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Oct 2022 17:34:58 +0800 Subject: [PATCH 117/135] refact: remove bnode codes --- include/common/systable.h | 2 +- include/common/tglobal.h | 1 - include/common/tmsg.h | 9 +- include/common/tmsgdef.h | 11 +- include/dnode/bnode/bnode.h | 77 --- include/libs/monitor/monitor.h | 1 - include/util/tdef.h | 1 - source/common/src/systable.c | 7 - source/common/src/tglobal.c | 9 - source/common/src/tmsg.c | 4 +- source/dnode/CMakeLists.txt | 1 - source/dnode/bnode/CMakeLists.txt | 14 - source/dnode/bnode/inc/bndInt.h | 40 -- source/dnode/bnode/src/bnode.c | 28 -- source/dnode/mgmt/CMakeLists.txt | 1 - source/dnode/mgmt/mgmt_bnode/CMakeLists.txt | 9 - source/dnode/mgmt/mgmt_bnode/inc/bmInt.h | 53 --- source/dnode/mgmt/mgmt_bnode/src/bmHandle.c | 105 ---- source/dnode/mgmt/mgmt_bnode/src/bmInt.c | 79 ---- source/dnode/mgmt/mgmt_bnode/src/bmWorker.c | 150 ------ source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 2 - source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 6 - source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 6 +- source/dnode/mgmt/mgmt_qnode/src/qmHandle.c | 4 +- source/dnode/mgmt/mgmt_snode/src/smHandle.c | 4 +- source/dnode/mgmt/node_mgmt/CMakeLists.txt | 2 +- source/dnode/mgmt/node_mgmt/inc/dmNodes.h | 1 - source/dnode/mgmt/node_mgmt/src/dmMgmt.c | 1 - source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 17 - source/dnode/mgmt/node_mgmt/src/dmNodes.c | 2 - source/dnode/mgmt/node_util/CMakeLists.txt | 2 +- source/dnode/mgmt/node_util/inc/dmUtil.h | 3 +- source/dnode/mgmt/node_util/src/dmUtil.c | 6 - source/dnode/mgmt/test/CMakeLists.txt | 1 - source/dnode/mgmt/test/bnode/CMakeLists.txt | 10 - source/dnode/mgmt/test/bnode/dbnode.cpp | 152 ------ source/dnode/mgmt/test/mnode/dmnode.cpp | 12 +- source/dnode/mgmt/test/qnode/dqnode.cpp | 36 +- source/dnode/mgmt/test/snode/dsnode.cpp | 36 +- source/dnode/mnode/impl/inc/mndBnode.h | 32 -- source/dnode/mnode/impl/inc/mndDef.h | 9 - source/dnode/mnode/impl/src/mndBnode.c | 447 ------------------ source/dnode/mnode/impl/src/mndDump.c | 5 - source/dnode/mnode/impl/src/mndMain.c | 2 - source/dnode/mnode/impl/src/mndMnode.c | 8 +- source/dnode/mnode/impl/src/mndQnode.c | 16 +- source/dnode/mnode/impl/src/mndShow.c | 2 - source/dnode/mnode/impl/src/mndSnode.c | 16 +- source/dnode/mnode/impl/test/CMakeLists.txt | 1 - .../mnode/impl/test/bnode/CMakeLists.txt | 11 - source/dnode/mnode/impl/test/bnode/mbnode.cpp | 293 ------------ source/dnode/mnode/impl/test/mnode/mnode.cpp | 52 +- source/dnode/mnode/impl/test/qnode/qnode.cpp | 56 +-- source/dnode/mnode/impl/test/snode/snode.cpp | 56 +-- source/dnode/mnode/impl/test/trans/trans1.cpp | 20 +- source/dnode/mnode/sdb/inc/sdb.h | 1 - source/dnode/mnode/sdb/src/sdbHash.c | 2 - source/libs/monitor/src/monMain.c | 1 - source/libs/parser/src/parTranslater.c | 4 +- source/libs/parser/test/parInitialCTest.cpp | 19 - source/libs/parser/test/parInitialDTest.cpp | 6 - source/util/src/terror.c | 2 - tests/script/jenkins/basic.txt | 3 - tests/script/tmp/monitor.sim | 1 - tests/script/tsim/bnode/basic1.sim | 135 ------ tests/script/tsim/testsuit.sim | 3 +- tests/script/tsim/user/privilege_sysinfo.sim | 1 - tests/system-test/0-others/taosdMonitor.py | 2 +- 68 files changed, 178 insertions(+), 1933 deletions(-) delete mode 100644 include/dnode/bnode/bnode.h delete mode 100644 source/dnode/bnode/CMakeLists.txt delete mode 100644 source/dnode/bnode/inc/bndInt.h delete mode 100644 source/dnode/bnode/src/bnode.c delete mode 100644 source/dnode/mgmt/mgmt_bnode/CMakeLists.txt delete mode 100644 source/dnode/mgmt/mgmt_bnode/inc/bmInt.h delete mode 100644 source/dnode/mgmt/mgmt_bnode/src/bmHandle.c delete mode 100644 source/dnode/mgmt/mgmt_bnode/src/bmInt.c delete mode 100644 source/dnode/mgmt/mgmt_bnode/src/bmWorker.c delete mode 100644 source/dnode/mgmt/test/bnode/CMakeLists.txt delete mode 100644 source/dnode/mgmt/test/bnode/dbnode.cpp delete mode 100644 source/dnode/mnode/impl/inc/mndBnode.h delete mode 100644 source/dnode/mnode/impl/src/mndBnode.c delete mode 100644 source/dnode/mnode/impl/test/bnode/CMakeLists.txt delete mode 100644 source/dnode/mnode/impl/test/bnode/mbnode.cpp delete mode 100644 tests/script/tsim/bnode/basic1.sim diff --git a/include/common/systable.h b/include/common/systable.h index 102465c8c2..8b29525db3 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -27,7 +27,7 @@ extern "C" { #define TSDB_INS_TABLE_MNODES "ins_mnodes" #define TSDB_INS_TABLE_MODULES "ins_modules" #define TSDB_INS_TABLE_QNODES "ins_qnodes" -#define TSDB_INS_TABLE_BNODES "ins_bnodes" +#define TSDB_INS_TABLE_BNODES "ins_bnodes" // no longer used #define TSDB_INS_TABLE_SNODES "ins_snodes" #define TSDB_INS_TABLE_CLUSTER "ins_cluster" #define TSDB_INS_TABLE_DATABASES "ins_databases" diff --git a/include/common/tglobal.h b/include/common/tglobal.h index fd0521af3b..2d196be44c 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -51,7 +51,6 @@ extern int32_t tsMnodeShmSize; extern int32_t tsVnodeShmSize; extern int32_t tsQnodeShmSize; extern int32_t tsSnodeShmSize; -extern int32_t tsBnodeShmSize; extern int32_t tsNumOfShmThreads; // queue & threads diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5e2ca8896a..e25fbe6201 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -95,7 +95,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QNODE, TSDB_MGMT_TABLE_SNODE, - TSDB_MGMT_TABLE_BNODE, + TSDB_MGMT_TABLE_BNODE, // no longer used TSDB_MGMT_TABLE_CLUSTER, TSDB_MGMT_TABLE_DB, TSDB_MGMT_TABLE_FUNC, @@ -1485,11 +1485,10 @@ int32_t tDeserializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq typedef struct { int32_t dnodeId; } SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq, - SMCreateSnodeReq, SMDropSnodeReq, SDCreateSnodeReq, SDDropSnodeReq, SMCreateBnodeReq, SMDropBnodeReq, - SDCreateBnodeReq, SDDropBnodeReq; + SMCreateSnodeReq, SMDropSnodeReq, SDCreateSnodeReq, SDDropSnodeReq; -int32_t tSerializeSCreateDropMQSBNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); -int32_t tDeserializeSCreateDropMQSBNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); +int32_t tSerializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); +int32_t tDeserializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); typedef struct { int8_t replica; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 3f917ff0d1..1a8865bf83 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -74,8 +74,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_DROP_QNODE, "dnode-drop-qnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CREATE_SNODE, "dnode-create-snode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_SNODE, "dnode-drop-snode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_CREATE_BNODE, "dnode-create-bnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_DROP_BNODE, "dnode-drop-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_BNODE, "dnode-create-bnode", NULL, NULL) // no longer used + TD_DEF_MSG_TYPE(TDMT_DND_DROP_BNODE, "dnode-drop-bnode", NULL, NULL) // no longer used TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "create-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "drop-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "server-status", NULL, NULL) @@ -107,9 +107,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SNODE, "create-snode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_SNODE, "alter-snode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_SNODE, "drop-snode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_BNODE, "create-bnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_BNODE, "alter-bnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_BNODE, "drop-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_BNODE, "create-bnode", NULL, NULL) // no longer used + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_BNODE, "alter-bnode", NULL, NULL) // no longer used + TD_DEF_MSG_TYPE(TDMT_MND_DROP_BNODE, "drop-bnode", NULL, NULL) // no longer used TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DB, "create-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_DB, "drop-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_USE_DB, "use-db", NULL, NULL) @@ -243,7 +243,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_MON_VM_INFO, "monitor-vinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_QM_INFO, "monitor-qinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_SM_INFO, "monitor-sinfo", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MON_BM_INFO, "monitor-binfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_VM_LOAD, "monitor-vload", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_MM_LOAD, "monitor-mload", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_QM_LOAD, "monitor-qload", NULL, NULL) diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h deleted file mode 100644 index d5958d432b..0000000000 --- a/include/dnode/bnode/bnode.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_BNODE_H_ -#define _TD_BNODE_H_ - -#include "tmsgcb.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SBnode SBnode; - -typedef struct { -#ifdef WINDOWS - size_t avoidCompilationErrors; -#endif -} SBnodeLoad; - -typedef struct { - SMsgCb msgCb; -} SBnodeOpt; - -/* ------------------------ SBnode ------------------------ */ -/** - * @brief Start one Bnode in Dnode. - * - * @param path Path of the bnode. - * @param pOption Option of the bnode. - * @return SBnode* The bnode object. - */ -SBnode *bndOpen(const char *path, const SBnodeOpt *pOption); - -/** - * @brief Stop Bnode in Dnode. - * - * @param pBnode The bnode object to close. - */ -void bndClose(SBnode *pBnode); - -/** - * @brief Get the statistical information of Bnode - * - * @param pBnode The bnode object. - * @param pLoad Statistics of the bnode. - * @return int32_t 0 for success, -1 for failure. - */ -int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad); - -/** - * @brief Process a query message. - * - * @param pBnode The bnode object. - * @param pMsgs The array of SRpcMsg - * @return int32_t 0 for success, -1 for failure - */ -int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_BNODE_H_*/ \ No newline at end of file diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 30254d61a9..12e465e0f8 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -80,7 +80,6 @@ typedef struct { int8_t has_mnode; int8_t has_qnode; int8_t has_snode; - int8_t has_bnode; SMonDiskDesc logdir; SMonDiskDesc tempdir; } SMonDnodeInfo; diff --git a/include/util/tdef.h b/include/util/tdef.h index d9bb558b74..10dd686a5f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -483,7 +483,6 @@ enum { #define QNODE_HANDLE -1 #define SNODE_HANDLE -2 #define VNODE_HANDLE -3 -#define BNODE_HANDLE -4 #define CLIENT_HANDLE -5 #define TSDB_CONFIG_OPTION_LEN 32 diff --git a/source/common/src/systable.c b/source/common/src/systable.c index a45f7b2913..28a4732abd 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -62,12 +62,6 @@ static const SSysDbTableSchema snodesSchema[] = { {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, }; -static const SSysDbTableSchema bnodesSchema[] = { - {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, -}; - static const SSysDbTableSchema clusterSchema[] = { {.name = "id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true}, {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, @@ -277,7 +271,6 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema), true}, {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema), true}, // {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, - // {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema), true}, {TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema), false}, {TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema), false}, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 56daa4075a..2b64d3f1ee 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -46,7 +46,6 @@ int32_t tsMnodeShmSize = TSDB_MAX_MSG_SIZE * 2 + 1024; int32_t tsVnodeShmSize = TSDB_MAX_MSG_SIZE * 10 + 1024; int32_t tsQnodeShmSize = TSDB_MAX_MSG_SIZE * 4 + 1024; int32_t tsSnodeShmSize = TSDB_MAX_MSG_SIZE * 4 + 1024; -int32_t tsBnodeShmSize = TSDB_MAX_MSG_SIZE * 4 + 1024; int32_t tsNumOfShmThreads = 1; // queue & threads @@ -358,7 +357,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "vnodeShmSize", tsVnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; if (cfgAddInt32(pCfg, "qnodeShmSize", tsQnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; if (cfgAddInt32(pCfg, "snodeShmSize", tsSnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "bnodeShmSize", tsBnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; if (cfgAddInt32(pCfg, "numOfShmThreads", tsNumOfShmThreads, 1, 1024, 0) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; @@ -707,7 +705,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsVnodeShmSize = cfgGetItem(pCfg, "vnodeShmSize")->i32; tsQnodeShmSize = cfgGetItem(pCfg, "qnodeShmSize")->i32; tsSnodeShmSize = cfgGetItem(pCfg, "snodeShmSize")->i32; - tsBnodeShmSize = cfgGetItem(pCfg, "bnodeShmSize")->i32; tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32; tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; @@ -780,12 +777,6 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { } break; } - case 'b': { - if (strcasecmp("bnodeShmSize", name) == 0) { - tsBnodeShmSize = cfgGetItem(pCfg, "bnodeShmSize")->i32; - } - break; - } case 'c': { if (strcasecmp("charset", name) == 0) { const char *locale = cfgGetItem(pCfg, "locale")->str; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 55e5616dd5..1f0023278f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1448,7 +1448,7 @@ void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) { taosHashCleanup(pRsp->writeDbs); } -int32_t tSerializeSCreateDropMQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { +int32_t tSerializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -1461,7 +1461,7 @@ int32_t tSerializeSCreateDropMQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnod return tlen; } -int32_t tDeserializeSCreateDropMQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { +int32_t tDeserializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); diff --git a/source/dnode/CMakeLists.txt b/source/dnode/CMakeLists.txt index 87e4c5fc46..035cea2dc4 100644 --- a/source/dnode/CMakeLists.txt +++ b/source/dnode/CMakeLists.txt @@ -2,5 +2,4 @@ add_subdirectory(mnode) add_subdirectory(vnode) add_subdirectory(qnode) add_subdirectory(snode) -add_subdirectory(bnode) add_subdirectory(mgmt) \ No newline at end of file diff --git a/source/dnode/bnode/CMakeLists.txt b/source/dnode/bnode/CMakeLists.txt deleted file mode 100644 index 77f4d15ad2..0000000000 --- a/source/dnode/bnode/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -aux_source_directory(src BNODE_SRC) -add_library(bnode STATIC ${BNODE_SRC}) -target_include_directories( - bnode - PUBLIC "${TD_SOURCE_DIR}/include/dnode/bnode" - private "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) -target_link_libraries( - bnode - PRIVATE transport - PRIVATE os - PRIVATE common - PRIVATE util -) \ No newline at end of file diff --git a/source/dnode/bnode/inc/bndInt.h b/source/dnode/bnode/inc/bndInt.h deleted file mode 100644 index e611d230a3..0000000000 --- a/source/dnode/bnode/inc/bndInt.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_BNODE_INT_H_ -#define _TD_BNODE_INT_H_ - -#include "os.h" - -#include "tarray.h" -#include "tlog.h" -#include "tmsg.h" -#include "trpc.h" - -#include "bnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SBnode { - SMsgCb msgCb; -} SBnode; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_BNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/bnode/src/bnode.c b/source/dnode/bnode/src/bnode.c deleted file mode 100644 index b9c41ebf43..0000000000 --- a/source/dnode/bnode/src/bnode.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "bndInt.h" - -SBnode *bndOpen(const char *path, const SBnodeOpt *pOption) { - SBnode *pBnode = taosMemoryCalloc(1, sizeof(SBnode)); - pBnode->msgCb = pOption->msgCb; - return pBnode; -} - -void bndClose(SBnode *pBnode) { taosMemoryFree(pBnode); } - -int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; } - -int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; } diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 581686ba90..423028b167 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -1,6 +1,5 @@ add_subdirectory(node_mgmt) add_subdirectory(node_util) -add_subdirectory(mgmt_bnode) add_subdirectory(mgmt_mnode) add_subdirectory(mgmt_qnode) add_subdirectory(mgmt_snode) diff --git a/source/dnode/mgmt/mgmt_bnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_bnode/CMakeLists.txt deleted file mode 100644 index 0a6cf52fb8..0000000000 --- a/source/dnode/mgmt/mgmt_bnode/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -aux_source_directory(src MGMT_BNODE) -add_library(mgmt_bnode STATIC ${MGMT_BNODE}) -target_include_directories( - mgmt_bnode - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) -target_link_libraries( - mgmt_bnode node_util -) \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_bnode/inc/bmInt.h b/source/dnode/mgmt/mgmt_bnode/inc/bmInt.h deleted file mode 100644 index c05ad46189..0000000000 --- a/source/dnode/mgmt/mgmt_bnode/inc/bmInt.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_BNODE_INT_H_ -#define _TD_DND_BNODE_INT_H_ - -#include "dmUtil.h" - -#include "bnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SBnodeMgmt { - SDnodeData *pData; - SBnode *pBnode; - SMsgCb msgCb; - const char *path; - const char *name; - SMultiWorker writeWorker; - SSingleWorker monitorWorker; -} SBnodeMgmt; - -// bmHandle.c -SArray *bmGetMsgHandles(); -int32_t bmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); -int32_t bmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); -int32_t bmProcessGetMonBmInfoReq(SBnodeMgmt *pMgmt, SRpcMsg *pMsg); - -// bmWorker.c -int32_t bmStartWorker(SBnodeMgmt *pMgmt); -void bmStopWorker(SBnodeMgmt *pMgmt); -int32_t bmPutNodeMsgToWriteQueue(SBnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t bmPutNodeMsgToMonitorQueue(SBnodeMgmt *pMgmt, SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_BNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c b/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c deleted file mode 100644 index 9ec445c69c..0000000000 --- a/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "bmInt.h" - -void bmGetMonitorInfo(SBnodeMgmt *pMgmt, SMonBmInfo *bmInfo) {} - -int32_t bmProcessGetMonBmInfoReq(SBnodeMgmt *pMgmt, SRpcMsg *pMsg) { - SMonBmInfo bmInfo = {0}; - bmGetMonitorInfo(pMgmt, &bmInfo); - dmGetMonitorSystemInfo(&bmInfo.sys); - monGetLogs(&bmInfo.log); - - int32_t rspLen = tSerializeSMonBmInfo(NULL, 0, &bmInfo); - if (rspLen < 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - void *pRsp = rpcMallocCont(rspLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - tSerializeSMonBmInfo(pRsp, rspLen, &bmInfo); - pMsg->info.rsp = pRsp; - pMsg->info.rspLen = rspLen; - tFreeSMonBmInfo(&bmInfo); - return 0; -} - -int32_t bmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { - SDCreateBnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) { - terrno = TSDB_CODE_INVALID_OPTION; - dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pInput->pData->dnodeId); - return -1; - } - - bool deployed = true; - if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { - dError("failed to write bnode file since %s", terrstr()); - return -1; - } - - return 0; -} - -int32_t bmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { - SDDropBnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) { - terrno = TSDB_CODE_INVALID_OPTION; - dError("failed to drop bnode since %s", terrstr()); - return -1; - } - - bool deployed = false; - if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { - dError("failed to write bnode file since %s", terrstr()); - return -1; - } - - return 0; -} - -SArray *bmGetMsgHandles() { - int32_t code = -1; - SArray *pArray = taosArrayInit(2, sizeof(SMgmtHandle)); - if (pArray == NULL) goto _OVER; - - if (dmSetMgmtHandle(pArray, TDMT_MON_BM_INFO, bmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; - - code = 0; -_OVER: - if (code != 0) { - taosArrayDestroy(pArray); - return NULL; - } else { - return pArray; - } -} diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmInt.c b/source/dnode/mgmt/mgmt_bnode/src/bmInt.c deleted file mode 100644 index 2c5d23cae9..0000000000 --- a/source/dnode/mgmt/mgmt_bnode/src/bmInt.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "bmInt.h" - -static int32_t bmRequire(const SMgmtInputOpt *pInput, bool *required) { - return dmReadFile(pInput->path, pInput->name, required); -} - -static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) { pOption->msgCb = pMgmt->msgCb; } - -static void bmClose(SBnodeMgmt *pMgmt) { - if (pMgmt->pBnode != NULL) { - bmStopWorker(pMgmt); - bndClose(pMgmt->pBnode); - pMgmt->pBnode = NULL; - } - - taosMemoryFree(pMgmt); -} - -int32_t bmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { - SBnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SBnodeMgmt)); - if (pMgmt == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pMgmt->pData = pInput->pData; - pMgmt->path = pInput->path; - pMgmt->name = pInput->name; - pMgmt->msgCb = pInput->msgCb; - pMgmt->msgCb.mgmt = pMgmt; - - SBnodeOpt option = {0}; - bmInitOption(pMgmt, &option); - pMgmt->pBnode = bndOpen(pMgmt->path, &option); - if (pMgmt->pBnode == NULL) { - dError("failed to open bnode since %s", terrstr()); - bmClose(pMgmt); - return -1; - } - tmsgReportStartup("bnode-impl", "initialized"); - - if (bmStartWorker(pMgmt) != 0) { - dError("failed to start bnode worker since %s", terrstr()); - bmClose(pMgmt); - return -1; - } - tmsgReportStartup("bnode-worker", "initialized"); - - pOutput->pMgmt = pMgmt; - return 0; -} - -SMgmtFunc bmGetMgmtFunc() { - SMgmtFunc mgmtFunc = {0}; - mgmtFunc.openFp = bmOpen; - mgmtFunc.closeFp = (NodeCloseFp)bmClose; - mgmtFunc.createFp = (NodeCreateFp)bmProcessCreateReq; - mgmtFunc.dropFp = (NodeDropFp)bmProcessDropReq; - mgmtFunc.requiredFp = bmRequire; - mgmtFunc.getHandlesFp = bmGetMsgHandles; - - return mgmtFunc; -} diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c deleted file mode 100644 index 08c9edd854..0000000000 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "bmInt.h" - -static void bmSendErrorRsp(SRpcMsg *pMsg, int32_t code) { - SRpcMsg rsp = {.code = code, .info = pMsg->info}; - tmsgSendRsp(&rsp); - - dTrace("msg:%p, is freed", pMsg); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void bmSendErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) { - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - if (pMsg != NULL) { - bmSendErrorRsp(pMsg, code); - } - } -} - -static inline void bmSendRsp(SRpcMsg *pMsg, int32_t code) { - SRpcMsg rsp = { - .code = code, - .pCont = pMsg->info.rsp, - .contLen = pMsg->info.rspLen, - .info = pMsg->info, - }; - tmsgSendRsp(&rsp); -} - -static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SBnodeMgmt *pMgmt = pInfo->ahandle; - int32_t code = -1; - dTrace("msg:%p, get from bnode-monitor queue", pMsg); - - if (pMsg->msgType == TDMT_MON_BM_INFO) { - code = bmProcessGetMonBmInfoReq(pMgmt, pMsg); - } else { - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - } - - if (IsReq(pMsg)) { - if (code != 0 && terrno != 0) code = terrno; - bmSendRsp(pMsg, code); - } - - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void bmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { - SBnodeMgmt *pMgmt = pInfo->ahandle; - - SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); - if (pArray == NULL) { - bmSendErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); - return; - } - - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - if (pMsg != NULL) { - dTrace("msg:%p, get from bnode-write queue", pMsg); - if (taosArrayPush(pArray, &pMsg) == NULL) { - bmSendErrorRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); - } - } - } - - bndProcessWMsgs(pMgmt->pBnode, pArray); - - for (size_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - if (pMsg != NULL) { - dTrace("msg:%p, is freed", pMsg); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - } - taosArrayDestroy(pArray); -} - -int32_t bmPutNodeMsgToWriteQueue(SBnodeMgmt *pMgmt, SRpcMsg *pMsg) { - SMultiWorker *pWorker = &pMgmt->writeWorker; - - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); - taosWriteQitem(pWorker->queue, pMsg); - return 0; -} - -int32_t bmPutNodeMsgToMonitorQueue(SBnodeMgmt *pMgmt, SRpcMsg *pMsg) { - SSingleWorker *pWorker = &pMgmt->monitorWorker; - - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); - taosWriteQitem(pWorker->queue, pMsg); - return 0; -} - -int32_t bmStartWorker(SBnodeMgmt *pMgmt) { - SMultiWorkerCfg cfg = { - .max = 1, - .name = "bnode-write", - .fp = (FItems)bmProcessWriteQueue, - .param = pMgmt, - }; - if (tMultiWorkerInit(&pMgmt->writeWorker, &cfg) != 0) { - dError("failed to start bnode-write worker since %s", terrstr()); - return -1; - } - - SSingleWorkerCfg mCfg = { - .min = 1, - .max = 1, - .name = "bnode-monitor", - .fp = (FItem)bmProcessMonitorQueue, - .param = pMgmt, - }; - if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { - dError("failed to start bnode-monitor worker since %s", terrstr()); - return -1; - } - - dDebug("bnode workers are initialized"); - return 0; -} - -void bmStopWorker(SBnodeMgmt *pMgmt) { - tSingleWorkerCleanup(&pMgmt->monitorWorker); - tMultiWorkerCleanup(&pMgmt->writeWorker); - dDebug("bnode workers are closed"); -} diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index dd98816161..782c828bb2 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -326,8 +326,6 @@ SArray *dmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index e7ccce1c8b..ca8a434d05 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -132,12 +132,6 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_DND_DROP_SNODE: code = (*pMgmt->processDropNodeFp)(SNODE, pMsg); break; - case TDMT_DND_CREATE_BNODE: - code = (*pMgmt->processCreateNodeFp)(BNODE, pMsg); - break; - case TDMT_DND_DROP_BNODE: - code = (*pMgmt->processDropNodeFp)(BNODE, pMsg); - break; case TDMT_DND_SERVER_STATUS: code = dmProcessServerRunStatus(pMgmt, pMsg); break; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index fc45dbf15f..105b6e542e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -105,7 +105,7 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; SDDropMnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -136,8 +136,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -163,8 +161,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DNODE_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_BNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_BNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c index 6382152cd1..0573be90ea 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c @@ -56,7 +56,7 @@ int32_t qmProcessGetMonitorInfoReq(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SDCreateQnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -78,7 +78,7 @@ int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t qmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SDDropQnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 7cb41ca77c..c6e3b3611f 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -45,7 +45,7 @@ int32_t smProcessGetMonitorInfoReq(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SDCreateSnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -67,7 +67,7 @@ int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t smProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SDDropSnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt index 98027d80d4..f1be20289a 100644 --- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt @@ -1,7 +1,7 @@ aux_source_directory(src IMPLEMENT_SRC) add_library(dnode STATIC ${IMPLEMENT_SRC}) target_link_libraries( - dnode mgmt_bnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode + dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode ) target_include_directories( dnode diff --git a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h index d3f1044f88..5f36309038 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h @@ -23,7 +23,6 @@ extern "C" { #endif SMgmtFunc dmGetMgmtFunc(); -SMgmtFunc bmGetMgmtFunc(); SMgmtFunc qmGetMgmtFunc(); SMgmtFunc smGetMgmtFunc(); SMgmtFunc vmGetMgmtFunc(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 2e149be3b6..75c76a1f54 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -141,7 +141,6 @@ int32_t dmInitDnode(SDnode *pDnode, EDndNodeType rtype) { pDnode->wrappers[VNODE].func = vmGetMgmtFunc(); pDnode->wrappers[QNODE].func = qmGetMgmtFunc(); pDnode->wrappers[SNODE].func = smGetMgmtFunc(); - pDnode->wrappers[BNODE].func = bmGetMgmtFunc(); for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index 1c71fe5e1f..be1de02e6a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -41,7 +41,6 @@ static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { pInfo->has_mnode = pDnode->wrappers[MNODE].required; pInfo->has_qnode = pDnode->wrappers[QNODE].required; pInfo->has_snode = pDnode->wrappers[SNODE].required; - pInfo->has_bnode = pDnode->wrappers[BNODE].required; tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); pInfo->logdir.size = tsLogSpace.size; tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); @@ -116,21 +115,6 @@ static void dmGetSmMonitorInfo(SDnode *pDnode) { } } -static void dmGetBmMonitorInfo(SDnode *pDnode) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[BNODE]; - if (dmMarkWrapper(pWrapper) == 0) { - SMonBmInfo bmInfo = {0}; - if (tsMultiProcess) { - dmSendLocalRecv(pDnode, TDMT_MON_BM_INFO, tDeserializeSMonBmInfo, &bmInfo); - } else if (pWrapper->pMgmt != NULL) { - bmGetMonitorInfo(pWrapper->pMgmt, &bmInfo); - } - dmReleaseWrapper(pWrapper); - monSetBmInfo(&bmInfo); - tFreeSMonBmInfo(&bmInfo); - } -} - void dmSendMonitorReport() { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); @@ -141,7 +125,6 @@ void dmSendMonitorReport() { dmGetVmMonitorInfo(pDnode); dmGetQmMonitorInfo(pDnode); dmGetSmMonitorInfo(pDnode); - dmGetBmMonitorInfo(pDnode); monSendReport(); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 99ffd73a7a..ec5f3589e0 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -26,8 +26,6 @@ static int32_t dmCreateShm(SMgmtWrapper *pWrapper) { shmsize = tsSnodeShmSize; } else if (pWrapper->ntype == MNODE) { shmsize = tsMnodeShmSize; - } else if (pWrapper->ntype == BNODE) { - shmsize = tsBnodeShmSize; } else { return -1; } diff --git a/source/dnode/mgmt/node_util/CMakeLists.txt b/source/dnode/mgmt/node_util/CMakeLists.txt index 5d879fdbcf..5c670cbdd3 100644 --- a/source/dnode/mgmt/node_util/CMakeLists.txt +++ b/source/dnode/mgmt/node_util/CMakeLists.txt @@ -6,5 +6,5 @@ target_include_directories( PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - node_util cjson mnode vnode qnode snode bnode wal sync taos_static tfs monitor + node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor ) \ No newline at end of file diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index f599de384c..2f9c0aff2f 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -68,8 +68,7 @@ typedef enum { VNODE = 2, QNODE = 3, SNODE = 4, - BNODE = 5, - NODE_END = 6, + NODE_END = 5, } EDndNodeType; typedef enum { diff --git a/source/dnode/mgmt/node_util/src/dmUtil.c b/source/dnode/mgmt/node_util/src/dmUtil.c index 832e15a1e0..f5364b3195 100644 --- a/source/dnode/mgmt/node_util/src/dmUtil.c +++ b/source/dnode/mgmt/node_util/src/dmUtil.c @@ -39,8 +39,6 @@ const char *dmNodeLogName(EDndNodeType ntype) { return "snode"; case MNODE: return "mnode"; - case BNODE: - return "bnode"; default: return "taosd"; } @@ -56,8 +54,6 @@ const char *dmNodeProcName(EDndNodeType ntype) { return "taoss"; case MNODE: return "taosm"; - case BNODE: - return "taosb"; default: return "taosd"; } @@ -73,8 +69,6 @@ const char *dmNodeName(EDndNodeType ntype) { return "snode"; case MNODE: return "mnode"; - case BNODE: - return "bnode"; default: return "dnode"; } diff --git a/source/dnode/mgmt/test/CMakeLists.txt b/source/dnode/mgmt/test/CMakeLists.txt index 6b1919bf18..6709a2aa5e 100644 --- a/source/dnode/mgmt/test/CMakeLists.txt +++ b/source/dnode/mgmt/test/CMakeLists.txt @@ -1,7 +1,6 @@ if(${BUILD_TEST}) enable_testing() add_subdirectory(qnode) - add_subdirectory(bnode) add_subdirectory(snode) #add_subdirectory(mnode) add_subdirectory(vnode) diff --git a/source/dnode/mgmt/test/bnode/CMakeLists.txt b/source/dnode/mgmt/test/bnode/CMakeLists.txt deleted file mode 100644 index 7108d3adb9..0000000000 --- a/source/dnode/mgmt/test/bnode/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -aux_source_directory(. DND_BNODE_TEST_SRC) -add_executable(dbnodeTest ${DND_BNODE_TEST_SRC}) -target_link_libraries( - dbnodeTest sut -) - -add_test( - NAME dbnodeTest - COMMAND dbnodeTest -) diff --git a/source/dnode/mgmt/test/bnode/dbnode.cpp b/source/dnode/mgmt/test/bnode/dbnode.cpp deleted file mode 100644 index 24d8c594ce..0000000000 --- a/source/dnode/mgmt/test/bnode/dbnode.cpp +++ /dev/null @@ -1,152 +0,0 @@ -/** - * @file dbnode.cpp - * @author slguan (slguan@taosdata.com) - * @brief DNODE module bnode tests - * @version 1.0 - * @date 2022-01-05 - * - * @copyright Copyright (c) 2022 - * - */ - -#include "sut.h" - -class DndTestBnode : public ::testing::Test { - protected: - static void SetUpTestSuite() { - test.Init(TD_TMP_DIR_PATH "dbnodeTest", 9112); - taosMsleep(1100); - } - static void TearDownTestSuite() { test.Cleanup(); } - static Testbase test; - - public: - void SetUp() override {} - void TearDown() override {} -}; - -Testbase DndTestBnode::test; - -TEST_F(DndTestBnode, 01_Create_Bnode) { - { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); - } - - { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - } - - { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); - } - - // test.Restart(); - - { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); - } -} - -TEST_F(DndTestBnode, 02_Drop_Bnode) { - { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); - } - - { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - } - - { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); - } - - // test.Restart(); - - { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); - } - - { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - } -} diff --git a/source/dnode/mgmt/test/mnode/dmnode.cpp b/source/dnode/mgmt/test/mnode/dmnode.cpp index 857f58befc..7b7eb97216 100644 --- a/source/dnode/mgmt/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/test/mnode/dmnode.cpp @@ -139,9 +139,9 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SDDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -152,9 +152,9 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SDDropMnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -165,9 +165,9 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SDDropMnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mgmt/test/qnode/dqnode.cpp b/source/dnode/mgmt/test/qnode/dqnode.cpp index ef51be47a6..a2c6a2c28c 100644 --- a/source/dnode/mgmt/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/test/qnode/dqnode.cpp @@ -30,9 +30,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -43,9 +43,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -56,9 +56,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -71,9 +71,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -87,9 +87,9 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -101,9 +101,9 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -114,9 +114,9 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -129,9 +129,9 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -142,9 +142,9 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SDCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mgmt/test/snode/dsnode.cpp b/source/dnode/mgmt/test/snode/dsnode.cpp index 9ae0fbdc54..e3ad65d831 100644 --- a/source/dnode/mgmt/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/test/snode/dsnode.cpp @@ -30,9 +30,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -43,9 +43,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -56,9 +56,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -71,9 +71,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -87,9 +87,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -101,9 +101,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -114,9 +114,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -129,9 +129,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -142,9 +142,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/inc/mndBnode.h b/source/dnode/mnode/impl/inc/mndBnode.h deleted file mode 100644 index cefc0b1859..0000000000 --- a/source/dnode/mnode/impl/inc/mndBnode.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_MND_BNODE_H_ -#define _TD_MND_BNODE_H_ - -#include "mndInt.h" - -#ifdef __cplusplus -extern "C" { -#endif - -int32_t mndInitBnode(SMnode *pMnode); -void mndCleanupBnode(SMnode *pMnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_MND_BNODE_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9cc920de04..48eea7daea 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -43,8 +43,6 @@ typedef enum { MND_OPER_CREATE_USER, MND_OPER_DROP_USER, MND_OPER_ALTER_USER, - MND_OPER_CREATE_BNODE, - MND_OPER_DROP_BNODE, MND_OPER_CREATE_DNODE, MND_OPER_DROP_DNODE, MND_OPER_CONFIG_DNODE, @@ -226,13 +224,6 @@ typedef struct { SDnodeObj* pDnode; } SSnodeObj; -typedef struct { - int32_t id; - int64_t createdTime; - int64_t updateTime; - SDnodeObj* pDnode; -} SBnodeObj; - typedef struct { int32_t maxUsers; int32_t maxDbs; diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c deleted file mode 100644 index 06209d89c7..0000000000 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ /dev/null @@ -1,447 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "mndBnode.h" -#include "mndDnode.h" -#include "mndPrivilege.h" -#include "mndShow.h" -#include "mndTrans.h" -#include "mndUser.h" - -#define BNODE_VER_NUMBER 1 -#define BNODE_RESERVE_SIZE 64 - -static SSdbRaw *mndBnodeActionEncode(SBnodeObj *pObj); -static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw); -static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); -static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew); -static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); -static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq); -static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq); -static int32_t mndRetrieveBnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); -static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); - -int32_t mndInitBnode(SMnode *pMnode) { - SSdbTable table = { - .sdbType = SDB_BNODE, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndBnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndBnodeActionDecode, - .insertFp = (SdbInsertFp)mndBnodeActionInsert, - .updateFp = (SdbUpdateFp)mndBnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndBnodeActionDelete, - }; - - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_BNODE, mndProcessCreateBnodeReq); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_BNODE, mndProcessDropBnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_BNODE_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_BNODE_RSP, mndTransProcessRsp); - - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndRetrieveBnodes); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndCancelGetNextBnode); - - return sdbSetTable(pMnode->pSdb, table); -} - -void mndCleanupBnode(SMnode *pMnode) {} - -static SBnodeObj *mndAcquireBnode(SMnode *pMnode, int32_t bnodeId) { - SBnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_BNODE, &bnodeId); - if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { - terrno = TSDB_CODE_MND_BNODE_NOT_EXIST; - } - return pObj; -} - -static void mndReleaseBnode(SMnode *pMnode, SBnodeObj *pObj) { - SSdb *pSdb = pMnode->pSdb; - sdbRelease(pSdb, pObj); -} - -static SSdbRaw *mndBnodeActionEncode(SBnodeObj *pObj) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - - SSdbRaw *pRaw = sdbAllocRaw(SDB_BNODE, BNODE_VER_NUMBER, sizeof(SBnodeObj) + BNODE_RESERVE_SIZE); - if (pRaw == NULL) goto _OVER; - - int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, pObj->id, _OVER) - SDB_SET_INT64(pRaw, dataPos, pObj->createdTime, _OVER) - SDB_SET_INT64(pRaw, dataPos, pObj->updateTime, _OVER) - SDB_SET_RESERVE(pRaw, dataPos, BNODE_RESERVE_SIZE, _OVER) - - terrno = 0; - -_OVER: - if (terrno != 0) { - mError("bnode:%d, failed to encode to raw:%p since %s", pObj->id, pRaw, terrstr()); - sdbFreeRaw(pRaw); - return NULL; - } - - mTrace("bnode:%d, encode to raw:%p, row:%p", pObj->id, pRaw, pObj); - return pRaw; -} - -static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - - int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - - if (sver != BNODE_VER_NUMBER) { - terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto _OVER; - } - - SSdbRow *pRow = sdbAllocRow(sizeof(SBnodeObj)); - if (pRow == NULL) goto _OVER; - - SBnodeObj *pObj = sdbGetRowObj(pRow); - if (pObj == NULL) goto _OVER; - - int32_t dataPos = 0; - SDB_GET_INT32(pRaw, dataPos, &pObj->id, _OVER) - SDB_GET_INT64(pRaw, dataPos, &pObj->createdTime, _OVER) - SDB_GET_INT64(pRaw, dataPos, &pObj->updateTime, _OVER) - SDB_GET_RESERVE(pRaw, dataPos, BNODE_RESERVE_SIZE, _OVER) - - terrno = 0; - -_OVER: - if (terrno != 0) { - mError("bnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - taosMemoryFreeClear(pRow); - return NULL; - } - - mTrace("bnode:%d, decode from raw:%p, row:%p", pObj->id, pRaw, pObj); - return pRow; -} - -static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj) { - mTrace("bnode:%d, perform insert action, row:%p", pObj->id, pObj); - pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); - if (pObj->pDnode == NULL) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - mError("bnode:%d, failed to perform insert action since %s", pObj->id, terrstr()); - return -1; - } - - return 0; -} - -static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj) { - mTrace("bnode:%d, perform delete action, row:%p", pObj->id, pObj); - if (pObj->pDnode != NULL) { - sdbRelease(pSdb, pObj->pDnode); - pObj->pDnode = NULL; - } - - return 0; -} - -static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew) { - mTrace("bnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); - pOld->updateTime = pNew->updateTime; - return 0; -} - -static int32_t mndSetCreateBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { - SSdbRaw *pRedoRaw = mndBnodeActionEncode(pObj); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; - return 0; -} - -static int32_t mndSetCreateBnodeUndoLogs(STrans *pTrans, SBnodeObj *pObj) { - SSdbRaw *pUndoRaw = mndBnodeActionEncode(pObj); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; - return 0; -} - -static int32_t mndSetCreateBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { - SSdbRaw *pCommitRaw = mndBnodeActionEncode(pObj); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; - return 0; -} - -static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = pDnode->id; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void *pReq = taosMemoryMalloc(contLen); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - STransAction action = {0}; - action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_CREATE_BNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - - return 0; -} - -static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = pDnode->id; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void *pReq = taosMemoryMalloc(contLen); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - STransAction action = {0}; - action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_BNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - - return 0; -} - -static int32_t mndCreateBnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateBnodeReq *pCreate) { - int32_t code = -1; - - SBnodeObj bnodeObj = {0}; - bnodeObj.id = pDnode->id; - bnodeObj.createdTime = taosGetTimestampMs(); - bnodeObj.updateTime = bnodeObj.createdTime; - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-bnode"); - if (pTrans == NULL) goto _OVER; - - mInfo("trans:%d, used to create bnode:%d", pTrans->id, pCreate->dnodeId); - if (mndSetCreateBnodeRedoLogs(pTrans, &bnodeObj) != 0) goto _OVER; - if (mndSetCreateBnodeUndoLogs(pTrans, &bnodeObj) != 0) goto _OVER; - if (mndSetCreateBnodeCommitLogs(pTrans, &bnodeObj) != 0) goto _OVER; - if (mndSetCreateBnodeRedoActions(pTrans, pDnode, &bnodeObj) != 0) goto _OVER; - if (mndSetCreateBnodeUndoActions(pTrans, pDnode, &bnodeObj) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - - code = 0; - -_OVER: - mndTransDrop(pTrans); - return code; -} - -static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SBnodeObj *pObj = NULL; - SDnodeObj *pDnode = NULL; - SMCreateBnodeReq createReq = {0}; - - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - mInfo("bnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { - goto _OVER; - } - - pObj = mndAcquireBnode(pMnode, createReq.dnodeId); - if (pObj != NULL) { - terrno = TSDB_CODE_MND_BNODE_ALREADY_EXIST; - goto _OVER; - } else if (terrno != TSDB_CODE_MND_BNODE_NOT_EXIST) { - goto _OVER; - } - - pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); - if (pDnode == NULL) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - goto _OVER; - } - - code = mndCreateBnode(pMnode, pReq, pDnode, &createReq); - if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - -_OVER: - if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("bnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); - } - - mndReleaseBnode(pMnode, pObj); - mndReleaseDnode(pMnode, pDnode); - return code; -} - -static int32_t mndSetDropBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { - SSdbRaw *pRedoRaw = mndBnodeActionEncode(pObj); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; - return 0; -} - -static int32_t mndSetDropBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { - SSdbRaw *pCommitRaw = mndBnodeActionEncode(pObj); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; - return 0; -} - -static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = pDnode->id; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void *pReq = taosMemoryMalloc(contLen); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - STransAction action = {0}; - action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_BNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - - return 0; -} - -static int32_t mndDropBnode(SMnode *pMnode, SRpcMsg *pReq, SBnodeObj *pObj) { - int32_t code = -1; - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-bnode"); - if (pTrans == NULL) goto _OVER; - - mInfo("trans:%d, used to drop bnode:%d", pTrans->id, pObj->id); - if (mndSetDropBnodeRedoLogs(pTrans, pObj) != 0) goto _OVER; - if (mndSetDropBnodeCommitLogs(pTrans, pObj) != 0) goto _OVER; - if (mndSetDropBnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - - code = 0; - -_OVER: - mndTransDrop(pTrans); - return code; -} - -static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SBnodeObj *pObj = NULL; - SMDropBnodeReq dropReq = {0}; - - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - mInfo("bnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { - goto _OVER; - } - - if (dropReq.dnodeId <= 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - pObj = mndAcquireBnode(pMnode, dropReq.dnodeId); - if (pObj == NULL) { - goto _OVER; - } - - code = mndDropBnode(pMnode, pReq, pObj); - if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - -_OVER: - if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("bnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); - } - - mndReleaseBnode(pMnode, pObj); - return code; -} - -static int32_t mndRetrieveBnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - int32_t cols = 0; - SBnodeObj *pObj = NULL; - - while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_BNODE, pShow->pIter, (void **)&pObj); - if (pShow->pIter == NULL) break; - - cols = 0; - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); - - char buf[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(buf, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, buf, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); - - numOfRows++; - sdbRelease(pSdb, pObj); - } - - pShow->numOfRows += numOfRows; - - return numOfRows; -} - -static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter) { - SSdb *pSdb = pMnode->pSdb; - sdbCancelFetch(pSdb, pIter); -} diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index c1c7581a7e..7d0f5742f8 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -447,10 +447,6 @@ void dumpDnode(SSdb *pSdb, SJson *json) { } } -void dumpBnode(SSdb *pSdb, SJson *json) { - // not implemented yet -} - void dumpSnode(SSdb *pSdb, SJson *json) { void *pIter = NULL; SJson *items = tjsonAddArrayToObject(json, "snodes"); @@ -616,7 +612,6 @@ void mndDumpSdb() { dumpAuth(pSdb, json); dumpUser(pSdb, json); dumpDnode(pSdb, json); - dumpBnode(pSdb, json); dumpSnode(pSdb, json); dumpQnode(pSdb, json); dumpMnode(pSdb, json); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 8314601df3..4eb2e2978b 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -15,7 +15,6 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" -#include "mndBnode.h" #include "mndCluster.h" #include "mndConsumer.h" #include "mndDb.h" @@ -296,7 +295,6 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-bnode", mndInitBnode, mndCleanupBnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index f25d436aee..72ac7dd287 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -324,9 +324,9 @@ static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pA } static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) { - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, pDropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, pDropReq); void *pReq = taosMemoryMalloc(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, pDropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, pDropReq); STransAction action = { .epSet = *pDroprEpSet, @@ -410,7 +410,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SMCreateMnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -533,7 +533,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { SMnodeObj *pObj = NULL; SMDropMnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index d48a446e20..70f9b35312 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -190,13 +190,13 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S SDCreateQnodeReq createReq = {0}; createReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -217,13 +217,13 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -273,7 +273,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SMCreateQnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -330,13 +330,13 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -384,7 +384,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { SQnodeObj *pObj = NULL; SMDropQnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 5a998dfe98..8b4cacf8d6 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -60,8 +60,6 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_MODULE; } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, len) == 0) { type = TSDB_MGMT_TABLE_QNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_BNODES, len) == 0) { - type = TSDB_MGMT_TABLE_BNODE; } else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, len) == 0) { type = TSDB_MGMT_TABLE_SNODE; } else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, len) == 0) { diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 916e8d0c44..8127d5912e 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -195,13 +195,13 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S SDCreateSnodeReq createReq = {0}; createReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -222,13 +222,13 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -282,7 +282,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SMCreateSnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -341,13 +341,13 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -398,7 +398,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { SSnodeObj *pObj = NULL; SMDropSnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 3b1ca0999c..7db8485561 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -1,7 +1,6 @@ enable_testing() add_subdirectory(acct) -add_subdirectory(bnode) add_subdirectory(db) #add_subdirectory(dnode) add_subdirectory(func) diff --git a/source/dnode/mnode/impl/test/bnode/CMakeLists.txt b/source/dnode/mnode/impl/test/bnode/CMakeLists.txt deleted file mode 100644 index 2dd7b9ef78..0000000000 --- a/source/dnode/mnode/impl/test/bnode/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -# aux_source_directory(. MNODE_BNODE_TEST_SRC) -# add_executable(mbnodeTest ${MNODE_BNODE_TEST_SRC}) -# target_link_libraries( -# mbnodeTest -# PUBLIC sut -# ) - -# add_test( -# NAME mbnodeTest -# COMMAND mbnodeTest -# ) diff --git a/source/dnode/mnode/impl/test/bnode/mbnode.cpp b/source/dnode/mnode/impl/test/bnode/mbnode.cpp deleted file mode 100644 index c93e2142d0..0000000000 --- a/source/dnode/mnode/impl/test/bnode/mbnode.cpp +++ /dev/null @@ -1,293 +0,0 @@ -/** - * @file bnode.cpp - * @author slguan (slguan@taosdata.com) - * @brief MNODE module bnode tests - * @version 1.0 - * @date 2022-01-05 - * - * @copyright Copyright (c) 2022 - * - */ - -#include "sut.h" - -class MndTestBnode : public ::testing::Test { - public: - void SetUp() override {} - void TearDown() override {} - - public: - static void SetUpTestSuite() { - test.Init(TD_TMP_DIR_PATH "mnode_test_bnode1", 9018); - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9018"; - - server2.Start(TD_TMP_DIR_PATH "mnode_test_bnode2", 9019); - taosMsleep(300); - } - - static void TearDownTestSuite() { - server2.Stop(); - test.Cleanup(); - } - - static Testbase test; - static TestServer server2; -}; - -Testbase MndTestBnode::test; -TestServer MndTestBnode::server2; - -TEST_F(MndTestBnode, 01_Show_Bnode) { - test.SendShowReq(TSDB_MGMT_TABLE_BNODE, "bnodes", ""); - EXPECT_EQ(test.GetShowRows(), 0); -} - -TEST_F(MndTestBnode, 02_Create_Bnode) { - { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DNODE_NOT_EXIST); - } - - { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - - test.SendShowReq(TSDB_MGMT_TABLE_BNODE, "bnodes", ""); - EXPECT_EQ(test.GetShowRows(), 1); - } - - { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_BNODE_ALREADY_EXIST); - } -} - -TEST_F(MndTestBnode, 03_Drop_Bnode) { - { - SCreateDnodeReq createReq = {0}; - strcpy(createReq.fqdn, "localhost"); - createReq.port = 9019; - - int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDnodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - - taosMsleep(1300); - test.SendShowReq(TSDB_MGMT_TABLE_DNODE, "dnodes", ""); - EXPECT_EQ(test.GetShowRows(), 2); - } - - { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - - test.SendShowReq(TSDB_MGMT_TABLE_BNODE, "bnodes", ""); - EXPECT_EQ(test.GetShowRows(), 2); - } - - { - SMDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - - test.SendShowReq(TSDB_MGMT_TABLE_BNODE, "bnodes", ""); - EXPECT_EQ(test.GetShowRows(), 1); - } - - { - SMDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_BNODE_NOT_EXIST); - } -} - -TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { - { - // send message first, then dnode2 crash, result is returned, and rollback is started - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - server2.Stop(); - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); - } - - { - // continue send message, bnode is creating - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); - } - - { - // continue send message, bnode is creating - SMDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); - } - - { - // server start, wait until the rollback finished - server2.DoStart(); - taosMsleep(1000); - - int32_t retry = 0; - int32_t retryMax = 20; - - for (retry = 0; retry < retryMax; retry++) { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - if (pRsp->code == 0) break; - taosMsleep(1000); - } - - ASSERT_NE(retry, retryMax); - } -} - -TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { - { - // send message first, then dnode2 crash, result is returned, and rollback is started - SMDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - server2.Stop(); - SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); - } - - { - // continue send message, bnode is dropping - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); - } - - { - // continue send message, bnode is dropping - SMDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); - } - - { - // server start, wait until the rollback finished - server2.DoStart(); - taosMsleep(1000); - - int32_t retry = 0; - int32_t retryMax = 20; - - for (retry = 0; retry < retryMax; retry++) { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - if (pRsp->code == 0) break; - taosMsleep(1000); - } - - ASSERT_NE(retry, retryMax); - } -} diff --git a/source/dnode/mnode/impl/test/mnode/mnode.cpp b/source/dnode/mnode/impl/test/mnode/mnode.cpp index 1ed613c723..1f6dbd6dca 100644 --- a/source/dnode/mnode/impl/test/mnode/mnode.cpp +++ b/source/dnode/mnode/impl/test/mnode/mnode.cpp @@ -48,9 +48,9 @@ TEST_F(MndTestMnode, 02_Create_Mnode_Invalid_Id) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -63,9 +63,9 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Invalid_Id) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -98,9 +98,9 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -115,9 +115,9 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { SMDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -132,9 +132,9 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { SMDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -148,9 +148,9 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); @@ -163,9 +163,9 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -177,9 +177,9 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { SMDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -198,9 +198,9 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -218,9 +218,9 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { SMDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); @@ -233,9 +233,9 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -247,9 +247,9 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { SMDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -268,9 +268,9 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index 57b38e55c1..a3bcb7654e 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -48,9 +48,9 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -61,9 +61,9 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -77,9 +77,9 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -110,9 +110,9 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -126,9 +126,9 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { SMDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -142,9 +142,9 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { SMDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -158,9 +158,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); server2.Stop(); taosMsleep(1000); @@ -176,9 +176,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -190,9 +190,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -212,9 +212,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -231,9 +231,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); @@ -246,9 +246,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -259,9 +259,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -280,9 +280,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); if (pRsp->code == 0) break; diff --git a/source/dnode/mnode/impl/test/snode/snode.cpp b/source/dnode/mnode/impl/test/snode/snode.cpp index 1828fbd570..ac48eee4d0 100644 --- a/source/dnode/mnode/impl/test/snode/snode.cpp +++ b/source/dnode/mnode/impl/test/snode/snode.cpp @@ -48,9 +48,9 @@ TEST_F(MndTestSnode, 02_Create_Snode) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -61,9 +61,9 @@ TEST_F(MndTestSnode, 02_Create_Snode) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -77,9 +77,9 @@ TEST_F(MndTestSnode, 02_Create_Snode) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -110,9 +110,9 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -126,9 +126,9 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { SMDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -142,9 +142,9 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { SMDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -158,9 +158,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); @@ -173,9 +173,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -187,9 +187,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -208,9 +208,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -228,9 +228,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); @@ -243,9 +243,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -257,9 +257,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -278,9 +278,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/trans/trans1.cpp b/source/dnode/mnode/impl/test/trans/trans1.cpp index 5a470fc900..92a442aa5e 100644 --- a/source/dnode/mnode/impl/test/trans/trans1.cpp +++ b/source/dnode/mnode/impl/test/trans/trans1.cpp @@ -112,9 +112,9 @@ TEST_F(MndTestTrans1, 02_Create_Qnode1_Crash) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -129,9 +129,9 @@ TEST_F(MndTestTrans1, 02_Create_Qnode1_Crash) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -165,9 +165,9 @@ TEST_F(MndTestTrans1, 03_Create_Qnode2_Crash) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); @@ -209,9 +209,9 @@ TEST_F(MndTestTrans1, 03_Create_Qnode2_Crash) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -232,9 +232,9 @@ TEST_F(MndTestTrans1, 03_Create_Qnode2_Crash) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index a2373a55ce..d4db4709ca 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -132,7 +132,6 @@ typedef enum { SDB_MNODE = 2, SDB_QNODE = 3, SDB_SNODE = 4, - SDB_BNODE = 5, SDB_DNODE = 6, SDB_USER = 7, SDB_AUTH = 8, diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 85e937fcc3..6c00a654e9 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -30,8 +30,6 @@ const char *sdbTableName(ESdbType type) { return "qnode"; case SDB_SNODE: return "snode"; - case SDB_BNODE: - return "bnode"; case SDB_DNODE: return "dnode"; case SDB_USER: diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index eab5a33450..890a59e4be 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -401,7 +401,6 @@ static void monGenDnodeJson(SMonInfo *pMonitor) { tjsonAddDoubleToObject(pJson, "has_mnode", pInfo->has_mnode); tjsonAddDoubleToObject(pJson, "has_qnode", pInfo->has_qnode); tjsonAddDoubleToObject(pJson, "has_snode", pInfo->has_snode); - tjsonAddDoubleToObject(pJson, "has_bnode", pInfo->has_bnode); } static void monGenDiskJson(SMonInfo *pMonitor) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a8bc8689d0..5a05440c73 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5166,7 +5166,7 @@ static int16_t getCreateComponentNodeMsgType(ENodeType type) { static int32_t translateCreateComponentNode(STranslateContext* pCxt, SCreateComponentNodeStmt* pStmt) { SMCreateQnodeReq createReq = {.dnodeId = pStmt->dnodeId}; return buildCmdMsg(pCxt, getCreateComponentNodeMsgType(nodeType(pStmt)), - (FSerializeFunc)tSerializeSCreateDropMQSBNodeReq, &createReq); + (FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &createReq); } static int16_t getDropComponentNodeMsgType(ENodeType type) { @@ -5188,7 +5188,7 @@ static int16_t getDropComponentNodeMsgType(ENodeType type) { static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponentNodeStmt* pStmt) { SDDropQnodeReq dropReq = {.dnodeId = pStmt->dnodeId}; return buildCmdMsg(pCxt, getDropComponentNodeMsgType(nodeType(pStmt)), - (FSerializeFunc)tSerializeSCreateDropMQSBNodeReq, &dropReq); + (FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &dropReq); } static int32_t checkTopicQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 121bbaa733..3ea6ae3db4 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -27,25 +27,6 @@ TEST_F(ParserInitialCTest, createAccount) { run("CREATE ACCOUNT ac_wxy PASS '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT, PARSER_STAGE_PARSE); } -TEST_F(ParserInitialCTest, createBnode) { - useDb("root", "test"); - - SMCreateQnodeReq expect = {0}; - - auto setCreateQnodeReq = [&](int32_t dnodeId) { expect.dnodeId = dnodeId; }; - - setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { - ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_BNODE_STMT); - SMCreateQnodeReq req = {0}; - ASSERT_TRUE(TSDB_CODE_SUCCESS == - tDeserializeSCreateDropMQSBNodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); - ASSERT_EQ(req.dnodeId, expect.dnodeId); - }); - - setCreateQnodeReq(1); - run("CREATE BNODE ON DNODE 1"); -} - /* * CREATE DATABASE [IF NOT EXISTS] db_name [database_options] * diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp index 870afa694a..c7865f3da3 100644 --- a/source/libs/parser/test/parInitialDTest.cpp +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -52,12 +52,6 @@ TEST_F(ParserInitialDTest, describe) { // todo describe // todo DROP account -TEST_F(ParserInitialDTest, dropBnode) { - useDb("root", "test"); - - run("DROP BNODE ON DNODE 1"); -} - // DROP CONSUMER GROUP [ IF EXISTS ] cgroup_name ON topic_name TEST_F(ParserInitialDTest, dropConsumerGroup) { useDb("root", "test"); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9a117c6eb4..37f9734eab 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -238,8 +238,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_ALREADY_EXIST, "Qnode already exists" TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_NOT_EXIST, "Qnode not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_ALREADY_EXIST, "Snode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_NOT_EXIST, "Snode not there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_ALREADY_EXIST, "Bnode already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_NOT_EXIST, "Bnode not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_FEW_MNODES, "The replica of mnode cannot less than 1") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "The replica of mnode cannot exceed 3") diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index d8f4a36261..67cca39b33 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -187,9 +187,6 @@ # ---- snode ---- # unsupport ./test.sh -f tsim/snode/basic1.sim -# ---- bnode -./test.sh -f tsim/bnode/basic1.sim - # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim diff --git a/tests/script/tmp/monitor.sim b/tests/script/tmp/monitor.sim index b410e1b6ad..1734b9a1a7 100644 --- a/tests/script/tmp/monitor.sim +++ b/tests/script/tmp/monitor.sim @@ -23,5 +23,4 @@ sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 bin print =============== create drop qnode 1 sql create qnode on dnode 1 #sql create snode on dnode 1 -#sql create bnode on dnode 1 diff --git a/tests/script/tsim/bnode/basic1.sim b/tests/script/tsim/bnode/basic1.sim deleted file mode 100644 index 0a20001636..0000000000 --- a/tests/script/tsim/bnode/basic1.sim +++ /dev/null @@ -1,135 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/exec.sh -n dnode1 -s start -system sh/exec.sh -n dnode2 -s start -sql connect - -print =============== select * from information_schema.ins_dnodes -sql select * from information_schema.ins_dnodes; -if $rows != 1 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -sql select * from information_schema.ins_mnodes; -if $rows != 1 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -if $data02 != leader then - return -1 -endi - -print =============== create dnodes -sql create dnode $hostname port 7200 -sleep 2000 - -sql select * from information_schema.ins_dnodes; -if $rows != 2 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -if $data10 != 2 then - return -1 -endi - -print $data02 -if $data02 != 0 then - return -1 -endi - -if $data12 != 0 then - return -1 -endi - -if $data04 != ready then - return -1 -endi - -if $data14 != ready then - return -1 -endi - -sql select * from information_schema.ins_mnodes; -if $rows != 1 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -if $data02 != leader then - return -1 -endi - -#print =============== create drop bnode 1 -#sql create bnode on dnode 1 -#sql show bnodes -#if $rows != 1 then -# return -1 -#endi -#if $data00 != 1 then -# return -1 -#endi -#sql_error create bnode on dnode 1 -# -#sql drop bnode on dnode 1 -#sql show bnodes -#if $rows != 0 then -# return -1 -#endi -#sql_error drop bnode on dnode 1 -# -#print =============== create drop bnode 2 -#sql create bnode on dnode 2 -#sql show bnodes -#if $rows != 1 then -# return -1 -#endi -#if $data00 != 2 then -# return -1 -#endi -#sql_error create bnode on dnode 2 -# -#sql drop bnode on dnode 2 -#sql show bnodes -#if $rows != 0 then -# return -1 -#endi -#sql_error drop bnode on dnode 2 -# -#print =============== create drop bnodes -#sql create bnode on dnode 1 -#sql create bnode on dnode 2 -#sql show bnodes -#if $rows != 2 then -# return -1 -#endi - -#print =============== restart -#system sh/exec.sh -n dnode1 -s stop -x SIGINT -#system sh/exec.sh -n dnode2 -s stop -x SIGINT -#system sh/exec.sh -n dnode1 -s start -#system sh/exec.sh -n dnode2 -s start -# -#sleep 2000 -#sql show bnodes -#if $rows != 2 then -# return -1 -#endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index 89a4babb0b..ad8d70b089 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -128,5 +128,4 @@ run tsim/sync/threeReplica1VgElectWihtInsert.sim run tsim/sma/tsmaCreateInsertQuery.sim run tsim/sma/rsmaCreateInsertQuery.sim run tsim/valgrind/basic.sim -run tsim/valgrind/checkError.sim -run tsim/bnode/basic1.sim \ No newline at end of file +run tsim/valgrind/checkError.sim \ No newline at end of file diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 86f95755d0..1614c3be55 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -110,7 +110,6 @@ sql_error show dnodes sql_error show snodes sql_error show qnodes sql_error show mnodes -sql_error show bnodes sql_error show db.vgroups sql_error show db.stables sql_error show db.tables diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index c713e9fd14..1d733191b7 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -125,7 +125,7 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): dnode_infos = ['uptime', 'cpu_engine', 'cpu_system', 'cpu_cores', 'mem_engine', 'mem_system', 'mem_total', 'disk_engine', 'disk_used', 'disk_total', 'net_in', 'net_out', 'io_read', 'io_write', 'io_read_disk', 'io_write_disk', 'req_select', 'req_select_rate', 'req_insert', 'req_insert_success', 'req_insert_rate', 'req_insert_batch', 'req_insert_batch_success', - 'req_insert_batch_rate', 'errors', 'vnodes_num', 'masters', 'has_mnode', 'has_qnode', 'has_snode', 'has_bnode'] + 'req_insert_batch_rate', 'errors', 'vnodes_num', 'masters', 'has_mnode', 'has_qnode', 'has_snode'] for elem in dnode_infos: if elem not in infoDict["dnode_info"] or infoDict["dnode_info"][elem] < 0: tdLog.exit(f"{elem} is null!") From aa906a0d3b46f56cf2f6ea9aef13c1b9c4272a8d Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 18 Oct 2022 17:39:58 +0800 Subject: [PATCH 118/135] test: modify timestamp for insert success anywhen --- tests/script/tsim/parser/like.sim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/parser/like.sim b/tests/script/tsim/parser/like.sim index 40dcec4080..5cac026b57 100644 --- a/tests/script/tsim/parser/like.sim +++ b/tests/script/tsim/parser/like.sim @@ -17,9 +17,9 @@ sql create table $table1 (ts timestamp, b binary(20)) sql create table $table2 (ts timestamp, b binary(20)) sql insert into $table1 values(now, "table_name") -sql insert into $table1 values(now-1m, "tablexname") +sql insert into $table1 values(now-3m, "tablexname") sql insert into $table1 values(now-2m, "tablexxx") -sql insert into $table1 values(now-2m, "table") +sql insert into $table1 values(now-1m, "table") sql select b from $table1 if $rows != 4 then From 0f5e02b0b40aef533fc08fb95d2ee9b1d9e40ca8 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 18 Oct 2022 17:52:40 +0800 Subject: [PATCH 119/135] fix(query): mode function output one random result instead of NULL if there're multiple highest frequency of occurrence. TD-19619 --- source/libs/function/src/builtinsimpl.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 7077a9b780..6a428af1ed 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5356,16 +5356,14 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t maxCount = 0; for (int32_t i = 0; i < pInfo->numOfPoints; ++i) { SModeItem* pItem = (SModeItem*)(pInfo->pItems + i * (sizeof(SModeItem) + pInfo->colBytes)); - if (pItem->count > maxCount) { + if (pItem->count >= maxCount) { maxCount = pItem->count; resIndex = i; - } else if (pItem->count == maxCount) { - resIndex = -1; } } SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); - colDataAppend(pCol, currentRow, pResItem->data, (resIndex == -1) ? true : false); + colDataAppend(pCol, currentRow, pResItem->data, false); return pResInfo->numOfRes; } From 25c2cec6028effa60312433b6af1b7da766362af Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 18 Oct 2022 17:58:17 +0800 Subject: [PATCH 120/135] change mode en docs --- docs/en/12-taos-sql/10-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 2b39c4b9e5..48bc5e743e 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -945,7 +945,7 @@ MIN(expr) MODE(expr) ``` -**Description**:The value which has the highest frequency of occurrence. NULL is returned if there are multiple values which have highest frequency of occurrence. +**Description**:The value which has the highest frequency of occurrence. One random value is returned if there are multiple values which have highest frequency of occurrence. **Return value type**: Same as the input data From 3ec1490fe2050e769f496c05347558d75e1ac3cc Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 18 Oct 2022 18:00:53 +0800 Subject: [PATCH 121/135] change mode function zh docs --- docs/zh/12-taos-sql/10-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 9346c3f763..e7ed74631d 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -946,7 +946,7 @@ MIN(expr) MODE(expr) ``` -**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，输出NULL。 +**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，则随机输出其中某个值。 **返回数据类型**：与输入数据类型一致。 From 797ed80f707c5ab3558176858e9167c63e45885f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Oct 2022 18:08:49 +0800 Subject: [PATCH 122/135] enh: comment out unused codes in tsim --- utils/tsim/src/simExe.c | 8 ++++++++ utils/tsim/src/simParse.c | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index 4fe5fdf672..6f693cf015 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -37,6 +37,7 @@ void simLogSql(char *sql, bool useSharp) { taosFsyncFile(pFile); } +#if 0 char *simParseArbitratorName(char *varName) { static char hostName[140]; #ifdef WINDOWS @@ -47,6 +48,7 @@ char *simParseArbitratorName(char *varName) { #endif return hostName; } +#endif char *simParseHostName(char *varName) { static char hostName[140]; @@ -102,9 +104,11 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { return simParseHostName(varName); } +#if 0 if (strncmp(varName, "arbitrator", 10) == 0) { return simParseArbitratorName(varName); } +#endif if (strncmp(varName, "error", varLen) == 0) return script->error; @@ -883,6 +887,7 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) { return simExecuteSqlImpCmd(script, rest, isSlow); } +#if 0 bool simExecuteRestfulCmd(SScript *script, char *rest) { TdFilePtr pFile = NULL; char filename[256]; @@ -924,6 +929,7 @@ bool simExecuteRestfulCmd(SScript *script, char *rest) { return simExecuteSystemCmd(script, cmd); } +#endif bool simExecuteSqlErrorCmd(SScript *script, char *rest) { char buf[3000]; @@ -981,6 +987,7 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { return false; } +#if 0 bool simExecuteLineInsertCmd(SScript *script, char *rest) { char buf[TSDB_MAX_BINARY_LEN] = {0}; @@ -1037,3 +1044,4 @@ bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { return true; } } +#endif \ No newline at end of file diff --git a/utils/tsim/src/simParse.c b/utils/tsim/src/simParse.c index 14be10bc8a..ae523edfc4 100644 --- a/utils/tsim/src/simParse.c +++ b/utils/tsim/src/simParse.c @@ -501,6 +501,7 @@ bool simParseEndwCmd(char *rest, SCommand *pCmd, int32_t lineNum) { return true; } +#if 0 bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) { char *token; int32_t tokenLen; @@ -647,6 +648,7 @@ bool simParseContinueCmd(char *rest, SCommand *pCmd, int32_t lineNum) { numOfLines++; return true; } +#endif bool simParsePrintCmd(char *rest, SCommand *pCmd, int32_t lineNum) { int32_t expLen; @@ -715,6 +717,7 @@ bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { return true; } +#if 0 bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int32_t lineNum) { simParseSqlCmd(rest, pCmd, lineNum); cmdLine[numOfLines - 1].cmdno = SIM_CMD_SQL_SLOW; @@ -726,6 +729,7 @@ bool simParseRestfulCmd(char *rest, SCommand *pCmd, int32_t lineNum) { cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; return true; } +#endif bool simParseSystemCmd(char *rest, SCommand *pCmd, int32_t lineNum) { int32_t expLen; @@ -838,6 +842,7 @@ bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) { return true; } +#if 0 bool simParseLineInsertCmd(char *rest, SCommand *pCmd, int32_t lineNum) { int32_t expLen; @@ -869,6 +874,7 @@ bool simParseLineInsertErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { numOfLines++; return true; } +#endif void simInitsimCmdList() { int32_t cmdno; @@ -930,6 +936,7 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); +#if 0 cmdno = SIM_CMD_SWITCH; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "switch"); @@ -977,6 +984,7 @@ void simInitsimCmdList() { simCmdList[cmdno].parseCmd = simParseEndsCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); +#endif cmdno = SIM_CMD_SLEEP; simCmdList[cmdno].cmdno = cmdno; @@ -1050,6 +1058,7 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = simExecuteSqlErrorCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); +#if 0 cmdno = SIM_CMD_SQL_SLOW; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "sql_slow"); @@ -1065,6 +1074,7 @@ void simInitsimCmdList() { simCmdList[cmdno].parseCmd = simParseRestfulCmd; simCmdList[cmdno].executeCmd = simExecuteRestfulCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); +#endif /* test is only an internal command */ cmdno = SIM_CMD_TEST; @@ -1082,6 +1092,7 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = simExecuteReturnCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); +#if 0 cmdno = SIM_CMD_LINE_INSERT; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "line_insert"); @@ -1097,4 +1108,5 @@ void simInitsimCmdList() { simCmdList[cmdno].parseCmd = simParseLineInsertErrorCmd; simCmdList[cmdno].executeCmd = simExecuteLineInsertErrorCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); +#endif } From abbfa0b8e57c923ba95411ce1773d8695f635d23 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 18 Oct 2022 18:13:00 +0800 Subject: [PATCH 123/135] docs: update mac docs --- docs/en/14-reference/03-connector/_macos_install.mdx | 2 +- docs/zh/08-connector/_macos_install.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-connector/_macos_install.mdx b/docs/en/14-reference/03-connector/_macos_install.mdx index 8813d65869..effabbbebe 100644 --- a/docs/en/14-reference/03-connector/_macos_install.mdx +++ b/docs/en/14-reference/03-connector/_macos_install.mdx @@ -2,7 +2,7 @@ import PkgListV3 from "/components/PkgListV3"; 1. Download the client installation package - + [All Downloads](../../releases/tdengine) diff --git a/docs/zh/08-connector/_macos_install.mdx b/docs/zh/08-connector/_macos_install.mdx index 4925ffabcd..081134dfee 100644 --- a/docs/zh/08-connector/_macos_install.mdx +++ b/docs/zh/08-connector/_macos_install.mdx @@ -2,7 +2,7 @@ import PkgListV3 from "/components/PkgListV3"; 1. 下载客户端安装包 - + [所有下载](../../releases/tdengine) From 8efd7e8763727d77d76c154fc9d66ad38b87d856 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 18 Oct 2022 17:44:00 +0800 Subject: [PATCH 124/135] feat(stream):replace disk buff --- include/common/tcommon.h | 11 +- include/libs/stream/streamState.h | 16 + source/common/src/tdatablock.c | 7 +- source/libs/executor/inc/executorimpl.h | 91 +- source/libs/executor/src/executorimpl.c | 116 +- source/libs/executor/src/scanoperator.c | 19 +- source/libs/executor/src/tfill.c | 4 +- source/libs/executor/src/timewindowoperator.c | 1564 +++++++---------- source/libs/function/src/builtinsimpl.c | 11 +- source/libs/stream/src/streamState.c | 297 +++- tests/script/jenkins/basic.txt | 8 +- tests/script/tsim/stream/session0.sim | 4 +- tests/script/tsim/stream/session1.sim | 6 +- tests/script/tsim/stream/state0.sim | 2 +- tests/script/tsim/stream/triggerSession0.sim | 9 +- 15 files changed, 1074 insertions(+), 1091 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index eaddf4e983..bb2519bea3 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -44,12 +44,17 @@ enum { ) // clang-format on -typedef struct { +typedef struct SWinKey { uint64_t groupId; TSKEY ts; } SWinKey; -static inline int sWinKeyCmprImpl(const void* pKey1, const void* pKey2) { +typedef struct SSessionKey { + STimeWindow win; + uint64_t groupId; +} SSessionKey; + +static inline int winKeyCmprImpl(const void* pKey1, const void* pKey2) { SWinKey* pWin1 = (SWinKey*)pKey1; SWinKey* pWin2 = (SWinKey*)pKey2; @@ -69,7 +74,7 @@ static inline int sWinKeyCmprImpl(const void* pKey1, const void* pKey2) { } static inline int winKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { - return sWinKeyCmprImpl(pKey1, pKey2); + return winKeyCmprImpl(pKey1, pKey2); } typedef struct { diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 9445a0baa5..ec68e8ed71 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -25,6 +25,8 @@ extern "C" { typedef struct SStreamTask SStreamTask; +typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2); + // incremental state storage typedef struct { SStreamTask* pOwner; @@ -32,6 +34,7 @@ typedef struct { TTB* pStateDb; TTB* pFuncStateDb; TTB* pFillStateDb; // todo refactor + TTB* pSessionStateDb; TXN txn; int32_t number; } SStreamState; @@ -57,6 +60,19 @@ int32_t streamStateDel(SStreamState* pState, const SWinKey* key); int32_t streamStateClear(SStreamState* pState); void streamStateSetNumber(SStreamState* pState, int32_t number); +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen); +int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key); +int32_t streamStateSessionClear(SStreamState* pState); +int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, const void** pVal, int32_t* pVLen); +int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + +SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key); +SStreamStateCur* streamStateSessionSeekKeyPrev(SStreamState* pState, const SSessionKey* key); +SStreamStateCur* streamStateSessionGetCur(SStreamState* pState, const SSessionKey* key); + int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 1028a899b6..62706db980 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1892,12 +1892,13 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) for (int32_t k = 0; k < colNum; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); if (colDataIsNull(pColInfoData, rows, j, NULL) || !pColInfoData->pData) { len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL"); if (len >= size - 1) return dumpBuf; continue; } + + void* var = colDataGetData(pColInfoData, j); switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: memset(pBuf, 0, sizeof(pBuf)); @@ -1926,8 +1927,8 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) if (len >= size - 1) return dumpBuf; break; case TSDB_DATA_TYPE_DOUBLE: - len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var); - if (len >= size - 1) return dumpBuf; + // len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var); + // if (len >= size - 1) return dumpBuf; break; case TSDB_DATA_TYPE_BOOL: len += snprintf(dumpBuf + len, size - len, " %15d |", *(bool*)var); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index fc996a6003..47450328be 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -53,6 +53,11 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData ? 1 : 0) +#define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0) +#define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN) +#define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0) +#define SET_SESSION_WIN_KEY_INVALID(pWinKey) ((pWinKey)->win.skey = INT64_MIN) + enum { // when this task starts to execute, this status will set TASK_NOT_COMPLETED = 0x1u, @@ -434,15 +439,15 @@ typedef struct SCatchSupporter { } SCatchSupporter; typedef struct SStreamAggSupporter { - SHashObj* pResultRows; - SArray* pCurWins; - int32_t valueSize; - int32_t keySize; - char* pKeyBuf; // window key buffer - SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - int32_t currentPageId; // buffer page that is active - SSDataBlock* pScanBlock; + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + SSDataBlock* pScanBlock; + SStreamState* pState; + int64_t gap; // stream session window gap + SqlFunctionCtx* pDummyCtx; // for combine + SSHashObj* pResultRows; + int32_t stateKeySize; + int16_t stateKeyType; + SDiskbasedBuf* pResultBuf; } SStreamAggSupporter; typedef struct SWindowSupporter { @@ -736,42 +741,54 @@ typedef struct SSessionAggOperatorInfo { } SSessionAggOperatorInfo; typedef struct SResultWindowInfo { - SResultRowPosition pos; - STimeWindow win; - uint64_t groupId; + void* pOutputBuf; + SSessionKey sessionWin; bool isOutput; - bool isClosed; } SResultWindowInfo; typedef struct SStateWindowInfo { SResultWindowInfo winInfo; - SStateKeys stateKey; + SStateKeys* pStateKey; } SStateWindowInfo; typedef struct SStreamSessionAggOperatorInfo { SOptrBasicInfo binfo; SStreamAggSupporter streamAggSup; - SExprSupp scalarSupp; // supporter for perform scalar function + SExprSupp scalarSupp; // supporter for perform scalar function SGroupResInfo groupResInfo; - int64_t gap; // session window gap int32_t primaryTsIndex; // primary timestamp slot id int32_t endTsIndex; // window end timestamp slot id int32_t order; // current SSDataBlock scan order STimeWindowAggSupp twAggSup; - SSDataBlock* pWinBlock; // window result - SqlFunctionCtx* pDummyCtx; // for combine - SSDataBlock* pDelRes; // delete result - SSDataBlock* pUpdateRes; // update window + SSDataBlock* pWinBlock; // window result + SSDataBlock* pDelRes; // delete result + SSDataBlock* pUpdateRes; // update window bool returnUpdate; - SHashObj* pStDeleted; + SSHashObj* pStDeleted; void* pDelIterator; - SArray* pChildren; // cache for children's result; final stream operator - SPhysiNode* pPhyNode; // create new child + SArray* pChildren; // cache for children's result; final stream operator + SPhysiNode* pPhyNode; // create new child bool isFinal; bool ignoreExpiredData; SHashObj* pGroupIdTbNameMap; } SStreamSessionAggOperatorInfo; +typedef struct SStreamStateAggOperatorInfo { + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; + int32_t primaryTsIndex; // primary timestamp slot id + STimeWindowAggSupp twAggSup; + SColumn stateCol; + SSDataBlock* pDelRes; + SSHashObj* pSeDeleted; + void* pDelIterator; + SArray* pChildren; // cache for children's result; + bool ignoreExpiredData; + SHashObj* pGroupIdTbNameMap; +} SStreamStateAggOperatorInfo; + typedef struct SStreamPartitionOperatorInfo { SOptrBasicInfo binfo; SPartitionBySupporter partitionSup; @@ -834,24 +851,6 @@ typedef struct SStateWindowOperatorInfo { const SNode* pCondition; } SStateWindowOperatorInfo; -typedef struct SStreamStateAggOperatorInfo { - SOptrBasicInfo binfo; - SStreamAggSupporter streamAggSup; - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; - int32_t primaryTsIndex; // primary timestamp slot id - int32_t order; // current SSDataBlock scan order - STimeWindowAggSupp twAggSup; - SColumn stateCol; - SqlFunctionCtx* pDummyCtx; // for combine - SSDataBlock* pDelRes; - SHashObj* pSeDeleted; - void* pDelIterator; - SArray* pChildren; // cache for children's result; - bool ignoreExpiredData; - SHashObj* pGroupIdTbNameMap; -} SStreamStateAggOperatorInfo; - typedef struct SSortOperatorInfo { SOptrBasicInfo binfo; uint32_t sortBufSize; // max buffer size for in-memory sort @@ -1064,13 +1063,8 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t size); SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize); -SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - int64_t gap, int32_t* pIndex); -SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - int64_t gap, int32_t* pIndex); +void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, SSessionKey* pKey); bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); @@ -1100,6 +1094,9 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); +int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size); +int32_t buildSessionResultDataBlock(SExecTaskInfo* pTaskInfo, SStreamState* pState, SSDataBlock* pBlock, + SExprSupp* pSup, SGroupResInfo* pGroupResInfo); int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a7e98527f4..641bcb0c5b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4192,42 +4192,6 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInf return TSDB_CODE_SUCCESS; } -int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t size) { - pSup->currentPageId = -1; - pSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); - pSup->keySize = sizeof(int64_t) + sizeof(TSKEY); - pSup->pKeyBuf = taosMemoryCalloc(1, pSup->keySize); - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pSup->pResultRows = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); - if (pSup->pKeyBuf == NULL || pSup->pResultRows == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pSup->valueSize = size; - - pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR); - int32_t pageSize = 4096; - while (pageSize < pSup->resultRowSize * 4) { - pageSize <<= 1u; - } - // at least four pages need to be in buffer - int32_t bufSize = 4096 * 256; - if (bufSize <= pageSize) { - bufSize = pageSize * 4; - } - if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; - qError("Init stream agg supporter failed since %s", terrstr(terrno)); - return terrno; - } - int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, pKey, tsTempDir); - for (int32_t i = 0; i < numOfOutput; ++i) { - pCtx[i].saveHandle.pBuf = pSup->pResultBuf; - } - - return code; -} - int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup) { SWinKey key = { @@ -4237,7 +4201,6 @@ int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResul char* value = NULL; int32_t size = pAggSup->resultRowSize; - tSimpleHashPut(pAggSup->pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0); if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -4342,3 +4305,82 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; } + +int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size) { + streamStateSessionPut(pState, key, (const void*)buf, size); + releaseOutputBuf(pState, NULL, (SResultRow*)buf); + return TSDB_CODE_SUCCESS; +} + +int32_t buildSessionResultDataBlock(SExecTaskInfo* pTaskInfo, SStreamState* pState, SSDataBlock* pBlock, + SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { + SExprInfo* pExprInfo = pSup->pExprInfo; + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; + + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + + for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { + SSessionKey* pKey = taosArrayGet(pGroupResInfo->pRows, i); + int32_t size = 0; + void* pVal = NULL; + int32_t code = streamStateSessionGet(pState, pKey, &pVal, &size); + ASSERT(code == 0); + SResultRow* pRow = (SResultRow*)pVal; + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); + // no results, continue to check the next one + if (pRow->numOfRows == 0) { + pGroupResInfo->index += 1; + releaseOutputBuf(pState, NULL, pRow); + continue; + } + + if (pBlock->info.groupId == 0) { + pBlock->info.groupId = pKey->groupId; + } else { + // current value belongs to different group, it can't be packed into one datablock + if (pBlock->info.groupId != pKey->groupId) { + releaseOutputBuf(pState, NULL, pRow); + break; + } + } + + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); + releaseOutputBuf(pState, NULL, pRow); + break; + } + + pGroupResInfo->index += 1; + + for (int32_t j = 0; j < numOfExprs; ++j) { + int32_t slotId = pExprInfo[j].base.resSchema.slotId; + + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); + if (pCtx[j].fpSet.finalize) { + int32_t code1 = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (TAOS_FAILED(code1)) { + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code1)); + T_LONG_JMP(pTaskInfo->env, code1); + } + } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { + // do nothing, todo refactor + } else { + // expand the result into multiple rows. E.g., _wstart, top(k, 20) + // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } + } + } + + pBlock->info.rows += pRow->numOfRows; + // saveSessionDiscBuf(pState, pKey, pVal, size); + releaseOutputBuf(pState, NULL, pRow); + } + blockDataUpdateTsWindow(pBlock, 0); + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8e1858d8c0..7385e9bb92 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1190,23 +1190,22 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - int32_t dummy = 0; int64_t version = pSrcBlock->info.version - 1; for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); // gap must be 0. - SResultWindowInfo* pStartWin = - getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy); - if (!pStartWin) { + SSessionKey startWin = {0}; + getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, &startWin); + if (IS_INVALID_SESSION_WIN_KEY(startWin)) { // window has been closed. continue; } - SResultWindowInfo* pEndWin = - getCurSessionWindow(pInfo->windowSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy); - ASSERT(pEndWin); - TSKEY ts = INT64_MIN; - colDataAppend(pDestStartCol, i, (const char*)&pStartWin->win.skey, false); - colDataAppend(pDestEndCol, i, (const char*)&pEndWin->win.ekey, false); + SSessionKey endWin = {0}; + getCurSessionWindow(pInfo->windowSup.pStreamAggSup, endData[i], endData[i], groupId, &endWin); + ASSERT(!IS_INVALID_SESSION_WIN_KEY(endWin)); + colDataAppend(pDestStartCol, i, (const char*)&startWin.win.skey, false); + colDataAppend(pDestEndCol, i, (const char*)&endWin.win.ekey, false); + colDataAppendNULL(pDestUidCol, i); colDataAppend(pDestGpCol, i, (const char*)&groupId, false); colDataAppendNULL(pDestCalStartTsCol, i); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 8b716531e5..e199cf8406 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -693,7 +693,7 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); tSimpleHashCleanup(pFillSup->pResMap); pFillSup->pResMap = NULL; - streamStateReleaseBuf(NULL, NULL, pFillSup->cur.pRowVal); + releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); pFillSup->cur.pRowVal = NULL; taosMemoryFree(pFillSup); @@ -736,7 +736,7 @@ static void resetFillWindow(SResultRowData* pRowData) { void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) { resetFillWindow(&pFillSup->prev); - streamStateReleaseBuf(NULL, NULL, pFillSup->cur.pRowVal); + releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); resetFillWindow(&pFillSup->cur); resetFillWindow(&pFillSup->next); resetFillWindow(&pFillSup->nextNext); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 1f2b394b9c..76fddc3982 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -38,8 +38,6 @@ typedef struct SOpenWindowInfo { uint64_t groupId; } SOpenWindowInfo; -static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator); - static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo); static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, @@ -828,31 +826,9 @@ int32_t compareResKey(void* pKey, void* data, int32_t index) { return -1; } -static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SArray* pUpdated) { - int32_t size = taosArrayGetSize(pUpdated); - SWinKey data = {.ts = ts, .groupId = groupId}; - int32_t index = binarySearchCom(pUpdated, size, &data, TSDB_ORDER_DESC, compareResKey); - if (index == -1) { - index = 0; - } else { - if (compareResKey(&data, pUpdated, index) > 0) { - index++; - } else { - return TSDB_CODE_SUCCESS; - } - } - - SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - if (newPos == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - newPos->groupId = groupId; - newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset}; - *(int64_t*)newPos->key = ts; - if (taosArrayInsert(pUpdated, index, &newPos) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - return TSDB_CODE_SUCCESS; +static int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) { + winInfo.sessionWin.win.ekey = winInfo.sessionWin.win.skey; + return tSimpleHashPut(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); } static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SHashObj* pUpdatedMap) { @@ -874,10 +850,6 @@ static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SHashObj* pUpdatedM return saveWinResult(ts, -1, -1, groupId, pUpdatedMap); } -static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) { - return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated); -} - static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) { int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { @@ -915,8 +887,8 @@ int32_t compareWinRes(void* pKey, void* data, int32_t index) { } static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { - taosArraySort(pDelWins, sWinKeyCmprImpl); - taosArrayRemoveDuplicate(pDelWins, sWinKeyCmprImpl, NULL); + taosArraySort(pDelWins, winKeyCmprImpl); + taosArrayRemoveDuplicate(pDelWins, winKeyCmprImpl, NULL); int32_t delSize = taosArrayGetSize(pDelWins); if (taosHashGetSize(pUpdatedMap) == 0 || delSize == 0) { return; @@ -1385,19 +1357,6 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, releaseBufPage(pResultBuf, bufPage); } -static bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, - int32_t numOfOutput) { - SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId); - SResultRowPosition* p1 = - (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - if (!p1) { - // window has been closed - return false; - } - doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput); - return true; -} - static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SWinKey key = {.ts = ts, .groupId = groupId}; @@ -1455,73 +1414,6 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa } } -bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) { - size_t bytes = sizeof(TSKEY); - SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId); - SResultRowPosition* p1 = - (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - if (!p1) { - // window has been closed - return false; - } - tSimpleHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - return true; -} - -static void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, STimeWindowAggSupp* pTwSup, SSDataBlock* pBlock, - SArray* pDelWins, SInterval* pInterval, SHashObj* pUpdatedMap) { - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* tsStarts = (TSKEY*)pStartCol->pData; - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - TSKEY* tsEnds = (TSKEY*)pEndCol->pData; - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* groupIds = (uint64_t*)pGroupCol->pData; - int64_t numOfWin = tSimpleHashGetSize(pAggSup->pResultRowHashTable); - for (int32_t i = 0; i < pBlock->info.rows; i++) { - TSKEY startTs = TMAX(tsStarts[i], pTwSup->minTs); - TSKEY endTs = TMIN(tsEnds[i], pTwSup->maxTs); - SResultRowInfo dumyInfo = {0}; - dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTs, pInterval, TSDB_ORDER_ASC); - do { - doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]); - SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]}; - if (pDelWins) { - taosArrayPush(pDelWins, &winRes); - } - if (pUpdatedMap) { - taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); - } - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); - } while (win.skey <= endTs); - } -} - -static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput, - SSDataBlock* pBlock, SArray* pUpWins) { - SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* startTsCols = (TSKEY*)pStartTsCol->pData; - SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - TSKEY* endTsCols = (TSKEY*)pEndTsCol->pData; - SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* pGpDatas = (uint64_t*)pGpCol->pData; - for (int32_t i = 0; i < pBlock->info.rows; i++) { - SResultRowInfo dumyInfo = {0}; - dumyInfo.cur.pageId = -1; - - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); - while (win.ekey <= endTsCols[i]) { - uint64_t winGpId = pGpDatas[i]; - bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput); - if (pUpWins && res) { - SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; - taosArrayPush(pUpWins, &winRes); - } - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); - } - } -} - static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { void* pIte = NULL; size_t keyLen = 0; @@ -2923,11 +2815,12 @@ bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { return TSDB_CODE_SUCCESS == streamStateGet(pState, pKey, NULL, 0); } -static void rebuildIntervalWindow(SOperatorInfo* pOperator, SExprSupp* pSup, SArray* pWinArray, SHashObj* pUpdatedMap) { +static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, SHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t size = taosArrayGetSize(pWinArray); int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + SExprSupp* pSup = &pOperator->exprSupp; if (!pInfo->pChildren) { return; } @@ -2935,8 +2828,7 @@ static void rebuildIntervalWindow(SOperatorInfo* pOperator, SExprSupp* pSup, SAr SWinKey* pWinRes = taosArrayGet(pWinArray, i); SResultRow* pCurResult = NULL; STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval); - if (isDeletedStreamWindow(&parentWin, pWinRes->groupId, pInfo->pState, &pInfo->twAggSup) && - isCloseWindow(&parentWin, &pInfo->twAggSup)) { + if (isDeletedStreamWindow(&parentWin, pWinRes->groupId, pInfo->pState, &pInfo->twAggSup)) { continue; } @@ -3223,6 +3115,14 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { saveWinResultInfo(pResult->win.skey, groupId, pUpdatedMap); } + + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + SWinKey key = { + .ts = pResult->win.skey, + .groupId = groupId, + }; + tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0); + } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pSDataBlock->info.rows, numOfOutput); @@ -3338,7 +3238,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; doDeleteWindows(pChildOp, &pChildInfo->interval, pBlock, NULL, NULL); - rebuildIntervalWindow(pOperator, pSup, delWins, pUpdatedMap); + rebuildIntervalWindow(pOperator, delWins, pUpdatedMap); addRetriveWindow(delWins, pInfo); taosArrayAddAll(pInfo->pDelWins, delWins); taosArrayDestroy(delWins); @@ -3564,42 +3464,18 @@ _error: } void destroyStreamAggSupporter(SStreamAggSupporter* pSup) { - taosMemoryFreeClear(pSup->pKeyBuf); - void** pIte = NULL; - while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) { - SArray* pWins = (SArray*)(*pIte); - taosArrayDestroy(pWins); - } - taosHashCleanup(pSup->pResultRows); - destroyDiskbasedBuf(pSup->pResultBuf); - blockDataDestroy(pSup->pScanBlock); -} - -void destroyStateWinInfo(void* ptr) { - if (ptr == NULL) { - return; - } - SStateWindowInfo* pWin = (SStateWindowInfo*)ptr; - taosMemoryFreeClear(pWin->stateKey.pData); -} - -void destroyStateStreamAggSupporter(SStreamAggSupporter* pSup) { - taosMemoryFreeClear(pSup->pKeyBuf); - void** pIte = NULL; - while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) { - SArray* pWins = (SArray*)(*pIte); - taosArrayDestroyEx(pWins, (FDelete)destroyStateWinInfo); - } - taosHashCleanup(pSup->pResultRows); + tSimpleHashCleanup(pSup->pResultRows); destroyDiskbasedBuf(pSup->pResultBuf); blockDataDestroy(pSup->pScanBlock); + taosMemoryFreeClear(pSup->pState); + taosMemoryFreeClear(pSup->pDummyCtx); } void destroyStreamSessionAggOperatorInfo(void* param) { SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); destroyStreamAggSupporter(&pInfo->streamAggSup); - cleanupGroupResInfo(&pInfo->groupResInfo); + if (pInfo->pChildren != NULL) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { @@ -3613,8 +3489,7 @@ void destroyStreamSessionAggOperatorInfo(void* param) { blockDataDestroy(pInfo->pDelRes); blockDataDestroy(pInfo->pWinBlock); blockDataDestroy(pInfo->pUpdateRes); - destroySqlFunctionCtx(pInfo->pDummyCtx, 0); - taosHashCleanup(pInfo->pStDeleted); + tSimpleHashCleanup(pInfo->pStDeleted); taosHashCleanup(pInfo->pGroupIdTbNameMap); taosMemoryFreeClear(param); @@ -3643,129 +3518,62 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num } } -void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark, - uint16_t type, int32_t tsColIndex) { +void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t waterMark, uint16_t type, + int32_t tsColIndex) { if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) { SStreamPartitionOperatorInfo* pScanInfo = downstream->info; pScanInfo->tsColIndex = tsColIndex; } if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - initDownStream(downstream->pDownstream[0], pAggSup, gap, waterMark, type, tsColIndex); + initDownStream(downstream->pDownstream[0], pAggSup, waterMark, type, tsColIndex); return; } SStreamScanInfo* pScanInfo = downstream->info; - pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type}; + pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark); } -int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, - int32_t numOfOutput) { - return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo)); -} - -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo) { - SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; - int32_t numOfCols = 0; - int32_t code = TSDB_CODE_OUT_OF_MEMORY; - - SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; +int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap, + SStreamState* pState, int32_t keySize, int16_t keyType) { + pSup->resultRowSize = keySize + getResultRowSize(pCtx, numOfOutput); + pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR); + pSup->gap = gap; + pSup->stateKeySize = keySize; + pSup->stateKeyType = keyType; + pSup->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); + if (pSup->pDummyCtx == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } - pOperator->pTaskInfo = pTaskInfo; + initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput); + pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + *(pSup->pState) = *pState; + streamStateSetNumber(pSup->pState, -1); - initResultSizeInfo(&pOperator->resultInfo, 4096); - if (pSessionNode->window.pExprs != NULL) { - int32_t numOfScalar = 0; - SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } - - SExprSupp* pSup = &pOperator->exprSupp; - - SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pSup->pCtx, numOfCols); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx)); - if (pInfo->pDummyCtx == NULL) { - goto _error; - } - - initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols); - pInfo->twAggSup = (STimeWindowAggSupp){ - .waterMark = pSessionNode->window.watermark, - .calTrigger = pSessionNode->window.triggerType, - .maxTs = INT64_MIN, - .minTs = INT64_MAX, - }; - - initResultRowInfo(&pInfo->binfo.resultRowInfo); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - - pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; - if (pSessionNode->window.pTsEnd) { - pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId; - } - pInfo->gap = pSessionNode->gap; - pInfo->binfo.pRes = pResBlock; - pInfo->order = TSDB_ORDER_ASC; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); - pInfo->pDelIterator = NULL; - pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); - pInfo->pChildren = NULL; - pInfo->isFinal = false; - pInfo->pPhyNode = pPhyNode; - pInfo->ignoreExpiredData = pSessionNode->window.igExpired; + pSup->pResultRows = tSimpleHashInit(32, hashFn); - pInfo->pGroupIdTbNameMap = - taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - - pOperator->name = "StreamSessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo, - aggEncodeResultRow, aggDecodeResultRow, NULL); - if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType, - pInfo->primaryTsIndex); - code = appendDownstream(pOperator, &downstream, 1); + int32_t pageSize = 4096; + while (pageSize < pSup->resultRowSize * 4) { + pageSize <<= 1u; } - return pOperator; - -_error: - if (pInfo != NULL) { - destroyStreamSessionAggOperatorInfo(pInfo); + // at least four pages need to be in buffer + int32_t bufSize = 4096 * 256; + if (bufSize <= pageSize) { + bufSize = pageSize * 4; + } + if (!osTempSpaceAvailable()) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + qError("Init stream agg supporter failed since %s", terrstr(terrno)); + return terrno; + } + int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir); + for (int32_t i = 0; i < numOfOutput; ++i) { + pCtx[i].saveHandle.pBuf = pSup->pResultBuf; } - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - -int64_t getSessionWindowEndkey(void* data, int32_t index) { - SArray* pWinInfos = (SArray*)data; - SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index); - return pWin->win.ekey; + return TSDB_CODE_SUCCESS; } bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) { @@ -3775,305 +3583,243 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) { return false; } -bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); } - -static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs, int32_t index) { - SResultWindowInfo win = { - .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false}; - return taosArrayInsert(pWinInfos, index, &win); +bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { + return isInTimeWindow(&pWinInfo->sessionWin.win, ts, gap); } -static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs) { - SResultWindowInfo win = { - .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false}; - return taosArrayPush(pWinInfos, &win); +void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SSessionKey* pKey) { + pKey->win.skey = startTs; + pKey->win.ekey = endTs; + pKey->groupId = groupId; + SStreamStateCur* pCur = streamStateSessionGetCur(pAggSup->pState, pKey); + int32_t code = streamStateSessionGetKVByCur(pCur, pKey, NULL, 0); + streamStateFreeCur(pCur); + if (code != TSDB_CODE_SUCCESS) { + SET_SESSION_WIN_KEY_INVALID(pKey); + } } -SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) { - void** ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t)); - SArray* pWinInfos = NULL; - if (ite == NULL) { - pWinInfos = taosArrayInit(1024, pAggSup->valueSize); - taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void*)); +bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->sessionWin.win.skey == 0; } + +void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SResultWindowInfo* pCurWin) { + pCurWin->sessionWin.groupId = groupId; + pCurWin->sessionWin.win.skey = startTs - pAggSup->gap; + pCurWin->sessionWin.win.ekey = endTs + pAggSup->gap; + int32_t size = pAggSup->resultRowSize; + int32_t code = streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, &pCurWin->pOutputBuf, &size); + if (code == TSDB_CODE_SUCCESS) { + pCurWin->isOutput = true; } else { - pWinInfos = *ite; + pCurWin->sessionWin.win.skey = startTs; + pCurWin->sessionWin.win.ekey = endTs; } - return pWinInfos; } -// don't add new window -SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - int64_t gap, int32_t* pIndex) { - STimeWindow searchWin = {.skey = startTs, .ekey = endTs}; - SArray* pWinInfos = getWinInfos(pAggSup, groupId); - pAggSup->pCurWins = pWinInfos; - - int32_t size = taosArrayGetSize(pWinInfos); - if (size == 0) { - return NULL; +int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) { + int32_t size = 0; + int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, (const void**)&pWinInfo->pOutputBuf, &size); + if (code != TSDB_CODE_SUCCESS) { + return code; } - // find the first position which is smaller than the key - int32_t index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey); - SResultWindowInfo* pWin = NULL; - if (index >= 0) { - pWin = taosArrayGet(pWinInfos, index); - if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) { - *pIndex = index; - return pWin; - } - } - - if (index + 1 < size) { - pWin = taosArrayGet(pWinInfos, index + 1); - if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) { - *pIndex = index + 1; - return pWin; - } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) { - *pIndex = index + 1; - return pWin; - } - } - - return NULL; + streamStateCurNext(pAggSup->pState, pCur); + return TSDB_CODE_SUCCESS; +} +void saveDeleteInfo(SArray* pWins, SSessionKey key) { + // key.win.ekey = key.win.skey; + taosArrayPush(pWins, &key); } -SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - int64_t gap, int32_t* pIndex) { - SArray* pWinInfos = getWinInfos(pAggSup, groupId); - pAggSup->pCurWins = pWinInfos; +void saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) { + key.win.ekey = key.win.skey; + tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0); +} - int32_t size = taosArrayGetSize(pWinInfos); - if (size == 0) { - *pIndex = 0; - return addNewSessionWindow(pWinInfos, startTs, endTs); - } - // find the first position which is smaller than the key - int32_t index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey); - SResultWindowInfo* pWin = NULL; - if (index >= 0) { - pWin = taosArrayGet(pWinInfos, index); - if (isInWindow(pWin, startTs, gap)) { - *pIndex = index; - return pWin; - } - } +static void removeSessionResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey key) { + key.win.ekey = key.win.skey; + tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); +} - if (index + 1 < size) { - pWin = taosArrayGet(pWinInfos, index + 1); - if (isInWindow(pWin, startTs, gap)) { - *pIndex = index + 1; - return pWin; - } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) { - *pIndex = index; - return pWin; - } +static void removeSessionResults(SSHashObj* pHashMap, SArray* pWins) { + if (tSimpleHashGetSize(pHashMap) == 0) { + return; } - - if (index == size - 1) { - *pIndex = taosArrayGetSize(pWinInfos); - return addNewSessionWindow(pWinInfos, startTs, endTs); + int32_t size = taosArrayGetSize(pWins); + for (int32_t i = 0; i < size; i++) { + SSessionKey* pWin = taosArrayGet(pWins, i); + if (!pWin) continue; + SSessionKey key = *pWin; + key.win.ekey = key.win.skey; + tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); } - *pIndex = index + 1; - return insertNewSessionWindow(pWinInfos, startTs, endTs, index + 1); } int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId, - int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted) { + int32_t rows, int32_t start, int64_t gap, SSHashObj* pResultRows, SSHashObj* pStUpdated, + SSHashObj* pStDeleted) { for (int32_t i = start; i < rows; ++i) { if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) { return i - start; } - if (pWinInfo->win.skey > pStartTs[i]) { + if (pWinInfo->sessionWin.win.skey > pStartTs[i]) { if (pStDeleted && pWinInfo->isOutput) { - SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId}; - taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); - pWinInfo->isOutput = false; + saveDeleteRes(pStDeleted, pWinInfo->sessionWin); } - pWinInfo->win.skey = pStartTs[i]; + removeSessionResult(pStUpdated, pResultRows, pWinInfo->sessionWin); + pWinInfo->sessionWin.win.skey = pStartTs[i]; } - pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pStartTs[i]); + pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pStartTs[i]); if (pEndTs) { - pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pEndTs[i]); + pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pEndTs[i]); } } return rows - start; } -static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx, - uint64_t groupId, int32_t numOfOutput, int32_t* rowEntryInfoOffset, - SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { - assert(pWinInfo->win.skey <= pWinInfo->win.ekey); - // too many time window in query - int32_t size = taosArrayGetSize(pAggSup->pCurWins); - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH && size > MAX_INTERVAL_TIME_WINDOW) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); - } - - if (pWinInfo->pos.pageId == -1) { - *pResult = getNewResultRow(pAggSup->pResultBuf, &pAggSup->currentPageId, pAggSup->resultRowSize); - if (*pResult == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - // add a new result set for a new group - pWinInfo->pos.pageId = (*pResult)->pageId; - pWinInfo->pos.offset = (*pResult)->offset; - } else { - *pResult = getResultRowByPos(pAggSup->pResultBuf, &pWinInfo->pos, true); - if (!(*pResult)) { - qError("getResultRowByPos return NULL, TID:%s", GET_TASKID(pTaskInfo)); - return TSDB_CODE_FAILED; - } - } - +static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowEntryInfoOffset) { + ASSERT(pWinInfo->sessionWin.win.skey <= pWinInfo->sessionWin.win.ekey); + *pResult = (SResultRow*)pWinInfo->pOutputBuf; // set time window for current result - (*pResult)->win = pWinInfo->win; + (*pResult)->win = pWinInfo->sessionWin.win; setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset); return TSDB_CODE_SUCCESS; } -static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup, - SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock, - SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, - int32_t numOutput, SOperatorInfo* pOperator) { +static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, + int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, + SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId); - TSKEY* tsCols = (int64_t*)pColDataInfo->pData; - int32_t code = setWindowOutputBuf(pCurWin, pResult, pSup->pCtx, pSDataBlock->info.groupId, numOutput, - pSup->rowEntryInfoOffset, pAggSup, pTaskInfo); + int32_t code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset); if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false); - doApplyFunctions(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, pSDataBlock->info.rows, numOutput); - SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pCurWin->pos.pageId); - setBufPageDirty(bufPage, true); - releaseBufPage(pAggSup->pResultBuf, bufPage); + updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, false); + doApplyFunctions(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput); return TSDB_CODE_SUCCESS; } -static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock, - SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, - int32_t numOutput, SOperatorInfo* pOperator) { - return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData, - pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator); +static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { + streamStateSessionDel(pAggSup->pState, pKey); + tSimpleHashRemove(pAggSup->pResultRows, pKey, sizeof(SSessionKey)); + return true; } -static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock, - SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, - int32_t winRows, int32_t numOutput, SOperatorInfo* pOperator) { - return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData, - pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator); -} - -int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) { - SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex); - int32_t size = taosArrayGetSize(pWinInfos); - // Just look for the window behind StartIndex - for (int32_t i = startIndex + 1; i < size; i++) { - SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i); - if (!isInWindow(pCurWin, pWinInfo->win.skey, gap)) { - return i - startIndex - 1; - } +static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) { + void* pVal = tSimpleHashGet(pStUpdated, &pWinInfo->sessionWin, sizeof(SSessionKey)); + if (pVal) { + SResultWindowInfo* pWin = pVal; + pWinInfo->isOutput = pWin->isOutput; } - - return size - startIndex - 1; + return TSDB_CODE_SUCCESS; } -void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId, - int32_t numOfOutput, SHashObj* pStUpdated, SHashObj* pStDeleted, SOperatorInfo* pOperator) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex); - SResultRow* pCurResult = NULL; - setWindowOutputBuf(pCurWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->streamAggSup, pTaskInfo); - num += startIndex + 1; - ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins)); - // Just look for the window behind StartIndex - for (int32_t i = startIndex + 1; i < num; i++) { - SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i); - SResultRow* pWinResult = NULL; - setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->streamAggSup, pTaskInfo); - pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true); - compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); - taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition)); - if (pWinInfo->isOutput && pStDeleted) { - SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId}; - taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); - pWinInfo->isOutput = false; - } - taosArrayRemove(pInfo->streamAggSup.pCurWins, i); - SFilePage* tmpPage = getBufPage(pInfo->streamAggSup.pResultBuf, pWinInfo->pos.pageId); - releaseBufPage(pInfo->streamAggSup.pResultBuf, tmpPage); +SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, + SResultWindowInfo* pNextWin) { + SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin); + pNextWin->isOutput = true; + setSessionWinOutputInfo(pStUpdated, pNextWin); + int32_t size = 0; + pNextWin->sessionWin = pCurWin->sessionWin; + int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, (const void**)&pNextWin->pOutputBuf, &size); + if (code != TSDB_CODE_SUCCESS) { + SET_SESSION_WIN_INVALID(*pNextWin); } - SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pCurWin->pos.pageId); - ASSERT(num > 0); - setBufPageDirty(bufPage, true); - releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage); + return pCur; } -static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated, - SHashObj* pStDeleted, bool hasEndTs) { +static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated, + SSHashObj* pStDeleted) { + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + SResultRow* pCurResult = NULL; + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + // Just look for the window behind StartIndex + while (1) { + SResultWindowInfo winInfo = {0}; + SStreamStateCur* pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo); + if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) { + streamStateFreeCur(pCur); + break; + } + SResultRow* pWinResult = NULL; + initSessionOutputBuf(&winInfo, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); + pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, winInfo.sessionWin.win.ekey); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->sessionWin.win, true); + compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); + tSimpleHashRemove(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey)); + if (winInfo.isOutput && pStDeleted) { + saveDeleteRes(pStDeleted, winInfo.sessionWin); + } + removeSessionResult(pStUpdated, pAggSup->pResultRows, winInfo.sessionWin); + doDeleteSessionWindow(pAggSup, &winInfo.sessionWin); + streamStateFreeCur(pCur); + } +} + +int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) { + saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize); + return TSDB_CODE_SUCCESS; +} + +static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated, + SSHashObj* pStDeleted, bool hasEndTs) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - bool masterScan = true; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.groupId; - int64_t gap = pInfo->gap; int64_t code = TSDB_CODE_SUCCESS; + SResultRow* pResult = NULL; + int32_t rows = pSDataBlock->info.rows; + int32_t winRows = 0; - int32_t step = 1; - bool ascScan = true; - TSKEY* startTsCols = NULL; - TSKEY* endTsCols = NULL; - SResultRow* pResult = NULL; - int32_t winRows = 0; - - ASSERT(pSDataBlock->pDataBlock); SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - startTsCols = (int64_t*)pStartTsCol->pData; + TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = NULL; if (hasEndTs) { pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex); } else { pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); } - endTsCols = (int64_t*)pEndTsCol->pData; + TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - for (int32_t i = 0; i < pSDataBlock->info.rows;) { + for (int32_t i = 0; i < rows;) { if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) { i++; continue; } - int32_t winIndex = 0; - SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex); - winRows = updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, groupId, pSDataBlock->info.rows, i, pInfo->gap, - pStDeleted); - code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator); + SResultWindowInfo winInfo = {0}; + setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo); + setSessionWinOutputInfo(pStUpdated, &winInfo); + winRows = updateSessionWindowInfo(&winInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap, + pAggSup->pResultRows, pStUpdated, pStDeleted); + code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, + pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } + compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted); + saveSessionOutputBuf(pAggSup, &winInfo); - int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap); - if (winNum > 0) { - compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, pStDeleted, pOperator); - } - pCurWin->isClosed = false; if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { - SWinKey value = {.ts = pCurWin->win.skey, .groupId = groupId}; - code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey)); + code = saveResult(winInfo, pStUpdated); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pCurWin->isOutput = true; } + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + SSessionKey key = winInfo.sessionWin; + key.win.ekey = key.win.skey; + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + } + i += winRows; } } @@ -4087,8 +3833,7 @@ void deleteWindow(SArray* pWinInfos, int32_t index, FDelete fp) { taosArrayRemove(pWinInfos, index); } -static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result, - FDelete fp) { +static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -4096,85 +3841,70 @@ static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { - int32_t winIndex = 0; - SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex); - if (!pCurWin) { - continue; - } - - do { - SResultWindowInfo delWin = *pCurWin; - deleteWindow(pAggSup->pCurWins, winIndex, fp); - if (result) { - delWin.groupId = gpDatas[i]; - taosArrayPush(result, &delWin); - } - if (winIndex >= taosArrayGetSize(pAggSup->pCurWins)) { + while (1) { + SSessionKey curWin = {0}; + getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], &curWin); + if (IS_INVALID_SESSION_WIN_KEY(curWin)) { break; } - pCurWin = taosArrayGet(pAggSup->pCurWins, winIndex); - } while (pCurWin->win.skey <= endDatas[i]); - } -} - -static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock, int32_t tsIndex, - int32_t numOfOutput, int64_t gap, SArray* result) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); - TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; - SColumnInfoData* pGpDataInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* gpCols = (uint64_t*)pGpDataInfo->pData; - int32_t step = 0; - for (int32_t i = 0; i < pBlock->info.rows; i += step) { - int32_t winIndex = 0; - SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, tsCols[i], INT64_MIN, gpCols[i], gap, &winIndex); - if (!pCurWin || pCurWin->pos.pageId == -1) { - // window has been closed. - step = 1; - continue; - } - step = updateSessionWindowInfo(pCurWin, tsCols, NULL, 0, pBlock->info.rows, i, gap, NULL); - ASSERT(isInWindow(pCurWin, tsCols[i], gap)); - doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput); - if (result) { - pCurWin->groupId = gpCols[i]; - taosArrayPush(result, pCurWin); + doDeleteSessionWindow(pAggSup, &curWin); + if (result) { + saveDeleteInfo(result, curWin); + } } } } -static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) { - void* pData = NULL; - size_t keyLen = 0; - while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) { - void* key = taosHashGetKey(pData, &keyLen); - ASSERT(keyLen == sizeof(SResultRowPosition)); - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - if (pos == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - pos->groupId = ((SWinKey*)pData)->groupId; - pos->pos = *(SResultRowPosition*)key; - *(int64_t*)pos->key = ((SWinKey*)pData)->ts; - taosArrayPush(pUpdated, &pos); +static inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) { + SSessionKey* pWin1 = (SSessionKey*)pKey1; + SSessionKey* pWin2 = (SSessionKey*)pKey2; + + if (pWin1->groupId > pWin2->groupId) { + return 1; + } else if (pWin1->groupId < pWin2->groupId) { + return -1; } - taosArraySort(pUpdated, resultrowComparAsc); + + if (pWin1->win.skey > pWin2->win.skey) { + return 1; + } else if (pWin1->win.skey < pWin2->win.skey) { + return -1; + } + + return 0; +} + +static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) { + void* pIte = NULL; + size_t keyLen = 0; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pStUpdated, pIte, &iter)) != NULL) { + void* key = tSimpleHashGetKey(pIte, &keyLen); + ASSERT(keyLen == sizeof(SSessionKey)); + taosArrayPush(pUpdated, key); + } + taosArraySort(pUpdated, sessionKeyCompareAsc); return TSDB_CODE_SUCCESS; } -void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { +void doBuildDeleteDataBlock(SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { blockDataCleanup(pBlock); - int32_t size = taosHashGetSize(pStDeleted); + int32_t size = tSimpleHashGetSize(pStDeleted); if (size == 0) { return; } blockDataEnsureCapacity(pBlock, size); - size_t keyLen = 0; - while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { - SWinKey* res = *Ite; + size_t keyLen = 0; + int32_t iter = 0; + while (((*Ite) = tSimpleHashIterate(pStDeleted, *Ite, &iter)) != NULL) { + if (pBlock->info.rows + 1 > pBlock->info.capacity) { + break; + } + SSessionKey* res = tSimpleHashGetKey(*Ite, &keyLen); SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->ts, false); + colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->ts, false); + colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); colDataAppendNULL(pUidCol, pBlock->info.rows); SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); @@ -4184,167 +3914,137 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); colDataAppendNULL(pCalEdCol, pBlock->info.rows); pBlock->info.rows += 1; - if (pBlock->info.rows + 1 >= pBlock->info.capacity) { - break; - } } if ((*Ite) == NULL) { - taosHashClear(pStDeleted); + tSimpleHashClear(pStDeleted); } } -static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t numOfOutput, - SOperatorInfo* pOperator, SHashObj* pStUpdated) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - int32_t size = taosArrayGetSize(pWinArray); +static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) { + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int32_t size = taosArrayGetSize(pWinArray); + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + int32_t numOfOutput = pSup->numOfExprs; + int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); ASSERT(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { - SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i); - uint64_t groupId = pParentWin->groupId; - int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); + SSessionKey* pWinKey = taosArrayGet(pWinArray, i); + int32_t num = 0; + SResultWindowInfo parentWin = {0}; for (int32_t j = 0; j < numOfChildren; j++) { SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; - SArray* pChWins = getWinInfos(&pChInfo->streamAggSup, groupId); - int32_t chWinSize = taosArrayGetSize(pChWins); - int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey); - if (index < 0) { - index = 0; - } - for (int32_t k = index; k < chWinSize; k++) { - SResultWindowInfo* pChWin = taosArrayGet(pChWins, k); - if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) { - int32_t winIndex = 0; - SResultWindowInfo* pNewParWin = - getSessionTimeWindow(&pInfo->streamAggSup, pChWin->win.skey, pChWin->win.ekey, groupId, 0, &winIndex); - SResultRow* pPareResult = NULL; - setWindowOutputBuf(pNewParWin, &pPareResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->streamAggSup, pTaskInfo); - SResultRow* pChResult = NULL; - setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput, - pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pNewParWin->win, true); - compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); - - int32_t winNum = getNumCompactWindow(pInfo->streamAggSup.pCurWins, winIndex, pInfo->gap); - if (winNum > 0) { - compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, NULL, pOperator); + SStreamAggSupporter* pChAggSup = &pChInfo->streamAggSup; + SStreamStateCur* pCur = streamStateSessionGetCur(pChAggSup->pState, pWinKey); + SResultRow* pResult = NULL; + SResultRow* pChResult = NULL; + while (1) { + SResultWindowInfo childWin = {0}; + childWin.sessionWin = *pWinKey; + int32_t code = getSessionWinBuf(pChAggSup, pCur, &childWin); + if (code == TSDB_CODE_SUCCESS && pWinKey->win.skey <= childWin.sessionWin.win.skey && + childWin.sessionWin.win.ekey <= pWinKey->win.ekey) { + if (num == 0) { + setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin); + code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + break; + } } - - SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId); - releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage); - - bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pNewParWin->pos.pageId); - setBufPageDirty(bufPage, true); - releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage); - SWinKey value = {.ts = pNewParWin->win.skey, .groupId = groupId}; - taosHashPut(pStUpdated, &pNewParWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey)); - } else if (!pChWin->isClosed) { + num++; + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, true); + initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput, + pChild->exprSupp.rowEntryInfoOffset); + compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); + compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL); + saveResult(parentWin, pStUpdated); + } else { break; } } + streamStateFreeCur(pCur); + } + if (num > 0) { + saveSessionOutputBuf(pAggSup, &parentWin); } } } -typedef SResultWindowInfo* (*__get_win_info_)(void*); -SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; } -SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; } - -int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn, - bool delete, FDelete fp) { - // Todo(liuyao) save window to tdb - void** pIte = NULL; - size_t keyLen = 0; - while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { - uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen); - SArray* pWins = (SArray*)(*pIte); - int32_t size = taosArrayGetSize(pWins); - for (int32_t i = 0; i < size; i++) { - void* pWin = taosArrayGet(pWins, i); - SResultWindowInfo* pSeWin = fn(pWin); - if (isCloseWindow(&pSeWin->win, pTwSup)) { - if (!pSeWin->isClosed) { - pSeWin->isClosed = true; - if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) { - int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - pSeWin->isOutput = true; - } - if (delete) { - deleteWindow(pWins, i, fp); - i--; - size = taosArrayGetSize(pWins); - } +int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed) { + void* pIte = NULL; + size_t keyLen = 0; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { + SResultWindowInfo* pWinInfo = pIte; + if (isCloseWindow(&pWinInfo->sessionWin.win, pTwSup)) { + if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) { + int32_t code = saveResult(*pWinInfo, pClosed); + if (code != TSDB_CODE_SUCCESS) { + return code; } - continue; } - break; + tSimpleHashIterateRemove(pHashMap, &pWinInfo->sessionWin, sizeof(SSessionKey), &pIte, &iter); } } return TSDB_CODE_SUCCESS; } -static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs, bool delete, FDelete fp) { +static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) { int32_t size = taosArrayGetSize(pChildren); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); - closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL, getResWinForSession, delete, fp); + closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL); } } -int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) { - void** pIte = NULL; - while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { - SArray* pWins = (SArray*)(*pIte); - int32_t size = taosArrayGetSize(pWins); - for (int32_t i = 0; i < size; i++) { - void* pWin = taosArrayGet(pWins, i); - SResultWindowInfo* pSeWin = fn(pWin); - if (!pSeWin->isClosed) { - int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed); - pSeWin->isOutput = true; - } - } +int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { + void* pIte = NULL; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { + SResultWindowInfo* pWinInfo = *(void**)pIte; + saveResult(*pWinInfo, pStUpdated); } return TSDB_CODE_SUCCESS; } -static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) { +static void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) { int32_t size = taosArrayGetSize(pResWins); for (int32_t i = 0; i < size; i++) { - SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i); - SWinKey res = {.ts = pWinInfo->win.skey, .groupId = pWinInfo->groupId}; - taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); + SSessionKey* pWinKey = taosArrayGet(pResWins, i); + if (!pWinKey) continue; + SSessionKey winInfo = *pWinKey; + winInfo.win.ekey = winInfo.win.skey; + tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0); } } -static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) { - int32_t size = taosArrayGetSize(pWins); - for (int32_t i = 0; i < size; i++) { - SResultWindowInfo* pWin = taosArrayGet(pWins, i); - taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition)); - } +void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) { + pGroupResInfo->pRows = pArrayList; + pGroupResInfo->index = 0; + ASSERT(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } -static void removeSessionDeleteResults(SArray* update, SHashObj* pStDeleted) { - int32_t size = taosHashGetSize(pStDeleted); - if (size == 0) { +void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo, + SSDataBlock* pBlock) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + // set output datablock version + pBlock->info.version = pTaskInfo->version; + + blockDataCleanup(pBlock); + if (!hasRemainResults(pGroupResInfo)) { + taosArrayDestroy(pGroupResInfo->pRows); + pGroupResInfo->pRows = NULL; return; } - int32_t num = taosArrayGetSize(update); - for (int32_t i = 0; i < num; i++) { - SResKeyPos* pos = taosArrayGetP(update, i); - SWinKey winKey = {.ts = *(int64_t*)pos->key, .groupId = pos->groupId}; - taosHashRemove(pStDeleted, &winKey, sizeof(SWinKey)); - } + // clear the existed group id + pBlock->info.groupId = 0; + buildSessionResultDataBlock(pTaskInfo, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); } static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { @@ -4352,6 +4052,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; TSKEY maxTs = INT64_MIN; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; if (pOperator->status == OP_EXEC_DONE) { return NULL; } else if (pOperator->status == OP_RES_TO_RETURN) { @@ -4360,18 +4061,20 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); return pInfo->pDelRes; } - doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes); + if (pBInfo->pRes->info.rows > 0) { + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); + return pBInfo->pRes; } - printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; + + doSetOperatorCompleted(pOperator); + return NULL; } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + SSHashObj* pStUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); // SResKeyPos + SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); // SResKeyPos while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -4385,38 +4088,25 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { /*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/ } - if (pBlock->info.type == STREAM_CLEAR) { - SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, START_TS_COLUMN_INDEX, - pOperator->exprSupp.numOfExprs, 0, pWins); - if (IS_FINAL_OP(pInfo)) { - int32_t childIndex = getChildIndex(pBlock); - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); - SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; - doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX, - pChildOp->exprSupp.numOfExprs, 0, NULL); - rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated); - } - taosArrayDestroy(pWins); - continue; - } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { - SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); + if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || + pBlock->info.type == STREAM_CLEAR) { + SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); // gap must be 0 - doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL); + doDeleteTimeWindows(pAggSup, pBlock, pWins); + removeSessionResults(pStUpdated, pWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 - doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL); - rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated); + doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL); + rebuildSessionWindow(pOperator, pWins, pStUpdated); } copyDeleteWindowInfo(pWins, pInfo->pStDeleted); - removeSessionResults(pStUpdated, pWins); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession); + getAllSessionWindow(pAggSup->pResultRows, pStUpdated); continue; } @@ -4424,7 +4114,6 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SExprSupp* pExprSup = &pInfo->scalarSupp; projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } - // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo)); if (IS_FINAL_OP(pInfo)) { @@ -4451,39 +4140,121 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession, - pInfo->ignoreExpiredData, NULL); - closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, NULL); + closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pStUpdated); + closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); copyUpdateResult(pStUpdated, pUpdated); - removeSessionDeleteResults(pUpdated, pInfo->pStDeleted); - taosHashCleanup(pStUpdated); - - finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + removeSessionResults(pInfo->pStDeleted, pUpdated); + tSimpleHashCleanup(pStUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); return pInfo->pDelRes; } - doBuildStreamResBlock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; + + doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes); + if (pBInfo->pRes->info.rows > 0) { + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); + return pBInfo->pRes; + } + + doSetOperatorCompleted(pOperator); + return NULL; +} + +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo) { + SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; + int32_t numOfCols = 0; + int32_t code = TSDB_CODE_OUT_OF_MEMORY; + SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pOperator->pTaskInfo = pTaskInfo; + + initResultSizeInfo(&pOperator->resultInfo, 4096); + if (pSessionNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + SExprSupp* pSup = &pOperator->exprSupp; + + SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, pSessionNode->gap, + pTaskInfo->streamInfo.pState, 0, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->twAggSup = (STimeWindowAggSupp){ + .waterMark = pSessionNode->window.watermark, + .calTrigger = pSessionNode->window.triggerType, + .maxTs = INT64_MIN, + .minTs = INT64_MAX, + }; + + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; + if (pSessionNode->window.pTsEnd) { + pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId; + } + pInfo->binfo.pRes = pResBlock; + pInfo->order = TSDB_ORDER_ASC; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pStDeleted = tSimpleHashInit(64, hashFn); + pInfo->pDelIterator = NULL; + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); + pInfo->pChildren = NULL; + pInfo->isFinal = false; + pInfo->pPhyNode = pPhyNode; + pInfo->ignoreExpiredData = pSessionNode->window.igExpired; + pInfo->pGroupIdTbNameMap = + taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); + + pOperator->name = "StreamSessionWindowAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); + if (downstream) { + initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, + pInfo->primaryTsIndex); + code = appendDownstream(pOperator, &downstream, 1); + } + return pOperator; + +_error: + if (pInfo != NULL) { + destroyStreamSessionAggOperatorInfo(pInfo); + } + + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; } static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { - void** pIte = NULL; - while ((pIte = taosHashIterate(pInfo->streamAggSup.pResultRows, pIte)) != NULL) { - SArray* pWins = (SArray*)(*pIte); - int32_t size = taosArrayGetSize(pWins); - for (int32_t i = 0; i < size; i++) { - SResultWindowInfo* pWin = (SResultWindowInfo*)taosArrayGet(pWins, i); - pWin->pos.pageId = -1; - pWin->pos.offset = -1; - } - } - clearDiskbasedBuf(pInfo->streamAggSup.pResultBuf); - pInfo->streamAggSup.currentPageId = -1; + tSimpleHashClear(pInfo->streamAggSup.pResultRows); + streamStateSessionClear(pInfo->streamAggSup.pState); } static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { @@ -4491,13 +4262,14 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { SOptrBasicInfo* pBInfo = &pInfo->binfo; TSKEY maxTs = INT64_MIN; SExprSupp* pSup = &pOperator->exprSupp; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; if (pOperator->status == OP_EXEC_DONE) { return NULL; } { - doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes); if (pBInfo->pRes->info.rows > 0) { printDataBlock(pBInfo->pRes, "semi session"); return pBInfo->pRes; @@ -4505,30 +4277,23 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "semi session"); + printDataBlock(pInfo->pDelRes, "semi session delete"); return pInfo->pDelRes; } - if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) { - pInfo->returnUpdate = false; - // process the rest of the data - printDataBlock(pInfo->pUpdateRes, "semi session"); - return pInfo->pUpdateRes; - } - if (pOperator->status == OP_RES_TO_RETURN) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamSessionOperator(pInfo); - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); return NULL; } } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + SSHashObj* pStUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); + SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -4544,24 +4309,17 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { /*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/ } - if (pBlock->info.type == STREAM_CLEAR) { - SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, START_TS_COLUMN_INDEX, pSup->numOfExprs, 0, pWins); - removeSessionResults(pStUpdated, pWins); - taosArrayDestroy(pWins); - copyDataBlock(pInfo->pUpdateRes, pBlock); - pInfo->returnUpdate = true; - break; - } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { + if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || + pBlock->info.type == STREAM_CLEAR) { // gap must be 0 SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL); - copyDeleteWindowInfo(pWins, pInfo->pStDeleted); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins); removeSessionResults(pStUpdated, pWins); + copyDeleteWindowInfo(pWins, pInfo->pStDeleted); taosArrayDestroy(pWins); break; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession); + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pStUpdated); continue; } @@ -4579,15 +4337,13 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs; copyUpdateResult(pStUpdated, pUpdated); - removeSessionDeleteResults(pUpdated, pInfo->pStDeleted); - taosHashCleanup(pStUpdated); + removeSessionResults(pInfo->pStDeleted, pUpdated); + tSimpleHashCleanup(pStUpdated); - finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, - pSup->rowEntryInfoOffset); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes); if (pBInfo->pRes->info.rows > 0) { printDataBlock(pBInfo->pRes, "semi session"); return pBInfo->pRes; @@ -4595,18 +4351,14 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "semi session"); + printDataBlock(pInfo->pDelRes, "semi session delete"); return pInfo->pDelRes; } - if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) { - pInfo->returnUpdate = false; - // process the rest of the data - printDataBlock(pInfo->pUpdateRes, "semi session"); - return pInfo->pUpdateRes; - } - - pOperator->status = OP_EXEC_DONE; + clearFunctionContext(&pOperator->exprSupp); + // semi interval operator clear disk buffer + clearStreamSessionOperator(pInfo); + doSetOperatorCompleted(pOperator); return NULL; } @@ -4639,11 +4391,14 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream if (numOfChild > 0) { pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*)); for (int32_t i = 0; i < numOfChild; i++) { - SOperatorInfo* pChild = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); - if (pChild == NULL) { + SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); + if (pChildOp == NULL) { goto _error; } - taosArrayPush(pInfo->pChildren, &pChild); + SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; + pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; + streamStateSetNumber(pChInfo->streamAggSup.pState, i); + taosArrayPush(pInfo->pChildren, &pChildOp); } } return pOperator; @@ -4652,7 +4407,6 @@ _error: if (pInfo != NULL) { destroyStreamSessionAggOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; @@ -4661,7 +4415,7 @@ _error: void destroyStreamStateOperatorInfo(void* param) { SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); - destroyStateStreamAggSupporter(&pInfo->streamAggSup); + destroyStreamAggSupporter(&pInfo->streamAggSup); cleanupGroupResInfo(&pInfo->groupResInfo); if (pInfo->pChildren != NULL) { int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -4674,151 +4428,75 @@ void destroyStreamStateOperatorInfo(void* param) { } colDataDestroy(&pInfo->twAggSup.timeWindowData); blockDataDestroy(pInfo->pDelRes); - taosHashCleanup(pInfo->pSeDeleted); + tSimpleHashCleanup(pInfo->pSeDeleted); taosHashCleanup(pInfo->pGroupIdTbNameMap); - destroySqlFunctionCtx(pInfo->pDummyCtx, 0); - taosMemoryFreeClear(param); } -int64_t getStateWinTsKey(void* data, int32_t index) { - SStateWindowInfo* pStateWin = taosArrayGet(data, index); - return pStateWin->winInfo.win.ekey; -} - -SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) { - SStateWindowInfo win = { - .stateKey.bytes = pCol->bytes, - .stateKey.type = pCol->type, - .stateKey.pData = taosMemoryCalloc(1, pCol->bytes), - .winInfo.pos.offset = -1, - .winInfo.pos.pageId = -1, - .winInfo.win.skey = ts, - .winInfo.win.ekey = ts, - .winInfo.isOutput = false, - .winInfo.isClosed = false, - }; - if (IS_VAR_DATA_TYPE(win.stateKey.type)) { - varDataCopy(win.stateKey.pData, pKeyData); - } else { - memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes); - } - return taosArrayPush(pWinInfos, &win); -} - -SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) { - SStateWindowInfo win = { - .stateKey.bytes = pCol->bytes, - .stateKey.type = pCol->type, - .stateKey.pData = taosMemoryCalloc(1, pCol->bytes), - .winInfo.pos.offset = -1, - .winInfo.pos.pageId = -1, - .winInfo.win.skey = ts, - .winInfo.win.ekey = ts, - .winInfo.isOutput = false, - .winInfo.isClosed = false, - }; - if (IS_VAR_DATA_TYPE(win.stateKey.type)) { - varDataCopy(win.stateKey.pData, pKeyData); - } else { - memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes); - } - return taosArrayInsert(pWinInfos, index, &win); -} - bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) { - if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) { + if (pWin->winInfo.sessionWin.win.skey <= ts && ts <= pWin->winInfo.sessionWin.win.ekey) { return true; } return false; } bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) { - return pKeyData && compareVal(pKeyData, &pWin->stateKey); + return pKeyData && compareVal(pKeyData, pWin->pStateKey); } -SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) { - SArray* pWinInfos = getWinInfos(pAggSup, groupId); - pAggSup->pCurWins = pWinInfos; - int32_t size = taosArrayGetSize(pWinInfos); - int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey); - SStateWindowInfo* pWin = NULL; - if (index >= 0) { - pWin = taosArrayGet(pWinInfos, index); - if (isTsInWindow(pWin, ts)) { - *pIndex = index; - return pWin; - } - } - - if (index + 1 < size) { - pWin = taosArrayGet(pWinInfos, index + 1); - if (isTsInWindow(pWin, ts)) { - *pIndex = index + 1; - return pWin; - } - } - *pIndex = 0; - return NULL; +bool compareStateKey(void* data, void* key) { + SStateKeys* stateKey = (SStateKeys*)key; + stateKey->pData = (char*)key + sizeof(SStateKeys); + return compareVal(data, stateKey); } -SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, - SColumn* pCol, int32_t* pIndex) { - SArray* pWinInfos = getWinInfos(pAggSup, groupId); - pAggSup->pCurWins = pWinInfos; - int32_t size = taosArrayGetSize(pWinInfos); - if (size == 0) { - *pIndex = 0; - return addNewStateWindow(pWinInfos, ts, pKeyData, pCol); - } - int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey); - SStateWindowInfo* pWin = NULL; - if (index >= 0) { - pWin = taosArrayGet(pWinInfos, index); - if (isTsInWindow(pWin, ts)) { - *pIndex = index; - return pWin; +void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, + SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) { + int32_t size = pAggSup->resultRowSize; + pCurWin->winInfo.sessionWin.groupId = groupId; + pCurWin->winInfo.sessionWin.win.skey = ts; + pCurWin->winInfo.sessionWin.win.ekey = ts; + int32_t code = + streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize, + compareStateKey, &pCurWin->winInfo.pOutputBuf, &size); + pCurWin->pStateKey = + (SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize)); + pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys); + pCurWin->pStateKey->type = pAggSup->stateKeyType; + pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys); + pCurWin->pStateKey->isNull = false; + + if (code == TSDB_CODE_SUCCESS) { + pCurWin->winInfo.isOutput = true; + } else { + if (IS_VAR_DATA_TYPE(pAggSup->stateKeyType)) { + varDataCopy(pCurWin->pStateKey->pData, pKeyData); + } else { + memcpy(pCurWin->pStateKey->pData, pKeyData, pCurWin->pStateKey->bytes); } } - if (index + 1 < size) { - pWin = taosArrayGet(pWinInfos, index + 1); - if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) { - *pIndex = index + 1; - return pWin; - } + pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin; + pNextWin->winInfo.pOutputBuf = NULL; + SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); + code = streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + SET_SESSION_WIN_INVALID(pNextWin->winInfo); } - - if (index >= 0) { - pWin = taosArrayGet(pWinInfos, index); - if (isEqualStateKey(pWin, pKeyData)) { - *pIndex = index; - return pWin; - } - } - - if (index == size - 1) { - *pIndex = taosArrayGetSize(pWinInfos); - return addNewStateWindow(pWinInfos, ts, pKeyData, pCol); - } - *pIndex = index + 1; - return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol); + streamStateFreeCur(pCur); } -int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, uint64_t groupId, +int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId, SColumnInfoData* pKeyCol, int32_t rows, int32_t start, bool* allEqual, - SHashObj* pSeDeleted) { + SSHashObj* pResultRows, SSHashObj* pSeUpdated, SSHashObj* pSeDeleted) { *allEqual = true; - SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex); for (int32_t i = start; i < rows; ++i) { char* pKeyData = colDataGetData(pKeyCol, i); if (!isTsInWindow(pWinInfo, pTs[i])) { if (isEqualStateKey(pWinInfo, pKeyData)) { - int32_t size = taosArrayGetSize(pWinInfos); - if (winIndex + 1 < size) { - SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1); + if (IS_VALID_SESSION_WIN(pNextWin->winInfo)) { // ts belongs to the next window - if (pTs[i] >= pNextWin->winInfo.win.skey) { + if (pTs[i] >= pNextWin->winInfo.sessionWin.win.skey) { return i - start; } } @@ -4826,15 +4504,15 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, u return i - start; } } - if (pWinInfo->winInfo.win.skey > pTs[i]) { + + if (pWinInfo->winInfo.sessionWin.win.skey > pTs[i]) { if (pSeDeleted && pWinInfo->winInfo.isOutput) { - SWinKey res = {.ts = pWinInfo->winInfo.win.skey, .groupId = groupId}; - taosHashPut(pSeDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); - pWinInfo->winInfo.isOutput = false; + saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin); } - pWinInfo->winInfo.win.skey = pTs[i]; + removeSessionResult(pSeUpdated, pResultRows, pWinInfo->winInfo.sessionWin); + pWinInfo->winInfo.sessionWin.win.skey = pTs[i]; } - pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]); + pWinInfo->winInfo.sessionWin.win.ekey = TMAX(pWinInfo->winInfo.sessionWin.win.ekey, pTs[i]); if (!isEqualStateKey(pWinInfo, pKeyData)) { *allEqual = false; } @@ -4842,36 +4520,13 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, u return rows - start; } -static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SHashObj* pSeUpdated, - SHashObj* pSeDeleted) { - SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pGroupColInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - TSKEY* tsCol = (TSKEY*)pTsColInfo->pData; - bool allEqual = false; - int32_t step = 1; - uint64_t* gpCol = (uint64_t*)pGroupColInfo->pData; - for (int32_t i = 0; i < pBlock->info.rows; i += step) { - int32_t winIndex = 0; - SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], gpCol[i], &winIndex); - if (!pCurWin) { - continue; - } - updateSessionWindowInfo(&pCurWin->winInfo, tsCol, NULL, 0, pBlock->info.rows, i, 0, NULL); - taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); - deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo); - } -} - -static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated, - SHashObj* pStDeleted) { +static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, + SSHashObj* pStDeleted) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamStateAggOperatorInfo* pInfo = pOperator->info; - bool masterScan = true; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; int64_t groupId = pSDataBlock->info.groupId; int64_t code = TSDB_CODE_SUCCESS; - int32_t step = 1; - bool ascScan = true; TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t winRows = 0; @@ -4883,39 +4538,44 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - blockDataEnsureCapacity(pAggSup->pScanBlock, pSDataBlock->info.rows); + int32_t rows = pSDataBlock->info.rows; + blockDataEnsureCapacity(pAggSup->pScanBlock, rows); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); - for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) { + for (int32_t i = 0; i < rows; i += winRows) { if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) { i++; continue; } - char* pKeyData = colDataGetData(pKeyColInfo, i); - int32_t winIndex = 0; - bool allEqual = true; - SStateWindowInfo* pCurWin = getStateWindow(pAggSup, tsCols[i], groupId, pKeyData, &pInfo->stateCol, &winIndex); - winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, groupId, pKeyColInfo, pSDataBlock->info.rows, - i, &allEqual, pStDeleted); + char* pKeyData = colDataGetData(pKeyColInfo, i); + int32_t winIndex = 0; + bool allEqual = true; + SStateWindowInfo curWin = {0}; + SStateWindowInfo nextWin = {0}; + setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); + setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); + winRows = updateStateWindowInfo(&curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual, + pAggSup->pResultRows, pSeUpdated, pStDeleted); if (!allEqual) { uint64_t uid = 0; - appendOneRowToStreamSpecialBlock(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, - &uid, &groupId, NULL); - taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); - deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo); + appendOneRowToStreamSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, + &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); + tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); + doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf); continue; } - code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pOperator); + code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, + pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pCurWin->winInfo.isClosed = false; + saveSessionOutputBuf(pAggSup, &curWin.winInfo); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - SWinKey value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId}; - code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey)); + code = saveResult(curWin.winInfo, pSeUpdated); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pCurWin->winInfo.isOutput = true; } } } @@ -4932,21 +4592,24 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "single state"); + printDataBlock(pInfo->pDelRes, "single state delete"); return pInfo->pDelRes; } - doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + + doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes); + if (pBInfo->pRes->info.rows > 0) { + printDataBlock(pBInfo->pRes, "single state"); + return pBInfo->pRes; } - printDataBlock(pBInfo->pRes, "single state"); - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; + + doSetOperatorCompleted(pOperator); + return NULL; } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + SSHashObj* pSeUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); + SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -4954,18 +4617,16 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { } printDataBlock(pBlock, "single state recv"); - if (pBlock->info.type == STREAM_CLEAR) { - doClearStateWindows(&pInfo->streamAggSup, pBlock, pSeUpdated, pInfo->pSeDeleted); - continue; - } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { - SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo); - copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); + if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || + pBlock->info.type == STREAM_CLEAR) { + SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins); removeSessionResults(pSeUpdated, pWins); + copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState); + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pSeUpdated); continue; } @@ -4982,28 +4643,27 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState, - pInfo->ignoreExpiredData, destroyStateWinInfo); - // closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, destroyStateWinInfo); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pSeUpdated); copyUpdateResult(pSeUpdated, pUpdated); - taosHashCleanup(pSeUpdated); + removeSessionResults(pInfo->pSeDeleted, pUpdated); + tSimpleHashCleanup(pSeUpdated); - finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, - pSup->rowEntryInfoOffset); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "single state"); + printDataBlock(pInfo->pDelRes, "single state delete"); return pInfo->pDelRes; } - doBuildStreamResBlock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - printDataBlock(pBInfo->pRes, "single state"); - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; -} -int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) { - return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo)); + doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes); + if (pBInfo->pRes->info.rows > 0) { + printDataBlock(pBInfo->pRes, "single state"); + return pBInfo->pRes; + } + doSetOperatorCompleted(pOperator); + return NULL; } SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -5020,11 +4680,6 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys goto _error; } - SExprSupp* pSup = &pOperator->exprSupp; - - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols); - pInfo->stateCol = extractColumnFromColumnNode(pColNode); initResultSizeInfo(&pOperator->resultInfo, 4096); if (pStateNode->window.pExprs != NULL) { @@ -5036,7 +4691,6 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys } } - initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->twAggSup = (STimeWindowAggSupp){ .waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType, @@ -5045,27 +4699,25 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys }; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + SExprSupp* pSup = &pOperator->exprSupp; + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); if (code != TSDB_CODE_SUCCESS) { goto _error; } - - code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pSup->pCtx, numOfCols); + int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes; + int16_t type = pColNode->node.resType.type; + code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, + type); if (code != TSDB_CODE_SUCCESS) { goto _error; } - pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx)); - if (pInfo->pDummyCtx == NULL) { - goto _error; - } - - initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols); pInfo->primaryTsIndex = tsSlotId; - pInfo->order = TSDB_ORDER_ASC; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + pInfo->pSeDeleted = tSimpleHashInit(64, hashFn); pInfo->pDelIterator = NULL; pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->pChildren = NULL; @@ -5082,7 +4734,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL, destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType, + initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5619,8 +5271,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge goto _error; } - int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); SInterval interval = {.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 7077a9b780..49df3aa136 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2527,6 +2527,8 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { int32_t start = pInput->startRowIndex; if (pInfo->algo == APERCT_ALGO_TDIGEST) { + buildTDigestInfo(pInfo); + tdigestAutoFill(pInfo->pTDigest, COMPRESSION); for (int32_t i = start; i < pInput->numOfRows + start; ++i) { if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; @@ -2540,12 +2542,11 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { tdigestAdd(pInfo->pTDigest, v, w); } } else { - qDebug("%s before add %d elements into histogram, total:%d, numOfEntry:%d, pHisto:%p, elems: %p", __FUNCTION__, - numOfElems, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); - // might be a race condition here that pHisto can be overwritten or setup function // has not been called, need to relink the buffer pHisto points to. buildHistogramInfo(pInfo); + qDebug("%s before add %d elements into histogram, total:%d, numOfEntry:%d, pHisto:%p, elems: %p", __FUNCTION__, + numOfElems, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); for (int32_t i = start; i < pInput->numOfRows + start; ++i) { if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; @@ -2579,8 +2580,9 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* buildTDigestInfo(pOutput); TDigest* pTDigest = pOutput->pTDigest; + tdigestAutoFill(pTDigest, COMPRESSION); - if (pTDigest->num_centroids <= 0) { + if (pTDigest->num_centroids <= 0 && pTDigest->num_buffered_pts == 0) { memcpy(pTDigest, pInput->pTDigest, (size_t)TDIGEST_SIZE(COMPRESSION)); tdigestAutoFill(pTDigest, COMPRESSION); } else { @@ -2652,6 +2654,7 @@ int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pInfo->algo == APERCT_ALGO_TDIGEST) { buildTDigestInfo(pInfo); + tdigestAutoFill(pInfo->pTDigest, COMPRESSION); if (pInfo->pTDigest->size > 0) { pInfo->result = tdigestQuantile(pInfo->pTDigest, pInfo->percent / 100); } else { // no need to free diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index c0ed135d74..6991f00bb7 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -24,6 +24,40 @@ typedef struct SStateKey { int64_t opNum; } SStateKey; +typedef struct SStateSessionKey { + SSessionKey key; + int64_t opNum; +} SStateSessionKey; + +static inline int sessionKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) { + if (pWin1->groupId > pWin2->groupId) { + return 1; + } else if (pWin1->groupId < pWin2->groupId) { + return -1; + } + + if (pWin1->win.skey > pWin2->win.ekey) { + return 1; + } else if (pWin1->win.ekey < pWin2->win.skey) { + return -1; + } + + return 0; +} + +static inline int stateSessionKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { + SStateSessionKey* pWin1 = (SStateSessionKey*)pKey1; + SStateSessionKey* pWin2 = (SStateSessionKey*)pKey2; + + if (pWin1->opNum > pWin2->opNum) { + return 1; + } else if (pWin1->opNum < pWin2->opNum) { + return -1; + } + + return sessionKeyCmpr(&pWin1->key, &pWin2->key); +} + static inline int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { SStateKey* pWin1 = (SStateKey*)pKey1; SStateKey* pWin2 = (SStateKey*)pKey2; @@ -79,6 +113,11 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int goto _err; } + if (tdbTbOpen("session.state.db", sizeof(SStateSessionKey), -1, stateSessionKeyCmpr, pState->db, + &pState->pSessionStateDb) < 0) { + goto _err; + } + if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->db, &pState->pFuncStateDb) < 0) { goto _err; } @@ -95,6 +134,7 @@ _err: tdbTbClose(pState->pStateDb); tdbTbClose(pState->pFuncStateDb); tdbTbClose(pState->pFillStateDb); + tdbTbClose(pState->pSessionStateDb); tdbClose(pState->db); taosMemoryFree(pState); return NULL; @@ -105,6 +145,7 @@ void streamStateClose(SStreamState* pState) { tdbTbClose(pState->pStateDb); tdbTbClose(pState->pFuncStateDb); tdbTbClose(pState->pFillStateDb); + tdbTbClose(pState->pSessionStateDb); tdbClose(pState->db); taosMemoryFree(pState); @@ -241,11 +282,11 @@ SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) { if (pCur == NULL) return NULL; tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL); - int32_t c; + int32_t c = 0; SStateKey sKey = {.key = *key, .opNum = pState->number}; tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c); if (c != 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } pCur->number = pState->number; @@ -257,7 +298,7 @@ SStreamStateCur* streamStateFillGetCur(SStreamState* pState, const SWinKey* key) if (pCur == NULL) return NULL; tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL); - int32_t c; + int32_t c = 0; tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c); if (c != 0) { streamStateFreeCur(pCur); @@ -348,21 +389,21 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key } pCur->number = pState->number; if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } SStateKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c) < 0) { tdbTbcClose(pCur->pCur); - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } if (c > 0) return pCur; if (tdbTbcMoveToNext(pCur->pCur) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } @@ -375,20 +416,20 @@ SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* return NULL; } if (tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } - int32_t c; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { tdbTbcClose(pCur->pCur); - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } if (c > 0) return pCur; if (tdbTbcMoveToNext(pCur->pCur) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } @@ -401,20 +442,20 @@ SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* return NULL; } if (tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } - int32_t c; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { tdbTbcClose(pCur->pCur); - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } if (c < 0) return pCur; if (tdbTbcMoveToPrev(pCur->pCur) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } @@ -445,3 +486,229 @@ void streamStateFreeCur(SStreamStateCur* pCur) { } void streamFreeVal(void* val) { tdbFree(val); } + +int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + return tdbTbUpsert(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, &pState->txn); +} + +SStreamStateCur* streamStateSessionGetRanomCur(SStreamState* pState, const SSessionKey* key) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) return NULL; + tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL); + + int32_t c = 0; + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c); + if (c != 0) { + streamStateFreeCur(pCur); + return NULL; + } + pCur->number = pState->number; + return pCur; +} + +int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { + SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); + void* tmp = NULL; + if (streamStateSessionGetKVByCur(pCur, key, (const void**)&tmp, pVLen) == 0) { + *pVal = tdbRealloc(NULL, *pVLen); + memcpy(*pVal, tmp, *pVLen); + streamStateFreeCur(pCur); + return 0; + } + streamStateFreeCur(pCur); + return -1; +} + +int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + return tdbTbDelete(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), &pState->txn); +} + +SStreamStateCur* streamStateSessionSeekKeyPrev(SStreamState* pState, const SSessionKey* key) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) { + return NULL; + } + pCur->number = pState->number; + if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + int32_t c = 0; + if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { + tdbTbcClose(pCur->pCur); + streamStateFreeCur(pCur); + return NULL; + } + if (c > 0) return pCur; + + if (tdbTbcMoveToPrev(pCur->pCur) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + return pCur; +} + +SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) { + return NULL; + } + pCur->number = pState->number; + if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + int32_t c = 0; + if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { + tdbTbcClose(pCur->pCur); + streamStateFreeCur(pCur); + return NULL; + } + if (c > 0) return pCur; + + if (tdbTbcMoveToNext(pCur->pCur) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + return pCur; +} + +int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, const void** pVal, int32_t* pVLen) { + if (!pCur) { + return -1; + } + const SStateSessionKey* pKTmp = NULL; + int32_t kLen; + if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { + return -1; + } + if (pKTmp->opNum != pCur->number) { + return -1; + } + if (pKey->groupId != 0 && pKey->groupId != pKTmp->key.groupId) { + return -1; + } + *pKey = pKTmp->key; + return 0; +} + +int32_t streamStateSessionClear(SStreamState* pState) { + SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; + streamStateSessionPut(pState, &key, NULL, 0); + SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pState, &key); + while (1) { + SSessionKey delKey = {0}; + void* buf = NULL; + int32_t size = 0; + int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, buf, &size); + if (code == 0) { + memset(buf, 0, size); + streamStateSessionPut(pState, &delKey, buf, size); + } else { + break; + } + streamStateCurNext(pState, pCur); + } + streamStateFreeCur(pCur); + streamStateSessionDel(pState, &key); + return 0; +} + +SStreamStateCur* streamStateSessionGetCur(SStreamState* pState, const SSessionKey* key) { + SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); + SSessionKey resKey = *key; + while (1) { + streamStateCurPrev(pState, pCur); + SSessionKey tmpKey = *key; + int32_t code = streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); + if (code == TSDB_CODE_SUCCESS && sessionKeyCmpr(key, &tmpKey) == 0) { + resKey = tmpKey; + } else { + break; + } + } + streamStateFreeCur(pCur); + return streamStateSessionGetRanomCur(pState, &resKey); +} + +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { + // todo refactor + SStreamStateCur* pCur = streamStateSessionGetCur(pState, key); + int32_t size = *pVLen; + void* tmp = NULL; + *pVal = tdbRealloc(NULL, size); + memset(*pVal, 0, size); + if (streamStateSessionGetKVByCur(pCur, key, (const void**)&tmp, pVLen) == 0) { + memcpy(*pVal, tmp, *pVLen); + streamStateFreeCur(pCur); + return 0; + } + streamStateFreeCur(pCur); + return 1; +} + +int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { + // todo refactor + int32_t res = TSDB_CODE_SUCCESS; + SSessionKey tmpKey = *key; + int32_t valSize = *pVLen; + void* tmp = tdbRealloc(NULL, valSize); + if (!tmp) { + return -1; + } + + SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); + int32_t code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); + if (code == TSDB_CODE_SUCCESS) { + memcpy(tmp, *pVal, valSize); + *pVal = tmp; + streamStateFreeCur(pCur); + return res; + } + streamStateFreeCur(pCur); + + streamStateSessionPut(pState, key, NULL, 0); + pCur = streamStateSessionGetRanomCur(pState, key); + streamStateCurPrev(pState, pCur); + code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); + if (code == TSDB_CODE_SUCCESS) { + void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); + if (fn(pKeyData, stateKey) == true) { + memcpy(tmp, *pVal, valSize); + goto _end; + } + } + + streamStateFreeCur(pCur); + *key = tmpKey; + pCur = streamStateSessionSeekKeyNext(pState, key); + code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); + if (code == TSDB_CODE_SUCCESS) { + void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); + if (fn(pKeyData, stateKey) == true) { + memcpy(tmp, *pVal, valSize); + goto _end; + } + } + + *key = tmpKey; + res = 1; + memset(tmp, 0, valSize); + +_end: + + *pVal = tmp; + streamStateSessionDel(pState, &tmpKey); + streamStateFreeCur(pCur); + return res; +} diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index d8f4a36261..47611a7712 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -249,12 +249,18 @@ ./test.sh -f tsim/stream/windowClose.sim ./test.sh -f tsim/stream/ignoreExpiredData.sim ./test.sh -f tsim/stream/sliding.sim -#./test.sh -f tsim/stream/partitionbyColumnInterval.sim +./test.sh -f tsim/stream/partitionbyColumnInterval.sim #./test.sh -f tsim/stream/partitionbyColumnSession.sim #./test.sh -f tsim/stream/partitionbyColumnState.sim #./test.sh -f tsim/stream/deleteInterval.sim #./test.sh -f tsim/stream/deleteSession.sim #./test.sh -f tsim/stream/deleteState.sim +#./test.sh -f tsim/stream/fillIntervalDelete0.sim +#./test.sh -f tsim/stream/fillIntervalDelete1.sim +./test.sh -f tsim/stream/fillIntervalLinear.sim +#./test.sh -f tsim/stream/fillIntervalPartitionBy.sim +./test.sh -f tsim/stream/fillIntervalPrevNext.sim +./test.sh -f tsim/stream/fillIntervalValue.sim # ---- transaction ---- ./test.sh -f tsim/trans/lossdata1.sim diff --git a/tests/script/tsim/stream/session0.sim b/tests/script/tsim/stream/session0.sim index 3e0af354d8..afae8ef5da 100644 --- a/tests/script/tsim/stream/session0.sim +++ b/tests/script/tsim/stream/session0.sim @@ -216,12 +216,12 @@ if $data02 != 3.274823935 then goto loop2 endi -if $data03 != 1.800000000 then +if $data03 != 1.500000000 then print ======$data03 return -1 endi -if $data04 != 3.350000000 then +if $data04 != 3.500000000 then print ======$data04 return -1 endi diff --git a/tests/script/tsim/stream/session1.sim b/tests/script/tsim/stream/session1.sim index ab173c5929..ee6eefde26 100644 --- a/tests/script/tsim/stream/session1.sim +++ b/tests/script/tsim/stream/session1.sim @@ -5,15 +5,15 @@ sleep 50 sql connect print =============== create database -sql create database test vgroups 1 -sql select * from information_schema.ins_databases +sql create database test vgroups 1; +sql select * from information_schema.ins_databases; if $rows != 3 then return -1 endi print $data00 $data01 $data02 -sql use test +sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double,id int); diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 877a2877b9..60a50f9560 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -349,7 +349,7 @@ endi if $rows != 3 then print ====loop4=rows=$rows -# goto loop4 + goto loop4 endi # row 0 diff --git a/tests/script/tsim/stream/triggerSession0.sim b/tests/script/tsim/stream/triggerSession0.sim index 2ea689ef78..1bef439884 100644 --- a/tests/script/tsim/stream/triggerSession0.sim +++ b/tests/script/tsim/stream/triggerSession0.sim @@ -13,7 +13,7 @@ endi print $data00 $data01 $data02 -sql use test +sql use test; sql create table t2(ts timestamp, a int, b int , c int, d double); sql create stream streams2 trigger window_close into streamt2 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t2 session(ts, 10s); @@ -58,16 +58,11 @@ endi sql insert into t2 values(1648791233002,1,2,3,1.0); sleep 300 sql select * from streamt2; -if $rows != 1 then +if $rows != 0 then print ======$rows return -1 endi -if $data01 != 6 then - print ======$data01 - return -1 -endi - sql insert into t2 values(1648791253003,1,2,3,1.0); sleep 300 sql select * from streamt2; From 2d1f7354a94b9fd6e447c2bf73f7741d837d31a6 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 18 Oct 2022 19:11:49 +0800 Subject: [PATCH 125/135] fix null value handling --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 6a428af1ed..0ab8fb0550 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5363,7 +5363,7 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { } SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); - colDataAppend(pCol, currentRow, pResItem->data, false); + colDataAppend(pCol, currentRow, pResItem->data, (maxCount == 0) ? true : false); return pResInfo->numOfRes; } From c77bc2b9b906f02e2eba6c7c397f30e397497e85 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Tue, 18 Oct 2022 19:12:35 +0800 Subject: [PATCH 126/135] doc : driver-go support WebSocket-based bulk pulling (#17435) * doc : driver-go support WebSocket-based bulk pulling * doc : update English translation * docs: remove preparation section Co-authored-by: Shuduo Sang --- docs/en/14-reference/03-connector/05-go.mdx | 58 ++++++++++++++++++++- docs/zh/08-connector/20-go.mdx | 55 +++++++++++++++++++ 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index f00e635af9..a33b302a92 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -7,7 +7,6 @@ title: TDengine Go Connector import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import Preparition from "./_preparation.mdx" import GoInsert from "../../07-develop/03-insert-data/_go_sql.mdx" import GoInfluxLine from "../../07-develop/03-insert-data/_go_line.mdx" import GoOpenTSDBTelnet from "../../07-develop/03-insert-data/_go_opts_telnet.mdx" @@ -176,6 +175,37 @@ func main() { } ``` + + +_taosRestful_ implements Go's `database/sql/driver` interface via `http client`. You can use the [`database/sql`](https://golang.org/pkg/database/sql/) interface by simply introducing the driver (driver-go minimum version 3.0.2). + +Use `taosWS` as `driverName` and use a correct [DSN](#DSN) as `dataSourceName` with the following parameters supported by the DSN. + +* `writeTimeout` The timeout to send data via WebSocket. +* `readTimeout` The timeout to receive response data via WebSocket. + +For example: + +```go +package main + +import ( + "database/sql" + "fmt" + + _ "github.com/taosdata/driver-go/v3/taosWS" +) + +func main() { + var taosUri = "root:taosdata@ws(localhost:6041)/" + taos, err := sql.Open("taosWS", taosUri) + if err != nil { + fmt.Println("failed to connect TDengine, err:", err) + return + } +} +``` + ## Usage examples @@ -331,7 +361,7 @@ Creates consumer group. * `func (c *Consumer) Subscribe(topics []string) error` -Subscribes to a topic. +Subscribes to topics. * `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` @@ -409,6 +439,30 @@ Close consumer. Closes the parameter binding. +### Subscribe via WebSocket + +* `func NewConsumer(config *Config) (*Consumer, error)` + + Creates consumer group. + +* `func (c *Consumer) Subscribe(topic []string) error` + + Subscribes to topics. + +* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` + + Polling information. + +* `func (c *Consumer) Commit(messageID uint64) error` + + Commit information. + +* `func (c *Consumer) Close() error` + + Close consumer. + +For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) + ## API Reference Full API see [driver-go documentation](https://pkg.go.dev/github.com/taosdata/driver-go/v3) diff --git a/docs/zh/08-connector/20-go.mdx b/docs/zh/08-connector/20-go.mdx index 515d1b030b..7a6058db3c 100644 --- a/docs/zh/08-connector/20-go.mdx +++ b/docs/zh/08-connector/20-go.mdx @@ -177,6 +177,37 @@ func main() { } ``` + + +_taosWS_ 通过 `WebSocket` 实现了 Go 的 `database/sql/driver` 接口。只需要引入驱动（driver-go 最低版本 3.0.2）就可以使用[`database/sql`](https://golang.org/pkg/database/sql/)的接口。 + +使用 `taosWS` 作为 `driverName` 并且使用一个正确的 [DSN](#DSN) 作为 `dataSourceName`，DSN 支持的参数： + +* `writeTimeout` 通过 WebSocket 发送数据的超时时间。 +* `readTimeout` 通过 WebSocket 接收响应数据的超时时间。 + +示例： + +```go +package main + +import ( + "database/sql" + "fmt" + + _ "github.com/taosdata/driver-go/v3/taosWS" +) + +func main() { + var taosUri = "root:taosdata@ws(localhost:6041)/" + taos, err := sql.Open("taosWS", taosUri) + if err != nil { + fmt.Println("failed to connect TDengine, err:", err) + return + } +} +``` + ## 使用示例 @@ -410,6 +441,30 @@ func main() { 结束参数绑定。 +### 通过 WebSocket 订阅 + +* `func NewConsumer(config *Config) (*Consumer, error)` + + 创建消费者。 + +* `func (c *Consumer) Subscribe(topic []string) error` + + 订阅主题。 + +* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` + + 轮询消息。 + +* `func (c *Consumer) Commit(messageID uint64) error` + + 提交消息。 + +* `func (c *Consumer) Close() error` + + 关闭消费者。 + +完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) + ## API 参考 全部 API 见 [driver-go 文档](https://pkg.go.dev/github.com/taosdata/driver-go/v3) From 4292ef6e7c253cc5b241bbfc5b6fcca9403bb76c Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 18 Oct 2022 12:26:05 +0000 Subject: [PATCH 127/135] test:add md5sum file of ci libtaos --- tests/parallel_test/run_case.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index 58dcb87345..e0b905375a 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -67,9 +67,13 @@ mkdir -p /var/lib/taos cd $CONTAINER_TESTDIR/tests/$exec_dir ulimit -c unlimited +md5sum /usr/lib/libtaos.so.1 +md5sum /home/TDinternal/debug/build/lib/libtaos.so $TIMEOUT_CMD $cmd RET=$? echo "cmd exit code: $RET" +md5sum /usr/lib/libtaos.so.1 +md5sum /home/TDinternal/debug/build/lib/libtaos.so if [ $RET -ne 0 ]; then pwd From 4a96af8037c29296676344ab777f05546af14e32 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 18 Oct 2022 14:04:49 +0000 Subject: [PATCH 128/135] test:add md5sum file of ci libtaos --- tests/parallel_test/run_container.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index bb57f238f0..f58aaaf29d 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -68,12 +68,16 @@ if [ $ent -ne 0 ]; then CONTAINER_TESTDIR=/home/TDinternal/community SIM_DIR=/home/TDinternal/sim REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" + REP_MOUNT_LIB="$INTERNAL_REPDIR/debug/build/lib:/home/TDinternal/debug/build/lib:ro" + else # community edition REPDIR=$WORKDIR/TDengine CONTAINER_TESTDIR=/home/TDengine SIM_DIR=/home/TDengine/sim REP_MOUNT_PARAM="$REPDIR:/home/TDengine" + REP_MOUNT_LIB="$REPDIR/debug/build/lib:/home/TDengine/debug/build/lib:ro" + fi ulimit -c unlimited @@ -103,6 +107,7 @@ coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` docker run \ -v $REP_MOUNT_PARAM \ + -v $REP_MOUNT_LIB \ -v $MOUNT_DIR \ -v ${SOURCEDIR}:/usr/local/src/ \ -v "$TMP_DIR/thread_volume/$thread_no/sim:${SIM_DIR}" \ From 312294bff4bf028c85ff5e92019bee64f406c882 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 19 Oct 2022 08:57:07 +0800 Subject: [PATCH 129/135] feat: taosbenchmark support partial col 9284147 (#17456) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index ea4d48a2b1..49e94896cd 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG c64858f + GIT_TAG 9284147 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From cb7ae02a1484190a6e1bd26b4dce355ca9977690 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:12:45 +0800 Subject: [PATCH 130/135] docs: fix interp cn docs --- docs/zh/12-taos-sql/10-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index e7ed74631d..ba3b17eae2 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -865,7 +865,7 @@ INTERP(expr) - INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。 - INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 - INTERP 的输出时间范围根据 RANGE(timestamp1,timestamp2)字段来指定，需满足 timestamp1<=timestamp2。其中 timestamp1（必选值）为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2（必选值）为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。 -- INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（EVERY 值）进行插值。如果没有指定 EVERY，则默认窗口大小为无穷大，即从 timestamp1 开始只有一个窗口。 +- INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（EVERY 值）进行插值。 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 - INTERP 只能在一个时间序列内进行插值，因此当作用于超级表时必须跟 partition by tbname 一起使用。 - INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.1.4版本以后支持)。 From 72a49b1e9daf25f30b7242e24d89c014b43fd079 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:16:42 +0800 Subject: [PATCH 131/135] Update 10-function.md --- docs/zh/12-taos-sql/10-function.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index ba3b17eae2..a2ff7580ff 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -864,6 +864,7 @@ INTERP(expr) - INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。 - INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 +- INTERP 需要同时与RANGE，EVERY和FILL关键字一起使用。 - INTERP 的输出时间范围根据 RANGE(timestamp1,timestamp2)字段来指定，需满足 timestamp1<=timestamp2。其中 timestamp1（必选值）为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2（必选值）为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。 - INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（EVERY 值）进行插值。 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 From f912c632027f63fcc30bea56a554be6b5fa26999 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:16:42 +0800 Subject: [PATCH 132/135] Update 10-function.md --- docs/en/12-taos-sql/10-function.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 48bc5e743e..6145d8c00c 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -863,6 +863,7 @@ INTERP(expr) - `INTERP` is used to get the value that matches the specified time slice from a column. If no such value exists an interpolation value will be returned based on `FILL` parameter. - The input data of `INTERP` is the value of the specified column and a `where` clause can be used to filter the original data. If no `where` condition is specified then all original data is the input. +- `INTERP` must be used along with `RANGE`, `EVERY`, `FILL` keywords. - The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1<=timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified. - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. The parameter `EVERY` must be an integer, with no quotes, with a time unit of: b(nanosecond), u(microsecond), a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds. - Interpolation is performed based on `FILL` parameter. From 26fe97e93fe70bde340c6d5ccaafa76100156504 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao <36554565+glzhao89@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:25:21 +0800 Subject: [PATCH 133/135] Update 10-function.md --- docs/zh/12-taos-sql/10-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index a2ff7580ff..68c7fe6f30 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -864,7 +864,7 @@ INTERP(expr) - INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。 - INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 -- INTERP 需要同时与RANGE，EVERY和FILL关键字一起使用。 +- INTERP 需要同时与 RANGE，EVERY 和 FILL 关键字一起使用。 - INTERP 的输出时间范围根据 RANGE(timestamp1,timestamp2)字段来指定，需满足 timestamp1<=timestamp2。其中 timestamp1（必选值）为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2（必选值）为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。 - INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（EVERY 值）进行插值。 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 From 6780f4cebf2e36cece4f44b7b059e4037a3b9094 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Oct 2022 10:39:56 +0800 Subject: [PATCH 134/135] test: comment out unstable case --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 81625d55e5..3e0e3ff7cf 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -313,7 +313,7 @@ # --- sma ./test.sh -f tsim/sma/drop_sma.sim -./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim +# ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim # temp disable ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim From 05bf2bc4fae9ad04833dcefc96a7d1a614c884b9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 19 Oct 2022 10:48:51 +0800 Subject: [PATCH 135/135] fix: comment out unstable case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 4c0ef5a527..cc5836cbfd 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -232,7 +232,7 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 +# python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 5 -M 3