From ab604310d3e08ac3f5d34914111b4fa8349e970d Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 13 Oct 2022 17:44:48 +0800 Subject: [PATCH 01/63] test:add test case for alter buffer and pages --- tests/system-test/1-insert/alter_database.py | 56 ++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tests/system-test/1-insert/alter_database.py diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py new file mode 100644 index 0000000000..fa23f9ef4f --- /dev/null +++ b/tests/system-test/1-insert/alter_database.py @@ -0,0 +1,56 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.buffer_boundary = [3,4097,8193,12289,16384] + self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96] + # pages_boundary >= 64 + self.pages_boundary = [64,128,512] + self.pages_error = [256,self.pages_boundary[0]-1] + def alter_buffer(self): + tdSql.execute('create database db') + for buffer in self.buffer_boundary: + tdSql.execute(f'alter database db buffer {buffer}') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][8],buffer) + tdSql.execute('drop database db') + tdSql.execute('create database db vgroups 10') + for buffer in self.buffer_error: + tdSql.error(f'alter database db buffer {buffer}') + tdSql.execute('drop database db') + + def alter_pages(self): + tdSql.execute('create database db') + for pages in self.pages_boundary: + tdSql.execute(f'alter database db pages {pages}') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][10],pages) + tdSql.execute('drop database db') + tdSql.execute('create database db') + for pages in self.pages_error: + tdSql.error(f'alter database db pages {pages}') + tdSql.execute('drop database db') + + def run(self): + tdSql.error('create database db1 vgroups 10 buffer 12289') + self.alter_buffer() + self.alter_pages() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 03469918b8c645c9c5e0e1f1be0fe65d39ad6a7c Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 13 Oct 2022 17:51:54 +0800 Subject: [PATCH 02/63] update --- tests/system-test/1-insert/alter_database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index fa23f9ef4f..d3a55ee0a6 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -18,7 +18,7 @@ class TDTestCase: self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96] # pages_boundary >= 64 self.pages_boundary = [64,128,512] - self.pages_error = [256,self.pages_boundary[0]-1] + self.pages_error = [self.pages_boundary[0]-1] def alter_buffer(self): tdSql.execute('create database db') for buffer in self.buffer_boundary: @@ -39,6 +39,8 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][10],pages) tdSql.execute('drop database db') tdSql.execute('create database db') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + self.pages_error.append(tdSql.queryResult[0][10]) for pages in self.pages_error: tdSql.error(f'alter database db pages {pages}') tdSql.execute('drop database db') From bc803c3f07db1f6b01968d83ad9f6eed826b38d6 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 13 Oct 2022 17:54:19 +0800 Subject: [PATCH 03/63] add test case into ci --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 1c6aef6286..32eaa6dc34 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -18,7 +18,7 @@ python3 ./test.py -f 0-others/sysinfo.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py python3 ./test.py -f 0-others/compatibility.py - +python3 ./test.py -f 1-insert/alter_database.py python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py From bb3bb18c010cf3412a103221c0cb60699fae89fc Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 14 Oct 2022 16:24:00 +0800 Subject: [PATCH 04/63] fix: problem when last is used for system table query --- source/libs/parser/src/parTranslater.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index dbe0c97ab6..4739a852c0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2357,7 +2357,8 @@ static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNo } static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSelect) { - if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable)) { + if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) || + TSDB_SYSTEM_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType) { return TSDB_CODE_SUCCESS; } From 36bf800e154193aa450313e54d9fe8b43fe516ad Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 30 Sep 2022 13:57:38 +0800 Subject: [PATCH 05/63] fix: improve fault tolerance on disk space full with error handling enhanced --- source/dnode/mnode/impl/src/mndMain.c | 2 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 27 +++++---- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 10 +++- source/libs/sync/src/syncMain.c | 2 +- source/libs/sync/src/syncRaftLog.c | 2 - source/libs/tdb/src/db/tdbDb.c | 6 +- source/libs/tdb/src/db/tdbPager.c | 75 ++++++++++++++++++------ source/libs/wal/src/walMeta.c | 20 +++++++ source/libs/wal/src/walMgmt.c | 63 +++++++++----------- 10 files changed, 132 insertions(+), 77 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 275fc76e36..c8459fb53b 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -201,7 +201,7 @@ static int32_t mndInitWal(SMnode *pMnode) { pMnode->pWal = walOpen(path, &cfg); if (pMnode->pWal == NULL) { - mError("failed to open wal since %s", terrstr()); + mError("failed to open wal since %s. wal:%s", terrstr(), path); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 07d9b96261..bfb5cad44c 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -89,6 +89,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { data = NULL; if (vnodeEncodeInfo(pInfo, &data) < 0) { + vError("failed to encode json info."); return -1; } @@ -101,7 +102,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { } if (taosWriteFile(pFile, data, strlen(data)) < 0) { - vError("failed to write info file:%s data:%s", fname, terrstr()); + vError("failed to write info file:%s error:%s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -233,15 +234,15 @@ int vnodeCommit(SVnode *pVnode) { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } if (vnodeSaveInfo(dir, &info) < 0) { - ASSERT(0); + vError("vgId:%d, failed to save vnode info since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } walBeginSnapshot(pVnode->pWal, pVnode->state.applied); // preCommit // smaSyncPreCommit(pVnode->pSma); - if (smaAsyncPreCommit(pVnode->pSma) < 0) { - ASSERT(0); + if(smaAsyncPreCommit(pVnode->pSma) < 0){ + vError("vgId:%d, failed to async pre-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -250,44 +251,44 @@ int vnodeCommit(SVnode *pVnode) { // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit meta since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (VND_IS_RSMA(pVnode)) { if (smaAsyncCommit(pVnode->pSma) < 0) { - ASSERT(0); + vError("vgId:%d, failed to async commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA1(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA2(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } else { if (tsdbCommit(pVnode->pTsdb) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } if (tqCommit(pVnode->pTq) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tq since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // walCommit (TODO) // commit info if (vnodeCommitInfo(dir, &info) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit vnode info since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -296,7 +297,7 @@ int vnodeCommit(SVnode *pVnode) { // postCommit // smaSyncPostCommit(pVnode->pSma); if (smaAsyncPostCommit(pVnode->pSma) < 0) { - ASSERT(0); + vError("vgId:%d, failed to async post-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index de3af7cde6..54ecba6d49 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -140,7 +140,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { - vError("vgId:%d, failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode wal since %s. wal:%s", TD_VID(pVnode), tstrerror(terrno), tdir); goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 131327976f..b8047abe69 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -265,10 +265,16 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp _do_commit: vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change - vnodeCommit(pVnode); + if (vnodeCommit(pVnode) < 0) { + vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } // start a new one - vnodeBegin(pVnode); + if (vnodeBegin(pVnode) < 0) { + vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } } return 0; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 378ac743b3..d6da42fd42 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2726,7 +2726,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); if (code != 0) { // del resp mgr, call FpCommitCb - ASSERT(0); + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 496c8419de..1cfe37dc7e 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -234,8 +234,6 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pEntry->index, err, err, errStr, sysErr, sysErrStr); syncNodeErrorLog(pData->pSyncNode, logBuf); - - ASSERT(0); return -1; } pEntry->index = index; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index ea16e80562..a7fc50dc86 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -106,7 +106,7 @@ int32_t tdbBegin(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerBegin(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to begin pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -121,7 +121,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerCommit(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to commit pager. dbName:%s, txnId:%d", pDb->dbName, pTxn->txnId); return -1; } } @@ -136,7 +136,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerAbort(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to abort pager. dbName:%s, txnId:%d", pDb->dbName, pTxn->txnId); return -1; } } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 04711eb6a0..ee4fc90228 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -210,7 +210,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page to journal since %s", tstrerror(terrno)); return -1; } } @@ -226,6 +226,8 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { // Open the journal pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); if (pPager->jfd < 0) { + tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -243,9 +245,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { // sync the journal file ret = tdbOsFSync(pPager->jfd); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } // loop to write the dirty pages to file @@ -255,7 +257,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage = (SPage *)pNode; ret = tdbPagerWritePageToDB(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page to db since %s", tstrerror(terrno)); return -1; } } @@ -277,11 +279,25 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { tRBTreeCreate(&pPager->rbt, pageCmpFn); // sync the db file - tdbOsFSync(pPager->fd); + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } // remove the journal file - tdbOsClose(pPager->jfd); - tdbOsRemove(pPager->jFileName); + if (tdbOsClose(pPager->jfd) < 0) { + tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pPager->inTran = 0; return 0; @@ -297,14 +313,14 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // 0, sync the journal file ret = tdbOsFSync(pPager->jfd); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to fsync jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755); if (jfd == NULL) { - return 0; + return -1; } ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); @@ -516,11 +532,15 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno)); if (ret < 0) { + tdbError("failed to write pgno due to %s. file:%s, pgno:%u", strerror(errno), pPager->jFileName, pgno); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -540,13 +560,15 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - ASSERT(0); + tdbError("failed to lseek due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName, pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -590,23 +612,38 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - ASSERT(0); + tdbError("failed to lseek fd due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize); if (ret < 0) { - ASSERT(0); + tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, pPager->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - tdbOsFSync(pPager->fd); + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } tdbOsFree(pageBuf); - tdbOsClose(jfd); - tdbOsRemove(pPager->jFileName); + if (tdbOsClose(jfd) < 0) { + tdbError("failed to close jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } return 0; } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index e49a963191..de9c115067 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -492,14 +492,20 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0); cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField; pRoot = cJSON_Parse(bytes); + if (!pRoot) goto _err; pMeta = cJSON_GetObjectItem(pRoot, "meta"); + if (!pMeta) goto _err; pField = cJSON_GetObjectItem(pMeta, "firstVer"); + if (!pField) goto _err; pWal->vers.firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "snapshotVer"); + if (!pField) goto _err; pWal->vers.snapshotVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "commitVer"); + if (!pField) goto _err; pWal->vers.commitVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "lastVer"); + if (!pField) goto _err; pWal->vers.lastVer = atoll(cJSON_GetStringValue(pField)); pFiles = cJSON_GetObjectItem(pRoot, "files"); @@ -510,16 +516,22 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { SWalFileInfo* pData = pArray->pData; for (int i = 0; i < sz; i++) { cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + if (!pInfoJson) goto _err; SWalFileInfo* pInfo = &pData[i]; pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); + if (!pField) goto _err; pInfo->firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "lastVer"); + if (!pField) goto _err; pInfo->lastVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "createTs"); + if (!pField) goto _err; pInfo->createTs = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "closeTs"); + if (!pField) goto _err; pInfo->closeTs = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "fileSize"); + if (!pField) goto _err; pInfo->fileSize = atoll(cJSON_GetStringValue(pField)); } taosArraySetSize(pArray, sz); @@ -527,6 +539,10 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { pWal->writeCur = sz - 1; cJSON_Delete(pRoot); return 0; + +_err: + cJSON_Delete(pRoot); + return -1; } static int walFindCurMetaVer(SWal* pWal) { @@ -626,6 +642,10 @@ int walLoadMeta(SWal* pWal) { } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); + if (code < 0) { + wError("failed to deserialize wal meta. file:%s", fnameStr); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + } taosCloseFile(&pFile); taosMemoryFree(buf); return code; diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 7974f3e32e..96f9522c9f 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -81,6 +81,12 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } + if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pWal); + return NULL; + } + // set config memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); @@ -98,15 +104,14 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { tstrncpy(pWal->path, path, sizeof(pWal->path)); if (taosMkDir(pWal->path) != 0) { wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - taosMemoryFree(pWal); - return NULL; + goto _err; } // init ref pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (pWal->pRefHash == NULL) { - taosMemoryFree(pWal); - return NULL; + wError("failed to init hash since %s", tstrerror(terrno)); + goto _err; } // open meta @@ -117,9 +122,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - taosHashCleanup(pWal->pRefHash); - taosMemoryFree(pWal); - return NULL; + goto _err; } // init status @@ -131,46 +134,36 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeHead.head.protoVer = WAL_PROTO_VER; pWal->writeHead.magic = WAL_MAGIC; - if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { - taosArrayDestroy(pWal->fileInfoSet); - taosHashCleanup(pWal->pRefHash); - taosMemoryFree(pWal); - return NULL; - } - - pWal->refId = taosAddRef(tsWal.refSetId, pWal); - if (pWal->refId < 0) { - taosHashCleanup(pWal->pRefHash); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - taosMemoryFree(pWal); - return NULL; - } - - walLoadMeta(pWal); + (void)walLoadMeta(pWal); if (walCheckAndRepairMeta(pWal) < 0) { wError("vgId:%d cannot open wal since repair meta file failed", pWal->cfg.vgId); - taosHashCleanup(pWal->pRefHash); - taosRemoveRef(tsWal.refSetId, pWal->refId); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - return NULL; + goto _err; } if (walCheckAndRepairIdx(pWal) < 0) { wError("vgId:%d cannot open wal since repair idx file failed", pWal->cfg.vgId); - taosHashCleanup(pWal->pRefHash); - taosRemoveRef(tsWal.refSetId, pWal->refId); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - return NULL; + goto _err; + } + + // add ref + pWal->refId = taosAddRef(tsWal.refSetId, pWal); + if (pWal->refId < 0) { + wError("failed to add ref for Wal since %s", tstrerror(terrno)); + goto _err; } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); - return pWal; + +_err: + taosArrayDestroy(pWal->fileInfoSet); + taosHashCleanup(pWal->pRefHash); + taosThreadMutexDestroy(&pWal->mutex); + taosMemoryFree(pWal); + pWal = NULL; + return NULL; } int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { From 76865258cf4b82c8ffcc04293b5aa3940ed55d5a Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 30 Sep 2022 15:49:23 +0800 Subject: [PATCH 06/63] enh: protect WAL meta by flushing to a tmpfile at first and then renaming --- source/libs/tdb/src/db/tdbDb.c | 4 +-- source/libs/wal/src/walMeta.c | 56 ++++++++++++++++++++++++++++------ 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index a7fc50dc86..6c01348bc2 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -121,7 +121,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerCommit(pPager, pTxn); if (ret < 0) { - tdbError("failed to commit pager. dbName:%s, txnId:%d", pDb->dbName, pTxn->txnId); + tdbError("failed to commit pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -136,7 +136,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerAbort(pPager, pTxn); if (ret < 0) { - tdbError("failed to abort pager. dbName:%s, txnId:%d", pDb->dbName, pTxn->txnId); + tdbError("failed to abort pager since %s. dbName:%s, txnId:%d", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index de9c115067..597fd0000b 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -35,8 +35,12 @@ int64_t FORCE_INLINE walGetCommittedVer(SWal* pWal) { return pWal->vers.commitVe int64_t FORCE_INLINE walGetAppliedVer(SWal* pWal) { return pWal->vers.appliedVer; } -static FORCE_INLINE void walBuildMetaName(SWal* pWal, int metaVer, char* buf) { - sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); +static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { + return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); +} + +static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { + return sprintf(buf, "%s/meta-ver.tmp", pWal->path); } static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { @@ -578,22 +582,51 @@ static int walFindCurMetaVer(SWal* pWal) { int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; - walBuildMetaName(pWal, metaVer + 1, fnameStr); - TdFilePtr pMetaFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE); + char tmpFnameStr[WAL_FILE_LEN]; + int n; + + // flush to a tmpfile + n = walBuildTmpMetaName(pWal, tmpFnameStr); + ASSERT(n < sizeof(tmpFnameStr) && "Buffer overflow of file name"); + + TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pMetaFile == NULL) { + wError("failed to open file due to %s. file:%s", strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); if (len != taosWriteFile(pMetaFile, serialized, len)) { // TODO:clean file - - taosCloseFile(&pMetaFile); - taosRemoveFile(fnameStr); - return -1; + wError("failed to write file due to %s. file:%s", strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosFsyncFile(pMetaFile) < 0) { + wError("failed to sync file due to %s. file:%s", strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosCloseFile(&pMetaFile) < 0) { + wError("failed to close file due to %s. file:%s", strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // rename it + n = walBuildMetaName(pWal, metaVer + 1, fnameStr); + ASSERT(n < sizeof(fnameStr) && "Buffer overflow of file name"); + + if (taosRenameFile(tmpFnameStr, fnameStr) < 0) { + wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } - taosCloseFile(&pMetaFile); // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); @@ -601,6 +634,11 @@ int walSaveMeta(SWal* pWal) { } taosMemoryFree(serialized); return 0; + +_err: + taosCloseFile(&pMetaFile); + taosMemoryFree(serialized); + return -1; } int walLoadMeta(SWal* pWal) { From 9055f63742231dc87284e36e0de60867b9d982f7 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 30 Sep 2022 18:54:49 +0800 Subject: [PATCH 07/63] fix: ftruncate WAL log after the last valid WAL record properly in walScanLogGetLastVer --- source/libs/wal/src/walMeta.c | 79 +++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 597fd0000b..797393c47e 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -45,12 +45,11 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - ASSERT(sz > 0); -#if 0 - for (int i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); + if (sz <= 0) { + wError("No WAL log file found."); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; } -#endif SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1); char fnameStr[WAL_FILE_LEN]; @@ -61,7 +60,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize); pLastFileInfo->fileSize = fileSize; - TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -69,28 +68,30 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { uint64_t magic = WAL_MAGIC; - char* buf = taosMemoryMalloc(readSize + 5); + char* buf = taosMemoryMalloc(readSize + sizeof(uint64_t)); if (buf == NULL) { - taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; - return -1; + goto _err; } int64_t offset; offset = taosLSeekFile(pFile, -readSize, SEEK_END); if (readSize != taosReadFile(pFile, buf, readSize)) { - taosMemoryFree(buf); - taosCloseFile(&pFile); terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + goto _err; } + int64_t walCkHeadSz = sizeof(SWalCkHead); char* found = NULL; while (1) { char* haystack = buf; - char* candidate; + char* candidate = NULL; while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { // read and validate + int64_t len = readSize - (candidate - buf); + if (len < walCkHeadSz) { + break; + } SWalCkHead* logContent = (SWalCkHead*)candidate; if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { found = candidate; @@ -98,35 +99,26 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { haystack = candidate + 1; } if (found || offset == 0) break; - offset = TMIN(0, offset - readSize + sizeof(uint64_t)); - int64_t offset2 = taosLSeekFile(pFile, offset, SEEK_SET); - ASSERT(offset == offset2); - if (readSize != taosReadFile(pFile, buf, readSize)) { - taosMemoryFree(buf); - taosCloseFile(&pFile); + + // go backwards, i.e. by at most one WAL scan buf size + offset = TMAX(0, offset - readSize + walCkHeadSz); + int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); + if (ret < 0) { + wError("failed to lseek file due to %s. offset:%lld", strerror(errno), offset); terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + goto _err; } -#if 0 - if (found == buf) { - SWalCkHead* logContent = (SWalCkHead*)found; - if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { - // file has to be deleted - taosMemoryFree(buf); - taosCloseFile(&pFile); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; - } + ASSERT(offset == ret); + if (readSize != taosReadFile(pFile, buf, readSize)) { + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } -#endif } if (found == NULL) { - // file corrupted, no complete log - // TODO delete and search in previous files - /*ASSERT(0);*/ + wError("WAL log file corrupted: no valid WAL record found. file: %s", fnameStr); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; + goto _err; } // truncate file @@ -137,15 +129,28 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { if (lastEntryEndOffset != fileSize) { wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, fileSize); - taosFtruncateFile(pFile, lastEntryEndOffset); + if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) { + wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (taosFsyncFile(pFile) < 0) { + wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset; pWal->totSize -= (fileSize - lastEntryEndOffset); } taosCloseFile(&pFile); taosMemoryFree(buf); - return retVer; + +_err: + taosCloseFile(&pFile); + taosMemoryFree(buf); + return -1; } int walCheckAndRepairMeta(SWal* pWal) { From 715f53a856c645b5324575ce2bcabd6401f6fe30 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 30 Sep 2022 22:57:51 +0800 Subject: [PATCH 08/63] fix: eliminate asserts on errors of appending or getting log entry --- source/libs/sync/src/syncAppendEntries.c | 4 ++++ source/libs/sync/src/syncCommit.c | 5 ++++- source/libs/sync/src/syncMain.c | 7 +++++-- source/libs/sync/src/syncRaftLog.c | 18 +++++++++++++----- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index e000ba8bf8..4a35a15d3e 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -579,6 +579,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -710,6 +711,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -859,6 +861,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -974,6 +977,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index b604d25816..e8ad22a10d 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -76,7 +76,10 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); } else { pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); - ASSERT(pEntry != NULL); + if (pEntry == NULL) { + sError("failed to get entry since %s. index:%lld", tstrerror(terrno), index); + return; + } } // cannot commit, even if quorum agree. need check term! if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index d6da42fd42..ad0390ac7c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2650,7 +2650,10 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { if (ths->state == TAOS_SYNC_STATE_LEADER) { int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); - ASSERT(code == 0); + if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); + return -1; + } syncNodeReplicate(ths, false); } @@ -2790,8 +2793,8 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); // del resp mgr, call FpCommitCb - ASSERT(0); return -1; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 1cfe37dc7e..c3dad104d1 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -412,13 +412,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { return 0; } -SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { +SSyncRaftEntry* logStoreGetEntryWithoutLock(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { - taosThreadMutexLock(&(pData->mutex)); - // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); SWalReader* pWalHandle = pData->pWalHandle; ASSERT(pWalHandle != NULL); @@ -442,7 +440,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { } } while (0); - ASSERT(0); + sError("failed to read ver since %s. index:%lld", tstrerror(terrno), index); + return NULL; } SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); @@ -463,7 +462,6 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { terrno = saveErr; */ - taosThreadMutexUnlock(&(pData->mutex)); return pEntry; } else { @@ -471,6 +469,16 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { } } +SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { + SSyncLogStoreData* pData = pLogStore->data; + SSyncRaftEntry *pEntry = NULL; + + taosThreadMutexLock(&pData->mutex); + pEntry = logStoreGetEntryWithoutLock(pLogStore, index); + taosThreadMutexUnlock(&pData->mutex); + return pEntry; +} + int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; From 56b32a77a4428b7e2752a83fe2d53ec197e418c0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 8 Oct 2022 15:10:02 +0800 Subject: [PATCH 09/63] fix: allow to restart on disk space full, e.g. in dmCheckDiskSpace --- include/os/osEnv.h | 8 +++++++- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 24 ++++++------------------ source/os/src/osEnv.c | 12 +++++++++--- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 293d9d17f8..d4e94d6173 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -49,9 +49,15 @@ extern SDiskSpace tsTempSpace; void osDefaultInit(); void osUpdate(); void osCleanup(); + bool osLogSpaceAvailable(); bool osDataSpaceAvailable(); bool osTempSpaceAvailable(); + +bool osLogSpaceSufficient(); +bool osDataSpaceSufficient(); +bool osTempSpaceSufficient(); + void osSetTimezone(const char *timezone); void osSetSystemLocale(const char *inLocale, const char *inCharSet); @@ -59,4 +65,4 @@ void osSetSystemLocale(const char *inLocale, const char *inCharSet); } #endif -#endif /*_TD_OS_ENV_H_*/ \ No newline at end of file +#endif /*_TD_OS_ENV_H_*/ diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index d07ec7abb0..076826ebc2 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -51,26 +51,14 @@ static int32_t dmInitMonitor() { static bool dmCheckDiskSpace() { osUpdate(); - if (!osDataSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least , quit", - (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osDataSpaceSufficient()) { + dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); } - if (!osLogSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least, quit", - (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osLogSpaceSufficient()) { + dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least", (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); } - if (!osTempSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least, quit", - (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + if (!osTempSpaceSufficient()) { + dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); } return true; } diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index f0442c6fd1..616ab7875d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -87,11 +87,17 @@ void osUpdate() { void osCleanup() {} -bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } +bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; } -bool osDataSpaceAvailable() { return tsDataSpace.reserved <= tsDataSpace.size.avail; } +bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; } -bool osTempSpaceAvailable() { return tsTempSpace.reserved <= tsTempSpace.size.avail; } +bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; } + +bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; } + +bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; } + +bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; } void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, tsTimezoneStr, &tsDaylight, &tsTimezone); } From df18ddbbb52f3b7436f2ff2b0debf862233214f1 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 8 Oct 2022 15:16:45 +0800 Subject: [PATCH 10/63] enh: remove WAL log file corrupted completely --- source/libs/wal/src/walMeta.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 797393c47e..be74b6bfc9 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -118,6 +118,8 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { if (found == NULL) { wError("WAL log file corrupted: no valid WAL record found. file: %s", fnameStr); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + taosRemoveFile(fnameStr); + wWarn("vgId:%d, remove corrupted WAL log file: %s", pWal->cfg.vgId, fnameStr); goto _err; } From 63b13db10f88597ce94f426eec9e22ffe0c372bd Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 8 Oct 2022 20:38:49 +0800 Subject: [PATCH 11/63] fix: eliminate asserts on getting and committing log entries --- source/libs/sync/src/syncCommit.c | 4 +++- source/libs/sync/src/syncMain.c | 5 ++++- source/libs/wal/src/walMeta.c | 1 + source/libs/wal/src/walRead.c | 6 +++++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index e8ad22a10d..511113352e 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -130,7 +130,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { // execute fsm if (pSyncNode->pFsm != NULL) { int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state); - ASSERT(code == 0); + if (code != 0) { + wError("failed to commit sync node since %s", tstrerror(terrno)); + } } } } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ad0390ac7c..bfd933e163 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -3046,7 +3046,10 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); } else { code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); - ASSERT(code == 0); + if (code != 0) { + sError("failed to get log entry since %s. index:%lld", tstrerror(terrno), i); + return -1; + } ASSERT(pEntry != NULL); } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index be74b6bfc9..7cc92c46e9 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -62,6 +62,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); if (pFile == NULL) { + wError("failed to open file due to %s. file:%s", strerror(errno), fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index c4f1a81eeb..248ab91fab 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -181,7 +181,11 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(pRet != NULL); + if (pRet == NULL) { + wError("failed to find WAL log file with ver:%lld", ver); + terrno = TSDB_CODE_WAL_INVALID_VER; + return -1; + } if (pReader->curFileFirstVer != pRet->firstVer) { // error code was set inner if (walReadChangeFile(pReader, pRet->firstVer) < 0) { From 948fa60ef0532fa94d8ff5de54e78a33ddcf2bcb Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 9 Oct 2022 11:52:06 +0800 Subject: [PATCH 12/63] fix: check the length of remaining content for computing CRC32 chksum of WAL record body --- source/libs/wal/src/walMeta.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 7cc92c46e9..7bf707a8ef 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -94,8 +94,10 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { break; } SWalCkHead* logContent = (SWalCkHead*)candidate; - if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { - found = candidate; + if (walValidHeadCksum(logContent) == 0 && + walCkHeadSz + logContent->head.bodyLen <= len && + walValidBodyCksum(logContent) == 0) { + found = candidate; } haystack = candidate + 1; } From c693384ec4c794087d3d7f1f603f9c72cdf1f3ed Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 9 Oct 2022 11:58:28 +0800 Subject: [PATCH 13/63] enh: replace asserts with error logs in walReadVer --- source/libs/wal/src/walRead.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 248ab91fab..1ac15e4bb8 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -476,7 +476,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - ASSERT(0); + wError("vgId:%d, failed to read WAL record head from log file since %s", pReader->pWal->cfg.vgId, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -509,6 +509,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } + wError("vgId:%d, failed to read WAL record body from log file since %s", pReader->pWal->cfg.vgId, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } From 3af05070cf9f1b82c6da496d858e9e9e952c2efe Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 9 Oct 2022 14:11:17 +0800 Subject: [PATCH 14/63] fix: assert on availability of inUse SVBufPool for vnode --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 3 +++ source/dnode/vnode/src/vnd/vnodeBufPool.c | 2 ++ 2 files changed, 5 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index a6628463f8..6679abd247 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -160,6 +160,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid goto _err; } + ASSERT(pPool != NULL); // do delete SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData)); if (pDelData == NULL) { @@ -353,6 +354,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; + ASSERT(pPool != NULL); pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); if (pTbData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -492,6 +494,7 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN // node level = tsdbMemSkipListRandLevel(&pTbData->sl); + ASSERT(pPool != NULL); pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow)); if (pNode == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 3c0a041197..d21cd957a9 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -112,6 +112,8 @@ void vnodeBufPoolReset(SVBufPool *pPool) { void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { SVBufPoolNode *pNode; void *p = NULL; + ASSERT(pPool != NULL); + taosThreadSpinLock(&pPool->lock); if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { // allocate from the anchor node From 7fb827debdea2ad8addb4de84fc3c321cc3d3246 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 9 Oct 2022 20:39:14 +0800 Subject: [PATCH 15/63] fix: refactor walScanLogGetLastVer, walCheckAndRepairMeta, and walCheckAndRepairIdx. * search for the last entry in the contiguous range of valid WAL from the last pos fsynced as firstTrial * find the last entry before the last pos fsynced backwards as second trial * reserve sufficient space for computing CRC32 checksum, esp. of WAL record body * rebuild meta info to resolve potential misalignment between lists of meta and actual log files * retract commitIndex and appliedIndex to lastLogIndex if needed * put an upper size limit on possibly corrupted WAL range to be recovered --- include/libs/wal/wal.h | 1 + include/util/taoserror.h | 1 + source/client/src/clientMain.c | 9 +- source/libs/wal/src/walMeta.c | 671 ++++++++++++++++++++------------- source/libs/wal/src/walMgmt.c | 8 +- source/libs/wal/src/walRead.c | 6 +- source/libs/wal/src/walSeek.c | 10 + source/libs/wal/src/walWrite.c | 5 + source/util/src/terror.c | 3 +- 9 files changed, 442 insertions(+), 272 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index de31a970df..adf244e32a 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -43,6 +43,7 @@ extern "C" { #define WAL_FILE_LEN (WAL_PATH_LEN + 32) #define WAL_MAGIC 0xFAFBFCFDF4F3F2F1ULL #define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3) +#define WAL_RECOV_SIZE_LIMIT (100 * WAL_SCAN_BUF_SIZE) typedef enum { TAOS_WAL_WRITE = 1, diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 6bc0e0e7dd..bbddb539c6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -450,6 +450,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) +#define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006) // tfs #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 6f8cef7c0d..fa0a4ac743 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -146,7 +146,7 @@ void taos_close(TAOS *taos) { int taos_errno(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return terrno; } @@ -154,13 +154,12 @@ int taos_errno(TAOS_RES *res) { return 0; } - return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL - : ((SRequestObj *)res)->code; + return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return (const char *)tstrerror(terrno); } @@ -172,7 +171,7 @@ const char *taos_errstr(TAOS_RES *res) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { - return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL) + return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY) : (const char *)tstrerror(pRequest->code); } } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 7bf707a8ef..42291cd264 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -43,110 +43,177 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { return sprintf(buf, "%s/meta-ver.tmp", pWal->path); } -static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { +static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - if (sz <= 0) { - wError("No WAL log file found."); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; - } + terrno = TSDB_CODE_SUCCESS; + ASSERT(fileIdx >= 0 && fileIdx < sz); - SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1); + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); int64_t fileSize = 0; taosStatFile(fnameStr, &fileSize, NULL); - int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize); - pLastFileInfo->fileSize = fileSize; TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); if (pFile == NULL) { - wError("failed to open file due to %s. file:%s", strerror(errno), fnameStr); + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + // ensure size as non-negative + pFileInfo->fileSize = TMAX(0, pFileInfo->fileSize); + uint64_t magic = WAL_MAGIC; + int64_t walCkHeadSz = sizeof(SWalCkHead); + int64_t end = fileSize; + int64_t offset = 0; + int32_t capacity = 0; + int32_t readSize = 0; + char* buf = NULL; + char* found = NULL; + bool firstTrial = pFileInfo->fileSize < fileSize; - char* buf = taosMemoryMalloc(readSize + sizeof(uint64_t)); - if (buf == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; - goto _err; - } - - int64_t offset; - offset = taosLSeekFile(pFile, -readSize, SEEK_END); - if (readSize != taosReadFile(pFile, buf, readSize)) { - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - int64_t walCkHeadSz = sizeof(SWalCkHead); - char* found = NULL; + // search for the valid last WAL entry, e.g. block by block while (1) { - char* haystack = buf; + offset = (firstTrial) ? pFileInfo->fileSize : TMAX(0, end - WAL_SCAN_BUF_SIZE); + ASSERT(offset <= end); + readSize = end - offset; + capacity = readSize + sizeof(magic); + + int64_t limit = WAL_RECOV_SIZE_LIMIT; + if (limit < readSize) { + wError("vgId:%d, possibly corrupted WAL range exceeds size limit (i.e. %" PRId64 " bytes). offset:%" PRId64 + ", end:%" PRId64 ", file:%s", + pWal->cfg.vgId, limit, offset, end, fnameStr); + terrno = TSDB_CODE_WAL_SIZE_LIMIT; + goto _err; + } + + void* ptr = taosMemoryRealloc(buf, capacity); + if (ptr == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + goto _err; + } + buf = ptr; + + int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (readSize != taosReadFile(pFile, buf, readSize)) { + wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + readSize, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + char* candidate = NULL; - while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { - // read and validate + char* haystack = buf; + + while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) { + // validate head int64_t len = readSize - (candidate - buf); if (len < walCkHeadSz) { - break; + break; } SWalCkHead* logContent = (SWalCkHead*)candidate; - if (walValidHeadCksum(logContent) == 0 && - walCkHeadSz + logContent->head.bodyLen <= len && - walValidBodyCksum(logContent) == 0) { - found = candidate; + if (walValidHeadCksum(logContent) != 0) { + wError("vgId:%d, failed to validate checksum of wal entry header. offset:% %" PRId64 ", file:%s", + ((char*)(logContent)-buf), fnameStr); + haystack = candidate + 1; + if (firstTrial) { + break; + } else { + continue; + } } + + // validate body + int64_t size = walCkHeadSz + logContent->head.bodyLen; + if (len < size) { + int64_t extraSize = size - len; + if (capacity < readSize + extraSize + sizeof(magic)) { + capacity += extraSize; + void* ptr = taosMemoryRealloc(buf, capacity); + if (ptr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + buf = ptr; + } + int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), + offset); + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } + if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) { + wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s", + pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } + } + if (walValidBodyCksum(logContent) != 0) { + terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; + wError("vgId:%d, failed to validate checksum of wal entry body. offset:% %" PRId64 ", file:%s", + ((char*)(logContent)-buf), fnameStr); + haystack = candidate + 1; + if (firstTrial) { + break; + } else { + continue; + } + } + + // found one + found = candidate; haystack = candidate + 1; } + if (found || offset == 0) break; - // go backwards, i.e. by at most one WAL scan buf size - offset = TMAX(0, offset - readSize + walCkHeadSz); - int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); - if (ret < 0) { - wError("failed to lseek file due to %s. offset:%lld", strerror(errno), offset); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - ASSERT(offset == ret); - if (readSize != taosReadFile(pFile, buf, readSize)) { - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // go backwards, e.g. by at most one WAL scan buf size + end = offset + walCkHeadSz - 1; + firstTrial = false; } - if (found == NULL) { - wError("WAL log file corrupted: no valid WAL record found. file: %s", fnameStr); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - taosRemoveFile(fnameStr); - wWarn("vgId:%d, remove corrupted WAL log file: %s", pWal->cfg.vgId, fnameStr); - goto _err; + // determine end of last entry + SWalCkHead* lastEntry = (SWalCkHead*)found; + int64_t retVer = -1; + int64_t lastEntryBeginOffset = 0; + int64_t lastEntryEndOffset = 0; + + if (lastEntry == NULL) { + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + } else { + retVer = lastEntry->head.version; + lastEntryBeginOffset = offset + (int64_t)((char*)lastEntry - (char*)buf); + lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; } // truncate file - SWalCkHead* lastEntry = (SWalCkHead*)found; - int64_t retVer = lastEntry->head.version; - int64_t lastEntryBeginOffset = offset + (int64_t)((char*)found - (char*)buf); - int64_t lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; if (lastEntryEndOffset != fileSize) { - wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, + wWarn("vgId:%d, repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, fileSize); if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) { - wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } if (taosFsyncFile(pFile) < 0) { - wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } - ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset; - pWal->totSize -= (fileSize - lastEntryEndOffset); } + pFileInfo->fileSize = lastEntryEndOffset; taosCloseFile(&pFile); taosMemoryFree(buf); @@ -158,13 +225,69 @@ _err: return -1; } +static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) { + int metaFileNum = taosArrayGetSize(metaLogList); + int actualFileNum = taosArrayGetSize(actualLogList); + int j = 0; + + // both of the lists in asc order + for (int i = 0; i < actualFileNum; i++) { + SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i); + while (j < metaFileNum) { + SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j); + ASSERT(pMetaInfo != NULL); + if (pMetaInfo->firstVer < pLogInfo->firstVer) { + j++; + } else if (pMetaInfo->firstVer == pLogInfo->firstVer) { + (*pLogInfo) = *pMetaInfo; + j++; + break; + } else { + break; + } + } + } + + taosArrayClear(metaLogList); + + for (int i = 0; i < actualFileNum; i++) { + SWalFileInfo* pFileInfo = taosArrayGet(actualLogList, i); + taosArrayPush(metaLogList, pFileInfo); + } +} + +void walAlignVersions(SWal* pWal) { + if (pWal->vers.firstVer > pWal->vers.snapshotVer) { + wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + pWal->vers.firstVer, pWal->vers.snapshotVer); + pWal->vers.firstVer = pWal->vers.snapshotVer; + } + if (pWal->vers.lastVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + pWal->vers.lastVer, pWal->vers.snapshotVer); + pWal->vers.lastVer = pWal->vers.snapshotVer; + } + if (pWal->vers.commitVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + pWal->vers.commitVer, pWal->vers.snapshotVer); + pWal->vers.commitVer = pWal->vers.snapshotVer; + } + if (pWal->vers.appliedVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + pWal->vers.appliedVer, pWal->vers.snapshotVer); + pWal->vers.appliedVer = pWal->vers.snapshotVer; + } + + pWal->vers.commitVer = TMIN(pWal->vers.lastVer, pWal->vers.commitVer); + pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer); +} + int walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info const char* logPattern = "^[0-9]+.log$"; const char* idxPattern = "^[0-9]+.idx$"; regex_t logRegPattern; regex_t idxRegPattern; - bool fixed = false; regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); @@ -198,222 +321,238 @@ int walCheckAndRepairMeta(SWal* pWal) { taosArraySort(actualLog, compareWalFileInfo); - int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); - int actualFileNum = taosArrayGetSize(actualLog); + int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); + int actualFileNum = taosArrayGetSize(actualLog); + int64_t firstVerPrev = pWal->vers.firstVer; + int64_t lastVerPrev = pWal->vers.lastVer; + int64_t totSize = 0; + bool updateMeta = (metaFileNum != actualFileNum); -#if 0 - for (int32_t fileNo = actualFileNum - 1; fileNo >= 0; fileNo--) { - SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, fileNo); + // rebuild meta of file info + walRebuildFileInfoSet(pWal->fileInfoSet, actualLog); + taosArrayDestroy(actualLog); + + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + ASSERT(sz == actualFileNum); + + // scan and determine the lastVer + int32_t fileIdx = sz; + + while (--fileIdx >= 0) { char fnameStr[WAL_FILE_LEN]; + int64_t fileSize = 0; + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - int64_t fileSize = 0; - taosStatFile(fnameStr, &fileSize, NULL); - if (fileSize == 0) { + int32_t code = taosStatFile(fnameStr, &fileSize, NULL); + if (code < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("failed to stat file since %s. file:%s", terrstr(), fnameStr); + return -1; + } + + ASSERT(pFileInfo->firstVer >= 0); + + if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) { + totSize += pFileInfo->fileSize; + continue; + } + updateMeta = true; + + int64_t lastVer = walScanLogGetLastVer(pWal, fileIdx); + if (lastVer < 0) { + if (terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { + wError("failed to scan wal last ver since %s", terrstr()); + return -1; + } + ASSERT(pFileInfo->fileSize == 0); + // remove the empty wal log, and its idx taosRemoveFile(fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); taosRemoveFile(fnameStr); - taosArrayPop(pLogInfoArray); - } else { - break; + // remove its meta entry + taosArrayRemove(pWal->fileInfoSet, fileIdx); + continue; } + + // update lastVer + pFileInfo->lastVer = lastVer; + totSize += pFileInfo->fileSize; } - actualFileNum = taosArrayGetSize(pLogInfoArray); -#endif - - { - int32_t i = 0, j = 0; - while (i < actualFileNum && j < metaFileNum) { - SWalFileInfo* pActualFile = taosArrayGet(actualLog, i); - SWalFileInfo* pMetaFile = taosArrayGet(pWal->fileInfoSet, j); - if (pActualFile->firstVer < pMetaFile->firstVer) { - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - i++; - } else if (pActualFile->firstVer > pMetaFile->firstVer) { - taosArrayRemove(pWal->fileInfoSet, j); - metaFileNum--; - } else { - i++; - j++; - } - } - if (i == actualFileNum && j == metaFileNum) { - if (j > 0) { - SWalFileInfo* pLastInfo = taosArrayGet(pWal->fileInfoSet, j - 1); - int64_t fsize = 0; - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pLastInfo->firstVer, fNameStr); - taosStatFile(fNameStr, &fsize, NULL); - if (pLastInfo->fileSize != fsize) { - fixed = true; - pLastInfo->fileSize = fsize; - pLastInfo->lastVer = walScanLogGetLastVer(pWal); - } - } - } else { - fixed = true; - while (i < actualFileNum) { - SWalFileInfo* pActualFile = taosArrayGet(actualLog, i); - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pActualFile->firstVer, fNameStr); - taosStatFile(fNameStr, &pActualFile->fileSize, NULL); - - if (pActualFile->fileSize == 0) { - ASSERT(i == actualFileNum - 1); - taosRemoveFile(fNameStr); - - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - break; - } - - if (i < actualFileNum - 1) { - pActualFile->lastVer = ((SWalFileInfo*)taosArrayGet(actualLog, i + 1))->firstVer - 1; - taosArrayPush(pWal->fileInfoSet, pActualFile); - i++; - } else { - pActualFile = taosArrayPush(pWal->fileInfoSet, pActualFile); - pActualFile->lastVer = walScanLogGetLastVer(pWal); - if (pActualFile->lastVer == -1) { - taosRemoveFile(fNameStr); - - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - taosArrayPop(pWal->fileInfoSet); - } - break; - } - } - } - } - -#if 0 - if (metaFileNum > actualFileNum) { - taosArrayPopFrontBatch(pWal->fileInfoSet, metaFileNum - actualFileNum); - } else if (metaFileNum < actualFileNum) { - for (int i = metaFileNum; i < actualFileNum; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(actualLog, i); - taosArrayPush(pWal->fileInfoSet, pFileInfo); - } - } -#endif - - taosArrayDestroy(actualLog); - + // reset vers info and so on actualFileNum = taosArrayGetSize(pWal->fileInfoSet); pWal->writeCur = actualFileNum - 1; - + pWal->totSize = totSize; + pWal->vers.firstVer = -1; + pWal->vers.lastVer = -1; if (actualFileNum > 0) { - int64_t fLastVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur))->lastVer; - if (fLastVer != -1 && pWal->vers.lastVer != fLastVer) { - fixed = true; - pWal->vers.lastVer = fLastVer; - } - int64_t fFirstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; - if (fFirstVer != pWal->vers.firstVer) { - fixed = true; - pWal->vers.firstVer = fFirstVer; - } + pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer; + } + (void)walAlignVersions(pWal); + + // update meta file + if (updateMeta) { + (void)walSaveMeta(pWal); + } + return 0; +} + +int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) { + if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } - if (fixed) { - walSaveMeta(pWal); + if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (walValidHeadCksum(pCkHead) != 0) { + terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; + return -1; } return 0; } -int walCheckAndRepairIdx(SWal* pWal) { +int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - for (int32_t i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); + ASSERT(fileIdx >= 0 && fileIdx < sz); + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); + char fLogNameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); + int64_t fileSize = 0; - char fnameStr[WAL_FILE_LEN]; - walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); - int64_t fsize; - TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); - if (pIdxFile == NULL) { - ASSERT(0); + if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) { + wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + ASSERT(pFileInfo->fileSize > 0 && pFileInfo->firstVer >= 0 && pFileInfo->lastVer >= pFileInfo->firstVer); + if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { + return 0; + } + + // start to repair + int64_t offset = fileSize - fileSize % sizeof(SWalIdxEntry); + TdFilePtr pLogFile = NULL; + TdFilePtr pIdxFile = NULL; + SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer - 1, .offset = -sizeof(SWalCkHead)}; + SWalCkHead ckHead; + memset(&ckHead, 0, sizeof(ckHead)); + ckHead.head.version = idxEntry.ver; + + pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); + if (pIdxFile == NULL) { + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); + if (pLogFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); + goto _err; + } + + // determine the last valid entry end, i.e. offset + while ((offset -= sizeof(SWalIdxEntry)) >= 0) { + if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) { + wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fnameStr, terrstr()); - return -1; + goto _err; } - taosFStatFile(pIdxFile, &fsize, NULL); - if (fsize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { - taosCloseFile(&pIdxFile); + if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (idxEntry.ver > pFileInfo->lastVer) { continue; } - int32_t left = fsize % sizeof(SWalIdxEntry); - int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END); - if (left != 0) { - taosFtruncateFile(pIdxFile, offset); - wWarn("vgId:%d wal truncate file %s to offset %ld since size invalid, file size %ld", pWal->cfg.vgId, fnameStr, - offset, fsize); - } - offset -= sizeof(SWalIdxEntry); - - SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer}; - while (1) { - if (offset < 0) { - taosLSeekFile(pIdxFile, 0, SEEK_SET); - taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - break; - } - taosLSeekFile(pIdxFile, offset, SEEK_SET); - int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if ((idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry) != offset) { - taosFtruncateFile(pIdxFile, offset); - wWarn("vgId:%d wal truncate file %s to offset %ld since entry invalid, entry ver %ld, entry offset %ld", - pWal->cfg.vgId, fnameStr, offset, idxEntry.ver, idxEntry.offset); - offset -= sizeof(SWalIdxEntry); - } else { - break; - } + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { + wWarn("vgId:%d, failed to read log file since %s. file:%s, offset:%" PRId64 ", idx entry ver:%" PRId64 "", + pWal->cfg.vgId, terrstr(), fLogNameStr, idxEntry.offset, idxEntry.ver); + continue; } - if (idxEntry.ver < pFileInfo->lastVer) { - char fLogNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); - TdFilePtr pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); - if (pLogFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); - return -1; - } - while (idxEntry.ver < pFileInfo->lastVer) { - if (taosLSeekFile(pLogFile, idxEntry.offset, SEEK_SET) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot seek file %s at %ld, since %s", pWal->cfg.vgId, fLogNameStr, idxEntry.offset, - terrstr()); - return -1; - } - SWalCkHead ckHead; - taosReadFile(pLogFile, &ckHead, sizeof(SWalCkHead)); - if (idxEntry.ver != ckHead.head.version) { - // todo truncate this idx also - taosCloseFile(&pLogFile); - wError("vgId:%d, invalid repair case, log seek to %ld to find ver %ld, actual ver %ld", pWal->cfg.vgId, - idxEntry.offset, idxEntry.ver, ckHead.head.version); - return -1; - } - idxEntry.ver = ckHead.head.version + 1; - idxEntry.offset = idxEntry.offset + sizeof(SWalCkHead) + ckHead.head.bodyLen; - wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); - taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - } - taosCloseFile(&pLogFile); + if (idxEntry.ver == ckHead.head.version) { + break; + } + } + offset += sizeof(SWalIdxEntry); + + // ftruncate idx file + if (offset < fileSize) { + if (taosFtruncateFile(pIdxFile, offset) < 0) { + wError("vgId:%d, failed to ftruncate file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, + strerror(errno), offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } + + // rebuild idx file + if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) { + wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + while (idxEntry.ver < pFileInfo->lastVer) { + ASSERT(idxEntry.ver == ckHead.head.version); + + idxEntry.ver += 1; + idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen; + + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { + wError("vgId:%d, failed to read wal log head since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), + idxEntry.offset, fLogNameStr); + goto _err; + } + wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); + if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) { + wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); + goto _err; + } + } + + if (taosFsyncFile(pIdxFile) < 0) { + wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); + goto _err; + } + + (void)taosCloseFile(&pLogFile); + (void)taosCloseFile(&pIdxFile); + return 0; + +_err: + (void)taosCloseFile(&pLogFile); + (void)taosCloseFile(&pIdxFile); + return -1; +} + +int walCheckAndRepairIdx(SWal* pWal) { + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + int32_t fileIdx = sz; + while (--fileIdx >= 0) { + if (walCheckAndRepairIdxFile(pWal, fileIdx) < 0) { + wError("vgId:%d, failed to repair idx file since %s. fileIdx:%d", pWal->cfg.vgId, terrstr(), fileIdx); + return -1; } - taosCloseFile(&pIdxFile); } return 0; } @@ -529,7 +668,7 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { taosArrayEnsureCap(pArray, sz); SWalFileInfo* pData = pArray->pData; for (int i = 0; i < sz; i++) { - cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); if (!pInfoJson) goto _err; SWalFileInfo* pInfo = &pData[i]; pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); @@ -593,7 +732,20 @@ int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; char tmpFnameStr[WAL_FILE_LEN]; - int n; + int n; + + // fsync the idx and log file at first to ensure validity of meta + if (taosFsyncFile(pWal->pIdxFile) < 0) { + wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (taosFsyncFile(pWal->pLogFile) < 0) { + wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } // flush to a tmpfile n = walBuildTmpMetaName(pWal, tmpFnameStr); @@ -601,7 +753,7 @@ int walSaveMeta(SWal* pWal) { TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pMetaFile == NULL) { - wError("failed to open file due to %s. file:%s", strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -609,20 +761,19 @@ int walSaveMeta(SWal* pWal) { char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); if (len != taosWriteFile(pMetaFile, serialized, len)) { - // TODO:clean file - wError("failed to write file due to %s. file:%s", strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } if (taosFsyncFile(pMetaFile) < 0) { - wError("failed to sync file due to %s. file:%s", strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } if (taosCloseFile(&pMetaFile) < 0) { - wError("failed to close file due to %s. file:%s", strerror(errno), tmpFnameStr); + wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 96f9522c9f..5972807f50 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -110,8 +110,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // init ref pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (pWal->pRefHash == NULL) { - wError("failed to init hash since %s", tstrerror(terrno)); - goto _err; + wError("failed to init hash since %s", tstrerror(terrno)); + goto _err; } // open meta @@ -149,8 +149,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // add ref pWal->refId = taosAddRef(tsWal.refSetId, pWal); if (pWal->refId < 0) { - wError("failed to add ref for Wal since %s", tstrerror(terrno)); - goto _err; + wError("failed to add ref for Wal since %s", tstrerror(terrno)); + goto _err; } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1ac15e4bb8..cf96c0dae6 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -476,7 +476,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - wError("vgId:%d, failed to read WAL record head from log file since %s", pReader->pWal->cfg.vgId, terrstr()); + wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -509,7 +510,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - wError("vgId:%d, failed to read WAL record body from log file since %s", pReader->pWal->cfg.vgId, terrstr()); + wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 1196914dae..4b75db52b7 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -79,6 +79,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; if (pWal->pLogFile != NULL) { + code = taosFsyncFile(pWal->pLogFile); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } code = taosCloseFile(&pWal->pLogFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -86,6 +91,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { } } if (pWal->pIdxFile != NULL) { + code = taosFsyncFile(pWal->pIdxFile); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } code = taosCloseFile(&pWal->pIdxFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 628b432446..ffadcd04c4 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -540,6 +540,11 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { + wTrace("vgId:%d, fileId:%" PRId64 ".idx, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); + if (taosFsyncFile(pWal->pIdxFile) < 0) { + wError("vgId:%d, file:%" PRId64 ".idx, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), + strerror(errno)); + } wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (taosFsyncFile(pWal->pLogFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), diff --git a/source/util/src/terror.c b/source/util/src/terror.c index eb13a08be4..9a117c6eb4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -447,12 +447,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND, "TQ table schema not f TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no commited offset") // wal -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Wal unexpected generic error") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "WAL unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") // tfs TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") From b6c31ee67d84ef3bb343e1835d5081e010dc399b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 10 Oct 2022 20:17:03 +0800 Subject: [PATCH 16/63] fix: fsync the current wal log and idx files at first to ensure validity of meta in walSaveMeta --- source/libs/wal/src/walMgmt.c | 3 ++- source/libs/wal/src/walWrite.c | 16 +++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 5972807f50..29058cada1 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -134,6 +134,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeHead.head.protoVer = WAL_PROTO_VER; pWal->writeHead.magic = WAL_MAGIC; + // load meta (void)walLoadMeta(pWal); if (walCheckAndRepairMeta(pWal) < 0) { @@ -188,11 +189,11 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { void walClose(SWal *pWal) { taosThreadMutexLock(&pWal->mutex); + (void)walSaveMeta(pWal); taosCloseFile(&pWal->pLogFile); pWal->pLogFile = NULL; taosCloseFile(&pWal->pIdxFile); pWal->pIdxFile = NULL; - walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; taosHashCleanup(pWal->pRefHash); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index ffadcd04c4..e3045cb8e0 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -209,10 +209,12 @@ int32_t walRollback(SWal *pWal, int64_t ver) { taosCloseFile(&pIdxFile); taosCloseFile(&pLogFile); - taosFsyncFile(pWal->pLogFile); - taosFsyncFile(pWal->pIdxFile); - - walSaveMeta(pWal); + code = walSaveMeta(pWal); + if (code < 0) { + wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } // unlock taosThreadMutexUnlock(&pWal->mutex); @@ -384,7 +386,11 @@ int32_t walRollImpl(SWal *pWal) { pWal->lastRollSeq = walGetSeq(); - walSaveMeta(pWal); + code = walSaveMeta(pWal); + if (code < 0) { + wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); + goto END; + } END: return code; From c70d10334cabfb59dc53821e4b7cd1a42ac19513 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 11 Oct 2022 15:26:43 +0800 Subject: [PATCH 17/63] enh: print vgId in error msg within syncNodeCommit --- source/libs/sync/src/syncMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index bfd933e163..cc686ba2fe 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -3047,7 +3047,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, } else { code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); if (code != 0) { - sError("failed to get log entry since %s. index:%lld", tstrerror(terrno), i); + sError("vgId:%d, failed to get log entry since %s. index:%lld", ths->vgId, tstrerror(terrno), i); return -1; } ASSERT(pEntry != NULL); From a4cc3f76d4f8a8698b857e8e45aad8cf085716a1 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 11 Oct 2022 19:58:01 +0800 Subject: [PATCH 18/63] fix: ignore file not exist error of tdbOsRemove --- source/libs/tdb/src/db/tdbPager.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index ee4fc90228..593b8cc589 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -156,6 +156,7 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) { ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); return -1; } From 6f39a36344e5de61af1f673f96547f7c09a9d05b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 11 Oct 2022 20:01:12 +0800 Subject: [PATCH 19/63] fix: replace asserts on failure of tdbPagerWrite with tdbError msg --- source/libs/tdb/src/db/tdbBtree.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a148c1e36b..97e1b3f9cd 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -121,6 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -483,9 +484,8 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN ret = tdbPagerWrite(pPager, pChild); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to write page since %s", terrstr()); + return -1; } // Copy the root page content to the child page @@ -556,8 +556,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pOlds[i]); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } } @@ -583,8 +582,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pParent); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -719,8 +717,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pNews[iNew]); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } } @@ -937,7 +934,7 @@ static int tdbFetchOvflPage(SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt) // mark dirty ret = tdbPagerWrite(pBt->pPager, *ppOfp); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -1942,7 +1939,7 @@ int tdbBtcDelete(SBTC *pBtc) { // drop the cell on the leaf ret = tdbPagerWrite(pPager, pBtc->pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -1964,7 +1961,7 @@ int tdbBtcDelete(SBTC *pBtc) { if (idx < nCells) { ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -2029,7 +2026,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // mark dirty ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } From 12d870ef3b69d8191b11481a1f732c59ed9a332f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 12 Oct 2022 20:48:23 +0800 Subject: [PATCH 20/63] fix: response TSDB_CODE_VND_NOT_SYNCED while not pVnode->inUse in vnodeProcessWriteMsg --- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b8047abe69..5f96fd8807 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -145,6 +145,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t len; int32_t ret; + if (!pVnode->inUse) { + terrno = TSDB_CODE_VND_NOT_SYNCED; + vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); + return -1; + } + vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); From 8f6a81a18cb3d421b07d0326b4d8c0fda4b5f478 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 12 Oct 2022 23:07:14 +0800 Subject: [PATCH 21/63] enh: indicate intentionally restarting taosd on tsdbCommit failure --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 995d7a3155..3db0b4f2b9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1038,7 +1038,9 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; - ASSERT(eno == 0); + ASSERT(eno == 0 && + "tsdbCommit failure" + "Restart taosd"); code = tsdbFSCommit1(pTsdb, &pCommitter->fs); TSDB_CHECK_CODE(code, lino, _exit); From 34ea8996a0dbe4f5e9f2d596dfe0da0b83e9eb33 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Wed, 12 Oct 2022 23:08:41 +0800 Subject: [PATCH 22/63] fix: check disk availability for committing vnode --- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index bfb5cad44c..07c4c32955 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -73,7 +73,7 @@ int vnodeBegin(SVnode *pVnode) { int vnodeShouldCommit(SVnode *pVnode) { if (pVnode->inUse) { - return pVnode->inUse->size > pVnode->inUse->node.size; + return osDataSpaceAvailable() && (pVnode->inUse->size > pVnode->inUse->node.size); } return false; } From e3f50c0d481eab72fae604c4f0d2d00af29acda2 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 16 Oct 2022 11:49:53 +0800 Subject: [PATCH 23/63] fix: align vers.firstVer with vers.snapshotVer+1 in WAL meta info --- source/libs/wal/src/walMeta.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 8808363bb2..279f4dc656 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -257,23 +257,23 @@ static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) { } void walAlignVersions(SWal* pWal) { - if (pWal->vers.firstVer > pWal->vers.snapshotVer) { - wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) { + wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId, pWal->vers.firstVer, pWal->vers.snapshotVer); - pWal->vers.firstVer = pWal->vers.snapshotVer; + pWal->vers.firstVer = pWal->vers.snapshotVer + 1; } if (pWal->vers.lastVer < pWal->vers.snapshotVer) { - wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, pWal->vers.lastVer, pWal->vers.snapshotVer); pWal->vers.lastVer = pWal->vers.snapshotVer; } if (pWal->vers.commitVer < pWal->vers.snapshotVer) { - wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, pWal->vers.commitVer, pWal->vers.snapshotVer); pWal->vers.commitVer = pWal->vers.snapshotVer; } if (pWal->vers.appliedVer < pWal->vers.snapshotVer) { - wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". reset it.", pWal->cfg.vgId, + wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, pWal->vers.appliedVer, pWal->vers.snapshotVer); pWal->vers.appliedVer = pWal->vers.snapshotVer; } @@ -384,7 +384,6 @@ int walCheckAndRepairMeta(SWal* pWal) { actualFileNum = taosArrayGetSize(pWal->fileInfoSet); pWal->writeCur = actualFileNum - 1; pWal->totSize = totSize; - pWal->vers.firstVer = -1; pWal->vers.lastVer = -1; if (actualFileNum > 0) { pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; From ef06a9d265c295e7b9c82be9713ef5fb8df92ae9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 17 Oct 2022 09:58:34 +0800 Subject: [PATCH 24/63] fix more coverity scan problem --- source/dnode/vnode/src/meta/metaTable.c | 7 ++++++- source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 19 +++++++++++-------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 97b0be07c1..1e0e757793 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -51,7 +51,7 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema return -1; } - strcpy(pMetaRsp->tbName, tbName); + strncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN); pMetaRsp->numOfColumns = pSchema->nCols; pMetaRsp->tableType = TSDB_NORMAL_TABLE; pMetaRsp->sversion = pSchema->version; @@ -817,6 +817,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA const void *pData = NULL; int nData = 0; + if (pAlterTbReq->tagName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + // search name index ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index d08b743763..6fd5629592 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -819,7 +819,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(fSet.pHeadF); } } else { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0f93e650c6..3556b47dd3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -41,7 +41,10 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t nReqs; tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); - tStartDecode(&dc); + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } if (tDecodeI32v(&dc, &nReqs) < 0) { code = TSDB_CODE_INVALID_MSG; @@ -167,9 +170,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t ret; if (!pVnode->inUse) { - terrno = TSDB_CODE_VND_NOT_SYNCED; - vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); - return -1; + terrno = TSDB_CODE_VND_NOT_SYNCED; + vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); + return -1; } vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), @@ -293,14 +296,14 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change if (vnodeCommit(pVnode) < 0) { - vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); - goto _err; + vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; } // start a new one if (vnodeBegin(pVnode) < 0) { - vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); - goto _err; + vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; } } From 6f9153c6e3b1de35c1f7f7511e966527fdf736ba Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 10:13:10 +0800 Subject: [PATCH 25/63] fix: error handling --- source/client/src/clientTmq.c | 4 ++-- source/dnode/mnode/impl/src/mndScheduler.c | 2 +- source/libs/wal/src/walWrite.c | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 922cab23c8..73f7c2bcc4 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -966,14 +966,14 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = -1; + tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz); + req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, 256); tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) goto FAIL; - tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz); - for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 511a84290d..1c8ec0a302 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -416,7 +416,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } SStreamTask* pTask = tNewSStreamTask(pStream->uid); - if (pInnerTask == NULL) { + if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index e7161079d9..bb6c3a91ef 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -45,6 +45,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { if (taosRemoveFile(fnameStr) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); return -1; } wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); @@ -53,6 +54,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { if (taosRemoveFile(fnameStr) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); return -1; } wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); From 980fdd6e915d48b15093faf8be4d7ec1a529562e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 10:27:15 +0800 Subject: [PATCH 26/63] fix: error handling --- source/dnode/vnode/src/tq/tq.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e1578b37f1..aaa57d49ed 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1010,16 +1010,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) { qError("stream task input del failed, task id %d", pTask->taskId); + + taosFreeQitem(pRefBlock); continue; } + if (streamSchedExec(pTask) < 0) { qError("stream task launch failed, task id %d", pTask->taskId); continue; } + } else { streamTaskInputFail(pTask); } } + int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { From 837495fa5af3e49ca92a91d39dd0a1386b834f1b Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 17 Oct 2022 10:45:47 +0800 Subject: [PATCH 27/63] test: reopen taosShell.py test case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index cc85b8ddc8..58d8efbed6 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -2,7 +2,7 @@ set -e set -x -#python3 ./test.py -f 0-others/taosShell.py +python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py python3 ./test.py -f 0-others/taosShellNetChk.py python3 ./test.py -f 0-others/telemetry.py From 132681a8bca37652e93c8abd5bad56bb9c0702bf Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 28/63] fix: coverity issues CID: 308488 --- source/common/src/ttime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index d9d4fb274e..bb82863e73 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -795,7 +795,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio // not enough time range if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - while (end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window + while (end < t) { // move forward to the correct time window start += pInterval->sliding; if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { From bcf2cfeed7284e04a8e224431bb718f522e55eb3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 29/63] fix: coverity issues CID: 354713 --- source/libs/scalar/src/filter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 81ee173648..5865370e8e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1709,9 +1709,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) ctx->isrange); if (ctx->isrange) { SFilterRangeNode *r = ctx->rs; + int32_t tlen = 0; while (r) { - char str[256] = {0}; - int32_t tlen = 0; + char str[256] = {0}; if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { strcat(str, "(NULL)"); } else { From 4d0c7732f13a5fd6da82ec0576105678970b868d Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Mon, 17 Oct 2022 11:20:10 +0800 Subject: [PATCH 30/63] test:add test case for create wrong topic --- tests/system-test/7-tmq/create_wrong_topic.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/system-test/7-tmq/create_wrong_topic.py diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py new file mode 100644 index 0000000000..d18cb0260e --- /dev/null +++ b/tests/system-test/7-tmq/create_wrong_topic.py @@ -0,0 +1,77 @@ +import taos +import sys +import time +import socket +import os +import threading +from util.common import * + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.setsql = TDSetSql() + self.rowNum = 10 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + } + self.error_topic = ['avg','count','spread','stddev','sum','hyperloglog'] + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) + def wrong_topic(self): + tdSql.prepare() + tdSql.execute('use db') + stbname = f'db.{tdCom.getLongName(5, "letters")}' + tag_dict = { + 't0':'int' + } + tag_values = [ + f'1' + ] + tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) + tdSql.execute(f"create table {stbname}_tb1 using {stbname} tags({tag_values[0]})") + self.insert_data(self.column_dict,f'{stbname}_tb1',self.rowNum) + for column in self.column_dict.keys(): + for func in self.error_topic: + if func.lower() != 'count' and column.lower() != 'ts': + tdSql.error(f'create topic tpn as select {func}({column}) from {stbname}') + elif func.lower() == 'count' : + tdSql.error(f'create topic tpn as select {func}(*) from {stbname}') + for column in self.column_dict.keys(): + if column.lower() != 'ts': + tdSql.error(f'create topic tpn as select apercentile({column},50) from {stbname}') + tdSql.error(f'create topic tpn as select leastquares({column},1,1) from {stbname}_tb1') + tdSql.error(f'create topic tpn as select HISTOGRAM({column},user_input,[1,3,5,7],0) from {stbname}') + tdSql.error(f'create topic tpn as select percentile({column},1) from {stbname}_tb1') + pass + def run(self): + self.wrong_topic() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 704dba9800668d663f540943b44fd0c33f0da70b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Oct 2022 11:23:53 +0800 Subject: [PATCH 31/63] docs: update examples/python/tmq_example.py with connector repo (#17387) * docs: update examples/python/tmq_example.py with connector repo * Update tmq_example.py * docs: remove tb variable after get_table_name removed Co-authored-by: Liu Jicong --- docs/examples/python/tmq_example.py | 62 ++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index cee036454e..836beb2417 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -1,6 +1,58 @@ import taos -from taos.tmq import TaosConsumer -consumer = TaosConsumer('topic_ctb_column', group_id='vg2') -for msg in consumer: - for row in msg: - print(row) +from taos.tmq import * + +conn = taos.connect() + +print("init") +conn.execute("drop database if exists py_tmq") +conn.execute("create database if not exists py_tmq vgroups 2") +conn.select_db("py_tmq") +conn.execute( + "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)" +) +conn.execute("create table if not exists tb1 using stb1 tags(1)") +conn.execute("create table if not exists tb2 using stb1 tags(2)") +conn.execute("create table if not exists tb3 using stb1 tags(3)") + +print("create topic") +conn.execute("drop topic if exists topic_ctb_column") +conn.execute( + "create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1" +) + +print("build consumer") +conf = TaosTmqConf() +conf.set("group.id", "tg2") +conf.set("td.connect.user", "root") +conf.set("td.connect.pass", "taosdata") +conf.set("enable.auto.commit", "true") + + +def tmq_commit_cb_print(tmq, resp, offset, param=None): + print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") + + +conf.set_auto_commit_cb(tmq_commit_cb_print, None) +tmq = conf.new_consumer() + +print("build topic list") + +topic_list = TaosTmqList() +topic_list.append("topic_ctb_column") + +print("basic consume loop") +tmq.subscribe(topic_list) + +sub_list = tmq.subscription() + +print("subscribed topics: ", sub_list) + +while 1: + res = tmq.poll(1000) + if res: + topic = res.get_topic_name() + vg = res.get_vgroup_id() + db = res.get_db_name() + print(f"topic: {topic}\nvgroup id: {vg}\ndb: {db}") + for row in res: + print(row) From b850c61720e81f6ad3df4aec6f6d5eae1c212b77 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Mon, 17 Oct 2022 11:26:00 +0800 Subject: [PATCH 32/63] test :add test case into ci --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 2686156f0f..fe2337662c 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -279,7 +279,7 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1 - +python3 ./test.py -f 7-tmq/create_wrong_topic.py python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py From 84817bf7438ddcfc959494328c8e29c9c8f4051a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 33/63] fix: coverity issues CID: 354715 --- source/common/src/tvariant.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index a01c393441..830a6199d8 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -366,6 +366,7 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { } if (pBuf != NULL) { + taosMemoryFree(pVariant->pz); *pDest = pBuf; } From 87516ea33243d03a9eb3d6a6b9cf725cccccada2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Oct 2022 13:00:31 +0800 Subject: [PATCH 34/63] fix(query): fix bug in group by tag. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 7 +++- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/scanoperator.c | 32 +++++++++---------- source/libs/executor/src/timewindowoperator.c | 4 +-- source/libs/planner/src/planOptimizer.c | 5 ++- 5 files changed, 26 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c346eeef1d..bc09b4810c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1515,6 +1515,11 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* taosMemoryFree(pReader->pMemSchema); int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + return pReader->pMemSchema; } @@ -3732,7 +3737,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { } int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { - if (isEmptyQueryTimeWindow(&pReader->window)) { + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 413813f58a..01915a7ffb 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -616,7 +616,7 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { int32_t nOptrWithVal = 0; int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); - if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { + if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) { taosMemoryFreeClear(*pOutput); *len = 0; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 66fad27d71..b878112545 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -284,6 +284,19 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, return true; } + +static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + if (pTableScanInfo->pseudoSup.numOfExprs > 0) { + SExprSupp* pSup = &pTableScanInfo->pseudoSup; + + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, + GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + } +} + static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -313,7 +326,6 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; @@ -322,6 +334,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -371,17 +384,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true); - - // currently only the tbname pseudo column - if (pTableScanInfo->pseudoSup.numOfExprs > 0) { - SExprSupp* pSup = &pTableScanInfo->pseudoSup; - - int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, - GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } - } + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); if (pTableScanInfo->pFilterNode != NULL) { int64_t st = taosGetTimestampUs(); @@ -1316,7 +1319,7 @@ static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* blockDataEnsureCapacity(pResBlock, 1); - projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, pTagCalSup->numOfExprs, NULL); + projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, 1, NULL); ASSERT(pResBlock->info.rows == 1); // build tagArray @@ -2120,9 +2123,6 @@ static void destroyStreamScanOperatorInfo(void* param) { taosMemoryFree(pStreamScan->pPseudoExpr); } - cleanupExprSupp(&pStreamScan->tbnameCalSup); - cleanupExprSupp(&pStreamScan->tagCalSup); - updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pUpdateRes); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9c219d2765..cbbd874bb9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2842,14 +2842,13 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc); + initBasicInfo(&pInfo->binfo, pResBlock); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initBasicInfo(&pInfo->binfo, pResBlock); - pInfo->twAggSup.waterMark = pSessionNode->window.watermark; pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType; pInfo->gap = pSessionNode->gap; @@ -4666,7 +4665,6 @@ void destroyStreamStateOperatorInfo(void* param) { SStreamSessionAggOperatorInfo* pChInfo = pChild->info; destroyStreamSessionAggOperatorInfo(pChInfo); taosMemoryFreeClear(pChild); - taosMemoryFreeClear(pChInfo); } } colDataDestroy(&pInfo->twAggSup.timeWindowData); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index cd02e7a136..7fd90dd4e8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -124,9 +124,8 @@ static void optSetParentOrder(SLogicNode* pNode, EOrder order) { EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { - // *((bool*)pContext) = - // (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); - *((bool*)pContext) = true; + *((bool*)pContext) = + (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; From 2df86022c0533dbc03956c728e2c6e2a20107f0c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 17 Oct 2022 10:05:45 +0800 Subject: [PATCH 35/63] fix(stream): coverity scan,type conversion --- source/util/src/tbloomfilter.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 7e1506c140..f3ccbb0aac 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -19,12 +19,12 @@ #include "taos.h" #include "taoserror.h" -#define UNIT_NUM_BITS 64 -#define UNIT_ADDR_NUM_BITS 6 +#define UNIT_NUM_BITS 64ULL +#define UNIT_ADDR_NUM_BITS 6ULL static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1 << (index % UNIT_NUM_BITS); + uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); uint64_t old = buf[unitIndex]; buf[unitIndex] |= mask; return buf[unitIndex] != old; @@ -32,7 +32,7 @@ static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1 << (index % UNIT_NUM_BITS); + uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); return buf[unitIndex] & mask; } From ed9ec795eba10c3745b3cb25ee9768a1a10e4c04 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Oct 2022 14:05:40 +0800 Subject: [PATCH 36/63] fix(query): fix coverity issues. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 15 ++++----- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 ++--- source/libs/executor/src/executil.c | 4 +-- source/libs/executor/src/executor.c | 14 ++++----- source/libs/executor/src/executorimpl.c | 35 +++------------------ source/libs/executor/src/scanoperator.c | 12 +++++-- source/libs/executor/src/tsimplehash.c | 4 +-- tools/shell/src/shellTire.c | 7 ++--- 8 files changed, 37 insertions(+), 62 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index a4c5c60e78..2f4fdfc5f8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -151,9 +151,6 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { ", last file index:%d, last block index:%d, entry:%d, %p, elapsed time:%.2f ms, %s", pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock, el, idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk; tsdbDebug("last block index list:%d, %d, %s", pInfo->blockIndex[0], pInfo->blockIndex[1], idStr); @@ -466,8 +463,8 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { } bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { - int32_t code = 0; int32_t step = pIter->backward ? -1 : 1; + terrno = TSDB_CODE_SUCCESS; // no qualified last file block in current file, no need to fetch row if (pIter->pSttBlk == NULL) { @@ -476,6 +473,10 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { int32_t iBlockL = pIter->iSttBlk; SBlockData *pBlockData = loadLastBlock(pIter, idStr); + if (pBlockData == NULL && terrno != TSDB_CODE_SUCCESS) { + goto _exit; + } + pIter->iRow += step; while (1) { @@ -501,11 +502,7 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow); _exit: - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - } - - return (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); + return (terrno == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); } SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bc09b4810c..670691ab1f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -340,7 +340,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); if (pIter->pLastBlockReader == NULL) { int32_t code = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr); + tsdbError("failed to prepare the last block iterator, code:%s %s", tstrerror(code), pReader->idStr); return code; } } @@ -646,7 +646,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug( - "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " + "load block of %"PRIzu" tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " "time:%.2f ms %s", numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, pReader->idStr); @@ -3090,7 +3090,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); tRowMerge(&merge, piRow); - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); } int32_t code = tRowMergerGetRow(&merge, pTSRow); @@ -3448,7 +3448,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl return code; _err: - tsdbError("failed to create data reader, code:%s %s", tstrerror(code), pReader->idStr); + tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); return code; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 8277a4ed88..f99c832381 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1138,7 +1138,7 @@ static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, i s.bytes = bytes; s.slotId = slotId; s.precision = precision; - strncpy(s.name, name, tListLen(s.name)); + tstrncpy(s.name, name, tListLen(s.name)); return s; } @@ -1366,7 +1366,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); } else { char* udfName = pExpr->pExpr->_function.pFunctNode->functionName; - strncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN); + tstrncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN); fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet); } pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 01915a7ffb..41f861cd38 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -701,10 +701,10 @@ int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { return 0; } -int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo mtInfo) { +int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) { memset(pCond, 0, sizeof(SQueryTableDataCond)); pCond->order = TSDB_ORDER_ASC; - pCond->numOfCols = mtInfo.schema->nCols; + pCond->numOfCols = pMtInfo->schema->nCols; pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); if (pCond->colList == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -712,15 +712,15 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s } pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; - pCond->suid = mtInfo.suid; + pCond->suid = pMtInfo->suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = sContext->snapVersion; for (int32_t i = 0; i < pCond->numOfCols; ++i) { - pCond->colList[i].type = mtInfo.schema->pSchema[i].type; - pCond->colList[i].bytes = mtInfo.schema->pSchema[i].bytes; - pCond->colList[i].colId = mtInfo.schema->pSchema[i].colId; + pCond->colList[i].type = pMtInfo->schema->pSchema[i].type; + pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes; + pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId; } return TSDB_CODE_SUCCESS; @@ -844,7 +844,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT taosArrayDestroy(pTaskInfo->tableqinfoList.pTableList); if (mtInfo.uid == 0) return 0; // no data - initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, mtInfo); + initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0}); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e59755787a..3ec8085b86 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -895,33 +895,6 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI } #endif -static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { -#if 0 - SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; - uint32_t status = BLK_DATA_NOT_LOAD; - - int32_t numOfOutput = 0; // pTableScanInfo->numOfOutput; - for (int32_t i = 0; i < numOfOutput; ++i) { - int32_t functionId = pCtx[i].functionId; - int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId; - - // group by + first/last should not apply the first/last block filter - if (functionId < 0) { - status |= BLK_DATA_DATA_LOAD; - return status; - } else { - // status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId); - // if ((status & BLK_DATA_DATA_LOAD) == BLK_DATA_DATA_LOAD) { - // return status; - // } - } - } - - return status; -#endif - return 0; -} - int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NOT_LOAD; @@ -1802,7 +1775,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); + qDebug("%s fetch rsp received, index:%d, code:%s", pSourceDataInfo->taskId, index, tstrerror(code)); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -3397,8 +3370,8 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; pSchema->colId = pColNode->colId; pSchema->type = pColNode->node.resType.type; - pSchema->type = pColNode->node.resType.bytes; - strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); + pSchema->bytes = pColNode->node.resType.bytes; + tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } // this the tags and pseudo function columns, we only keep the tag columns @@ -3412,7 +3385,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; pSchema->colId = pColNode->colId; pSchema->type = pColNode->node.resType.type; - pSchema->type = pColNode->node.resType.bytes; + pSchema->bytes = pColNode->node.resType.bytes; strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b878112545..15a7abf1ff 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -325,6 +325,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); pCost->skipBlocks += 1; return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { @@ -1082,12 +1084,13 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) { - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); STimeWindow endWin = win; STimeWindow preWin = win; uint64_t groupId = gpIdCol[*pRowIndex]; + while (1) { if (hasGroup) { (*pRowIndex) += 1; @@ -1151,6 +1154,9 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 pResult->info.rows++; } } + + blockDataDestroy(tmpBlock); + if (pResult->info.rows > 0) { pResult->info.calWin = pInfo->updateWin; return pResult; @@ -1546,7 +1552,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { tsdbReaderClose(pTSInfo->dataReader); pTSInfo->dataReader = NULL; tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer); - qDebug("queue scan tsdb over, switch to wal ver %d", pTaskInfo->streamInfo.snapshotVer + 1); + qDebug("queue scan tsdb over, switch to wal ver %"PRId64, pTaskInfo->streamInfo.snapshotVer + 1); if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) { return NULL; } @@ -2995,7 +3001,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { while (1) { int64_t startTs = taosGetTimestampUs(); - strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); strcpy(pInfo->req.user, pInfo->pUser); int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 16fd11f97d..134862d88c 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -18,7 +18,7 @@ #include "tlog.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 -#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_MAX_CAPACITY (1024 * 1024 * 16L) #define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) #define GET_SHASH_NODE_KEY(_n, _dl) ((char *)(_n) + sizeof(SHNode) + (_dl)) @@ -104,7 +104,7 @@ static void tSimpleHashTableResize(SSHashObj *pHashObj) { int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); if (newCapacity > HASH_MAX_CAPACITY) { - uDebug("current capacity:%zu, maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached", + uDebug("current capacity:%"PRIzu", maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached", pHashObj->capacity, HASH_MAX_CAPACITY); return; } diff --git a/tools/shell/src/shellTire.c b/tools/shell/src/shellTire.c index 2f1ee12d54..b310281119 100644 --- a/tools/shell/src/shellTire.c +++ b/tools/shell/src/shellTire.c @@ -243,8 +243,8 @@ void enumAllWords(STireNode** nodes, char* prefix, SMatch* match) { continue; } else { // combine word string - memset(word, 0, sizeof(word)); - strcpy(word, prefix); + memset(word, 0, tListLen(word)); + strncpy(word, prefix, len); word[len] = FIRST_ASCII + i; // append current char // chain middle node @@ -315,8 +315,7 @@ void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) { } } - // return - return; + taosMemoryFree(root); } SMatch* matchPrefix(STire* tire, char* prefix, SMatch* match) { From 54e3617a6fb21967a1227164e60e207997815382 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 17 Oct 2022 14:32:18 +0800 Subject: [PATCH 37/63] fix:scan converity defects --- include/common/tmsg.h | 2 +- source/client/src/clientHb.c | 2 +- source/client/src/clientImpl.c | 4 +- source/client/src/clientRawBlockWrite.c | 20 ++++++---- source/client/src/clientSml.c | 36 +++++++++++------ source/client/src/clientStmt.c | 3 ++ source/client/src/clientTmq.c | 3 +- source/common/src/tdataformat.c | 4 ++ tools/taosws-rs | 1 + utils/test/c/sml_test.c | 8 ++-- utils/test/c/tmqSim.c | 51 ++++++++++++++----------- utils/test/c/tmq_taosx_ci.c | 2 +- 12 files changed, 85 insertions(+), 51 deletions(-) create mode 160000 tools/taosws-rs diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9f21ee007f..748bd6afcc 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1902,7 +1902,7 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) { if (pRebInfo == NULL) { return NULL; } - strcpy(pRebInfo->key, key); + tstrncpy(pRebInfo->key, key, sizeof(pRebInfo->key)); pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebInfo->lostConsumers == NULL) { goto _err; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 096e2cdac8..3f24a08cf4 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -173,7 +173,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; - tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, + tscTrace("conn %u hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, pTscObj->pAppInfo->totalDnodes); if (pRsp->query->killRid) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 940dd745c8..b494140da5 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -186,7 +186,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, STscObj* pTscObj = (*pRequest)->pTscObj; if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self))) { - tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, + tscError("%" PRIx64 " failed to add to request container, reqId:0x%" PRIu64 ", conn:%" PRIx64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); taosMemoryFree(param); @@ -371,7 +371,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) { pInfo->pQnodeList = taosArrayDup(pNodeList); taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad); tscDebug("QnodeList updated in cluster 0x%" PRIx64 ", num:%d", pInfo->clusterId, - taosArrayGetSize(pInfo->pQnodeList)); + (int)taosArrayGetSize(pInfo->pQnodeList)); } taosThreadMutexUnlock(&pInfo->qnodeMutex); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index a257335931..1dc319be94 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -410,6 +410,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVAlterTbReq vAlterTbReq = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -419,7 +420,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -524,6 +525,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropStbReq req = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -533,7 +535,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -556,6 +558,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropTbBatchReq req = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -565,7 +568,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -684,7 +687,7 @@ end: static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { SVDropStbReq req = {0}; - SDecoder coder; + SDecoder coder = {0}; SMDropStbReq pReq = {0}; int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; @@ -1212,6 +1215,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; + SSubmitReq* subReq = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); if (!pRequest) { @@ -1228,8 +1232,8 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) } SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - strcpy(pName.dbname, pRequest->pDb); - strcpy(pName.tname, tbname); + tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname)); + tstrncpy(pName.tname, tbname, sizeof(pName.tname)); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); @@ -1278,7 +1282,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; int32_t totalLen = sizeof(SSubmitReq) + submitLen; - SSubmitReq* subReq = taosMemoryCalloc(1, totalLen); + subReq = taosMemoryCalloc(1, totalLen); SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); @@ -1352,6 +1356,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) if (NULL == pQuery) { uError("create SQuery error"); code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(subReq); goto end; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; @@ -1390,6 +1395,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); + taosMemoryFree(subReq); return code; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 24da5f7b70..5c37822222 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -299,6 +299,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { + taosHashCleanup(hashTmp); return -1; } } @@ -430,7 +431,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { STableMeta *pTableMeta = NULL; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); SRequestConnInfo conn = {0}; conn.pTrans = info->taos->pAppInfo->pTransporter; @@ -874,7 +875,8 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra kv->i = ts; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); + return TSDB_CODE_SUCCESS; } @@ -1009,6 +1011,7 @@ static void smlParseTelnetElement(const char **sql, const char **data, int32_t * static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { + if(!cols) return TSDB_CODE_OUT_OF_MEMORY; const char *sql = data; size_t childTableNameLen = strlen(tsSmlChildTableName); while (*sql != '\0') { @@ -1082,7 +1085,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab kv->length = valueLen; kv->type = TSDB_DATA_TYPE_NCHAR; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); } return TSDB_CODE_SUCCESS; @@ -1370,8 +1373,14 @@ static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { SHashObj *s2 = *(SHashObj **)key2; SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN); SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN); - ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); + if(!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP){ + uError("smlKvTimeHashCompare kv1"); + return -1; + } + if(!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP){ + uError("smlKvTimeHashCompare kv2"); + return -1; + } if (kv1->i < kv2->i) { return -1; } else if (kv1->i > kv2->i) { @@ -1735,7 +1744,7 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { kv->i = tsVal; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); return TSDB_CODE_SUCCESS; } @@ -1932,6 +1941,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { } static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { + if(!cols) return TSDB_CODE_OUT_OF_MEMORY; cJSON *metricVal = cJSON_GetObjectItem(root, "value"); if (metricVal == NULL) { return TSDB_CODE_TSC_INVALID_JSON; @@ -1941,7 +1951,7 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { if (!kv) { return TSDB_CODE_OUT_OF_MEMORY; } - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); kv->key = VALUE; kv->keyLen = VALUE_LEN; @@ -1955,7 +1965,9 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { int32_t ret = TSDB_CODE_SUCCESS; - + if (!pKVs){ + return TSDB_CODE_OUT_OF_MEMORY; + } cJSON *tags = cJSON_GetObjectItem(root, "tags"); if (tags == NULL || tags->type != cJSON_Object) { return TSDB_CODE_TSC_INVALID_JSON; @@ -1985,14 +1997,14 @@ static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableN return TSDB_CODE_TSC_INVALID_JSON; } memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); + tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); continue; } // add kv to SSmlKv SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - if (pKVs) taosArrayPush(pKVs, &kv); + taosArrayPush(pKVs, &kv); // key kv->keyLen = keyLen; @@ -2103,6 +2115,8 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { if (!oneTable) { tinfo = smlBuildTableInfo(); if (!tinfo) { + smlDestroyCols(cols); + if (info->dataFormat) taosArrayDestroy(cols); return TSDB_CODE_TSC_OUT_OF_MEMORY; } taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); @@ -2295,7 +2309,7 @@ static int32_t smlInsertData(SSmlHandle *info) { SSmlTableInfo *tableData = *oneTable; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); SRequestConnInfo conn = {0}; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 7a800bd334..bf3fd00f14 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -201,6 +201,9 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if(!pSrc){ + return TSDB_CODE_OUT_OF_MEMORY; + } STableDataBlocks* pDst = NULL; STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 922cab23c8..fc83c52cdd 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -870,8 +870,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("consumer %" PRId64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), - pTmq->groupId); + tscError("consumer setup failed since %s", terrstr()); return NULL; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ad5772e0fe..269e72f151 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -916,6 +916,10 @@ char *tTagValToData(const STagVal *value, bool isJson) { } bool tTagGet(const STag *pTag, STagVal *pTagVal) { + if(!pTag || !pTagVal){ + return false; + } + int16_t lidx = 0; int16_t ridx = pTag->nTag - 1; int16_t midx; diff --git a/tools/taosws-rs b/tools/taosws-rs new file mode 160000 index 0000000000..7a94ffab45 --- /dev/null +++ b/tools/taosws-rs @@ -0,0 +1 @@ +Subproject commit 7a94ffab45f08e16f09b3f430fe75d717054adb6 diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 56ba622a9c..b6d8d75ba0 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -119,7 +119,7 @@ int smlProcess_json1_Test() { " \"dc\": \"lga\"" " }" " }" - "]"}; + "]",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -159,7 +159,7 @@ int smlProcess_json2_Test() { " }," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -227,7 +227,7 @@ int smlProcess_json3_Test() { " }," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -286,7 +286,7 @@ int smlProcess_json4_Test() { " \"t9\": false," " \"id\": \"d1001\"" " }" - "}"}; + "}",}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index a043aa7a6d..d36fe0855a 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -155,7 +155,7 @@ static void printHelp() { printf("%s%s\n", indent, "-l"); - printf("%s%s%s\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); + printf("%s%s%s%d\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "producer thread number, default is 0"); printf("%s%s\n", indent, "-b"); @@ -238,7 +238,7 @@ void saveConfigToLogFile() { taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); } taosFprintfFile(g_fp, "\n"); - taosFprintfFile(g_fp, " expect rows: %d\n", g_stConfInfo.stThreads[i].expectMsgCnt); + taosFprintfFile(g_fp, " expect rows: %" PRIx64 "\n", g_stConfInfo.stThreads[i].expectMsgCnt); } char tmpString[128]; @@ -263,11 +263,11 @@ void parseArgument(int32_t argc, char* argv[]) { printHelp(); exit(0); } else if (strcmp(argv[i], "-d") == 0) { - strcpy(g_stConfInfo.dbName, argv[++i]); + tstrncpy(g_stConfInfo.dbName, argv[++i], sizeof(g_stConfInfo.dbName)); } else if (strcmp(argv[i], "-w") == 0) { - strcpy(g_stConfInfo.cdbName, argv[++i]); + tstrncpy(g_stConfInfo.cdbName, argv[++i], sizeof(g_stConfInfo.cdbName)); } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); + tstrncpy(configDir, argv[++i], PATH_MAX); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { @@ -279,9 +279,9 @@ void parseArgument(int32_t argc, char* argv[]) { } else if (strcmp(argv[i], "-e") == 0) { g_stConfInfo.useSnapshot = atol(argv[++i]); } else if (strcmp(argv[i], "-t") == 0) { - char tmpBuf[56]; - strcpy(tmpBuf, argv[++i]); - sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); + char tmpBuf[56] = {0}; + tstrncpy(tmpBuf, argv[++i], sizeof(tmpBuf)); + sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); } else if (strcmp(argv[i], "-x") == 0) { g_stConfInfo.numOfThread = atol(argv[++i]); } else if (strcmp(argv[i], "-l") == 0) { @@ -294,6 +294,10 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.producerRate = atol(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { g_stConfInfo.payloadLen = atol(argv[++i]); + if(g_stConfInfo.payloadLen <= 0 || g_stConfInfo.payloadLen > 1024 * 1024 * 1024){ + pError("%s calloc size is too large: %s %s", GREEN, argv[++i], NC); + exit(-1); + } } else { pError("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); @@ -354,8 +358,8 @@ void ltrim(char* str) { int queryDB(TAOS* taos, char* command) { int retryCnt = 10; - int code; - TAOS_RES* pRes; + int code = 0; + TAOS_RES* pRes = NULL; while (retryCnt--) { pRes = taos_query(taos, command); @@ -363,10 +367,11 @@ int queryDB(TAOS* taos, char* command) { if (code != 0) { taosSsleep(1); taos_free_result(pRes); + pRes = NULL; continue; } taos_free_result(pRes); - return 0; + return 0; } pError("failed to reason:%s, sql: %s", tstrerror(code), command); @@ -418,7 +423,7 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { char sqlStr[1100] = {0}; if (strlen(buf) > 1024) { - taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf)); + taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", (int)strlen(buf)); taosCloseFile(&g_fp); return -1; } @@ -592,7 +597,7 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex); taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId); @@ -644,7 +649,7 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex); taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId); @@ -960,7 +965,7 @@ void parseConsumeInfo() { ltrim(pstr); char* ret = strchr(pstr, ch); memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr); - strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1); + tstrncpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1, sizeof(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey])); // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], // g_stConfInfo.value[g_stConfInfo.numOfKey]); g_stConfInfo.stThreads[i].numOfKey++; @@ -1268,25 +1273,26 @@ void* ombProduceThreadFunc(void* param) { for (int i = 0; i < batchPerTblTimes; ++i) { uint32_t msgsOfSql = g_stConfInfo.batchSize; if ((i == batchPerTblTimes - 1) && (0 != remainder)) { - msgsOfSql = remainder; + msgsOfSql = remainder; } int len = 0; len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "insert into %s values ", ctbName); for (int j = 0; j < msgsOfSql; j++) { - int64_t timeStamp = taosGetTimestampNs(); - len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); + int64_t timeStamp = taosGetTimestampNs(); + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); sendMsgs++; pInfo->totalProduceMsgs++; } - totalMsgLen += len; + totalMsgLen += len; pInfo->totalMsgsLen += len; - int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); + int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); if (affectedRows < 0) { taos_close(pInfo->taos); - pInfo->taos = NULL; - return NULL; + pInfo->taos = NULL; + taosMemoryFree(sqlBuf); + return NULL; } affectedRowsTotal += affectedRows; @@ -1322,6 +1328,7 @@ void* ombProduceThreadFunc(void* param) { printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); taos_close(pInfo->taos); pInfo->taos = NULL; + taosMemoryFree(sqlBuf); return NULL; } diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 3a598ba98b..6540fdac4c 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -663,7 +663,7 @@ void initLogFile() { int main(int argc, char* argv[]) { for (int32_t i = 1; i < argc; i++) { if(strcmp(argv[i], "-c") == 0){ - strcpy(g_conf.dir, argv[++i]); + tstrncpy(g_conf.dir, argv[++i], sizeof(g_conf.dir)); }else if(strcmp(argv[i], "-s") == 0){ g_conf.snapShot = true; }else if(strcmp(argv[i], "-d") == 0){ From 4c2896236f0a1065a7d7cbf51400fba4aee65685 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 17 Oct 2022 14:34:34 +0800 Subject: [PATCH 38/63] fix: reserve enough capacity for inter/final result block when performing scalar calculation of fill operator --- source/libs/executor/src/executorimpl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e59755787a..64f09cf766 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2816,6 +2816,8 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId); blockDataCleanup(pInfo->pRes); + blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); + blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.groupId) { From 73662b3058eb670198675dcd5354fe569e76af00 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 11:17:41 +0800 Subject: [PATCH 39/63] refactor: table sink --- source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tq.c | 12 +- source/dnode/vnode/src/tq/tqSink.c | 206 ++++++++++++++++++++ tests/system-test/7-tmq/tmqAutoCreateTbl.py | 2 +- 4 files changed, 214 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index f96afe6fba..69e6cdce9f 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -115,7 +115,6 @@ typedef struct { typedef struct { SMqDataRsp dataRsp; - SMqRspHead rspHead; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; SRpcHandleInfo pInfo; } STqPushEntry; @@ -183,6 +182,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t ver); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index aaa57d49ed..5e1cc15063 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -192,7 +192,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { return -1; } - memcpy(buf, &pPushEntry->rspHead, sizeof(SMqRspHead)); + memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); @@ -215,7 +215,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { tFormatOffset(buf1, 80, &pRsp->reqOffset); tFormatOffset(buf2, 80, &pRsp->rspOffset); tqDebug("vgId:%d, from consumer:%" PRId64 ", (epoch %d) push rsp, block num: %d, reqOffset:%s, rspOffset:%s", - TD_VID(pTq->pVnode), pPushEntry->rspHead.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); + TD_VID(pTq->pVnode), pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); return 0; } @@ -560,9 +560,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { memcpy(pPushEntry->subKey, pHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN); dataRsp.withTbName = 0; memcpy(&pPushEntry->dataRsp, &dataRsp, sizeof(SMqDataRsp)); - pPushEntry->rspHead.consumerId = consumerId; - pPushEntry->rspHead.epoch = reqEpoch; - pPushEntry->rspHead.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + pPushEntry->dataRsp.head.consumerId = consumerId; + pPushEntry->dataRsp.head.epoch = reqEpoch; + pPushEntry->dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey) + 1, &pPushEntry, sizeof(void*)); tqDebug("tmq poll: consumer %ld, subkey %s, vg %d save handle to push mgr", consumerId, pHandle->subKey, TD_VID(pTq->pVnode)); @@ -924,7 +924,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqTableSink; + pTask->tbSink.tbSinkFunc = tqTableSink1; ASSERT(pTask->tbSink.pSchemaWrapper); ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 521d12fdab..76bfa72caa 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -284,6 +284,212 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem return ret; } +void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper; + + int32_t blockSz = taosArrayGetSize(pBlocks); + bool createTb = true; + + SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + + tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + for (int32_t i = 0; i < blockSz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + SBatchDeleteReq deleteReq = {0}; + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + deleteReq.suid = suid; + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq); + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + taosArrayDestroy(deleteReq.deleteReqs); + + ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; + + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = serializedDeleteReq, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(serializedDeleteReq); + tqDebug("failed to put delete req into write-queue since %s", terrstr()); + } + } else { + SVCreateTbReq createTbReq = {0}; + + // set const + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, + }; + taosArrayPush(tagArray, &tagVal); + createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(tagArray); + tdDestroySVCreateTbReq(&createTbReq); + return; + } + createTbReq.ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + createTbReq.ctb.tagName = tagName; + + // set table name + if (pDataBlock->info.parTbName[0]) { + createTbReq.name = strdup(pDataBlock->info.parTbName); + } else { + createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + } + + int32_t schemaLen; + int32_t code; + tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + // save schema str + void* schemaStr = taosMemoryMalloc(schemaLen); + if (schemaStr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, schemaStr, schemaLen); + code = tEncodeSVCreateTbReq(&encoder, &createTbReq); + if (code < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + tEncoderClear(&encoder); + taosMemoryFree(schemaStr); + return; + } + tEncoderClear(&encoder); + tdDestroySVCreateTbReq(&createTbReq); + + int32_t cap = sizeof(SSubmitReq); + + int32_t rows = pDataBlock->info.rows; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + + cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; + + SSubmitReq* ret = rpcMallocCont(cap); + ret->header.vgId = pVnode->config.vgId; + ret->length = sizeof(SSubmitReq); + ret->numOfBlocks = htonl(1); + + SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); + + blkHead->numOfRows = htonl(pDataBlock->info.rows); + blkHead->sversion = htonl(pTSchema->version); + blkHead->suid = htobe64(suid); + // uid is assigned by vnode + blkHead->uid = 0; + blkHead->schemaLen = 0; + + tqDebug("tq sink, convert block %d, rows: %d", i, rows); + + int32_t dataLen = 0; + void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); + STSRow* rowData = blkSchema; + if (createTb) { + memcpy(blkSchema, schemaStr, schemaLen); + blkHead->schemaLen = htonl(schemaLen); + rowData = POINTER_SHIFT(blkSchema, schemaLen); + } + + taosMemoryFree(schemaStr); + + for (int32_t j = 0; j < rows; j++) { + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTSchema->version); + tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); + tdSRowResetBuf(&rb, rowData); + + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pColumn = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); + } else { + void* colData = colDataGetData(pColData, j); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k); + } + } + tdSRowEnd(&rb); + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + dataLen += rowLen; + } + blkHead->dataLen = htonl(dataLen); + + ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; + ret->length = htonl(ret->length); + + SRpcMsg msg = { + .msgType = TDMT_VND_SUBMIT, + .pCont = ret, + .contLen = ntohl(ret->length), + }; + + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(ret); + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + } + taosArrayDestroy(tagArray); +} + void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pRes = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index a613f11267..de95fef148 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -52,7 +52,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1, wal_retention_size=-1,wal_retention_period=-1) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) # tdLog.info("create ctb") From 2abd0eec40474558a8d6a93b21156e7fcce56c55 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 40/63] fix: coverity issues CID: 354727 --- source/common/src/tvariant.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 830a6199d8..0533a49514 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -28,10 +28,6 @@ *exti = -1; \ break; \ } \ - if ((res) > (maxv)) { \ - *exti = 1; \ - break; \ - } \ assert(0); \ } while (0) From d2fc6215c6a0c9247b14c1fe49f320db573b8ca3 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 15:06:05 +0800 Subject: [PATCH 41/63] fix: client mem leak --- source/client/src/clientTmq.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 73f7c2bcc4..51005e3c30 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -709,6 +709,7 @@ void tmqSendHbReq(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { + taosMemoryFree(param); return; } int64_t consumerId = tmq->consumerId; @@ -939,10 +940,9 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { return NULL; } - int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); - *pRefId = pTmq->refId; - if (pTmq->hbBgEnable) { + int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); + *pRefId = pTmq->refId; pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer); } From 7e54b77024ab488222f6d7bf6e909f79e4fa5ffd Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 42/63] fix: coverity issues CID: 354728 --- source/common/src/tvariant.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 0533a49514..d0f4855549 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -28,6 +28,10 @@ *exti = -1; \ break; \ } \ + if ((res) > (maxv)) { \ + *exti = -1; \ + break; \ + } \ assert(0); \ } while (0) @@ -685,7 +689,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc case TSDB_DATA_TYPE_BIGINT: { if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { - SET_EXT_INFO(converted, (int64_t)result, INT64_MIN + 1, INT64_MAX, extInfo); + SET_EXT_INFO(converted, result, INT64_MIN + 1, INT64_MAX, extInfo); return -1; } *((int64_t *)payload) = (int64_t)result; @@ -694,7 +698,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc case TSDB_DATA_TYPE_UBIGINT: { if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) { - SET_EXT_INFO(converted, (uint64_t)result, 0, UINT64_MAX - 1, extInfo); + SET_EXT_INFO(converted, result, 0, UINT64_MAX - 1, extInfo); return -1; } *((uint64_t *)payload) = (uint64_t)result; From 5c66712b9123770465a784e8246154a4942ca4be Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 17 Oct 2022 15:28:10 +0800 Subject: [PATCH 43/63] fix: coverity issues --- source/dnode/vnode/src/meta/metaQuery.c | 2 +- source/dnode/vnode/src/meta/metaTable.c | 7 ++++--- source/libs/tdb/src/db/tdbBtree.c | 1 + source/libs/tdb/src/db/tdbPCache.c | 7 ++++--- source/libs/tdb/src/db/tdbPager.c | 21 +++++++++++++++------ source/libs/tdb/src/db/tdbTable.c | 2 ++ source/libs/tdb/src/inc/tdbOs.h | 4 +++- 7 files changed, 30 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 84ce087e55..631ef09d4b 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -675,7 +675,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv SSchemaWrapper *pSchemaWrapper = &schema; tDecoderInit(&dc, pData, nData); - tDecodeSSchemaWrapper(&dc, pSchemaWrapper); + (void)tDecodeSSchemaWrapper(&dc, pSchemaWrapper); tDecoderClear(&dc); tdbFree(pData); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 1e0e757793..2b7982d381 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -944,6 +944,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaUpdateTagIdx(pMeta, &ctbEntry); } + ASSERT(ctbEntry.ctbEntry.pTags); SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); @@ -952,7 +953,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tDecoderClear(&dc1); tDecoderClear(&dc2); - if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); + taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); tdbTbcClose(pTbDbc); @@ -1202,8 +1203,8 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { SMetaEntry stbEntry = {0}; STagIdxKey *pTagIdxKey = NULL; int32_t nTagIdxKey; - const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; - const void *pTagData = NULL; // + const SSchema *pTagColumn; + const void *pTagData = NULL; int32_t nTagData = 0; SDecoder dc = {0}; int32_t ret = 0; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 13b7e0ac0a..9e17f50dce 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1006,6 +1006,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const nLeft -= kLen; // pack partial val to local if any space left if (nLocal > nHeader + kLen + sizeof(SPgno)) { + ASSERT(pVal != NULL && vLen != 0); memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno)); nLeft -= nLocal - nHeader - kLen - sizeof(SPgno); } diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index b6d1f95c0e..c73ddce74c 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -105,6 +105,7 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) { for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) { if (tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL) < 0) { // TODO: handle error + tdbOsFree(pCache->aPage); return -1; } @@ -267,7 +268,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) // 4. Try a create new page if (!pPage) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); - if (ret < 0) { + if (ret < 0 && pPage != NULL) { // TODO ASSERT(0); return NULL; @@ -300,8 +301,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) pPage->pPager = pPageH->pPager; memcpy(pPage->pData, pPageH->pData, pPage->pageSize); - tdbDebug("pcache/pPageH: %p %d %p %p %d", pPageH, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize, pPage, - TDB_PAGE_PGNO(pPageH)); + // tdbDebug("pcache/pPageH: %p %d %p %p %d", pPageH, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize, pPage, + // TDB_PAGE_PGNO(pPageH)); tdbPageInit(pPage, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize); pPage->kLen = pPageH->kLen; pPage->vLen = pPageH->vLen; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index dd2416e5b8..57ad2a783a 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -84,7 +84,8 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { pPager->pCache = pCache; pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); - if (pPager->fd < 0) { + if (TDB_FD_INVALID(pPager->fd)) { + // if (pPager->fd < 0) { return -1; } @@ -226,7 +227,7 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { // Open the journal pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); - if (pPager->jfd < 0) { + if (TDB_FD_INVALID(pPager->jfd)) { tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -365,7 +366,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // 4, remove the journal file tdbOsClose(pPager->jfd); - tdbOsRemove(pPager->jFileName); + (void)tdbOsRemove(pPager->jFileName); pPager->inTran = 0; return 0; @@ -540,7 +541,8 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { - tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, pPage->pageSize); + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, + pPage->pageSize); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -568,7 +570,8 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize); if (ret < 0) { - tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName, pPage->pageSize); + tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->dbFileName, + pPage->pageSize); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -603,11 +606,13 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); if (ret < 0) { + tdbOsFree(pageBuf); return -1; } ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { + tdbOsFree(pageBuf); return -1; } @@ -615,13 +620,16 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { tdbError("failed to lseek fd due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); return -1; } ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize); if (ret < 0) { - tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, pPager->pageSize); + tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, + pPager->pageSize); terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); return -1; } } @@ -629,6 +637,7 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { if (tdbOsFSync(pPager->fd) < 0) { tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); return -1; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 380baefb46..008907ca77 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -106,11 +106,13 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF // pTb->pBt ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, &(pTb->pBt)); if (ret < 0) { + tdbOsFree(pTb); return -1; } ret = tdbPagerRestore(pPager, pTb->pBt); if (ret < 0) { + tdbOsFree(pTb); return -1; } diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index 503e109adb..b5dd27052c 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -37,6 +37,8 @@ extern "C" { /* file */ typedef TdFilePtr tdb_fd_t; +#define TDB_FD_INVALID(fd) (fd == NULL) + #define TDB_O_CREAT TD_FILE_CREATE #define TDB_O_WRITE TD_FILE_WRITE #define TDB_O_READ TD_FILE_READ @@ -141,4 +143,4 @@ typedef pthread_mutex_t tdb_mutex_t; } #endif -#endif /*_TDB_OS_H_*/ \ No newline at end of file +#endif /*_TDB_OS_H_*/ From 51dd72ec82b019fe7f44b00ef1a3b5a4112e0e56 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 44/63] fix: coverity issues CID: 372918 --- source/libs/scalar/src/filter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 5865370e8e..0cfeeedde8 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1644,7 +1644,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SColumnNode *refNode = (SColumnNode *)left->desc; - if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS) { + if (unit->compare.optr <= OP_TYPE_JSON_CONTAINS) { len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->dataBlockId, refNode->slotId, operatorTypeStr(unit->compare.optr)); } @@ -1664,7 +1664,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); - if (unit->compare.optr2 >= 0 && unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS) { + if (unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS) { sprintf(str + strlen(str), "[%d][%d] %s [", refNode->dataBlockId, refNode->slotId, operatorTypeStr(unit->compare.optr2)); } From c6956061847a3fc4297132d20cf811392e37d489 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 45/63] fix: coverity issues CID: 399357 --- source/common/src/tdataformat.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ad5772e0fe..ef550891b7 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -602,7 +602,7 @@ _exit: } int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { - int32_t n; + int32_t n = 0; TSROW_LEN(pRow, n); if (p) { @@ -613,7 +613,7 @@ int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { } int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) { - int32_t n; + int32_t n = 0; *ppRow = (STSRow2 *)p; TSROW_LEN(*ppRow, n); From 4639051e683eced1583508d47aa20d43b0500512 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Oct 2022 16:03:45 +0800 Subject: [PATCH 46/63] fix(query): fix coverity issues. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 69 +++++++++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 - 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 670691ab1f..1da5297396 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1506,7 +1506,12 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* if (pReader->pMemSchema == NULL) { int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); - return pReader->pMemSchema; + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } else { + return pReader->pMemSchema; + } } if (pReader->pMemSchema->version == sversion) { @@ -1518,9 +1523,9 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; + } else { + return pReader->pMemSchema; } - - return pReader->pMemSchema; } static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, @@ -1816,7 +1821,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -1826,8 +1835,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow1); } else { init = true; - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } @@ -1837,7 +1850,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pSchema); + int32_t code = tRowMergerInit(&merge, piRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); } @@ -1847,7 +1863,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, pRow); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } @@ -1855,7 +1874,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } @@ -1865,7 +1888,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pSchema); + int32_t code = tRowMergerInit(&merge, piRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); } @@ -1876,7 +1902,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow1); } else { init = true; - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } @@ -1884,7 +1913,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } else { tRowMerge(&merge, &fRow); } @@ -3046,19 +3078,30 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SRowMerger merge = {0}; // get the correct schema for data in memory + terrno = 0; STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } if (pReader->pSchema == NULL) { pReader->pSchema = pTSchema; } - tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + int32_t code = tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + if(pTSchema1 == NULL) { + return terrno; + } + tRowMergerAdd(&merge, pNextRow, pTSchema1); doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); - int32_t code = tRowMergerGetRow(&merge, pTSRow); + code = tRowMergerGetRow(&merge, pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 47c836d9c1..4e02a28cdf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -719,8 +719,6 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { } pMerger->version = key.version; - -_exit: return code; } From 89f80436c65d95166cc8f704c86d2548270d2790 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Oct 2022 16:17:16 +0800 Subject: [PATCH 47/63] fix(query): fix coverity issues. --- source/libs/executor/src/tsimplehash.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 134862d88c..b66a6f6770 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -16,6 +16,7 @@ #include "tsimplehash.h" #include "taoserror.h" #include "tlog.h" +#include "tdef.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 #define HASH_MAX_CAPACITY (1024 * 1024 * 16L) @@ -110,14 +111,14 @@ static void tSimpleHashTableResize(SSHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, POINTER_BYTES * newCapacity); if (!pNewEntryList) { uWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; } size_t inc = newCapacity - pHashObj->capacity; - memset((char *)pNewEntryList + pHashObj->capacity * sizeof(void *), 0, inc * sizeof(void *)); + memset((char *)pNewEntryList + pHashObj->capacity * POINTER_BYTES, 0, inc * sizeof(void *)); pHashObj->hashList = pNewEntryList; pHashObj->capacity = newCapacity; From 6d965b8b819b7116d0d76357ae5cad92383d4b7c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Oct 2022 16:27:44 +0800 Subject: [PATCH 48/63] fix(query): fix coverity issues. --- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/scanoperator.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 41f861cd38..0f4118257b 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -294,7 +294,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo if (!exists) { #endif taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo); - taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(uid), &keyInfo.groupId, sizeof(keyInfo.groupId)); + taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(*uid), &keyInfo.groupId, sizeof(keyInfo.groupId)); } /*}*/ diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 15a7abf1ff..c6bd3532ef 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -942,8 +942,7 @@ static FORCE_INLINE void doClearBufferedBlocks(SStreamScanInfo* pInfo) { } static bool isSessionWindow(SStreamScanInfo* pInfo) { - return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || - pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; + return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; } static bool isStateWindow(SStreamScanInfo* pInfo) { From 68e0ddfa8831905c1546600718b5178b2776c7d2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 16:28:55 +0800 Subject: [PATCH 49/63] refactor: tmq client --- include/common/tmsg.h | 2 +- source/client/src/clientTmq.c | 44 +++++++++++++++++++---------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9f21ee007f..5e2ca8896a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1902,7 +1902,7 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) { if (pRebInfo == NULL) { return NULL; } - strcpy(pRebInfo->key, key); + tstrncpy(pRebInfo->key, key, TSDB_SUBSCRIBE_KEY_LEN); pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebInfo->lostConsumers == NULL) { goto _err; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 51005e3c30..8f94d88b1c 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -233,12 +233,12 @@ void tmq_conf_destroy(tmq_conf_t* conf) { tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { if (strcmp(key, "group.id") == 0) { - strcpy(conf->groupId, value); + tstrncpy(conf->groupId, value, TSDB_CGROUP_LEN); return TMQ_CONF_OK; } if (strcmp(key, "client.id") == 0) { - strcpy(conf->clientId, value); + tstrncpy(conf->clientId, value, 256); return TMQ_CONF_OK; } @@ -452,7 +452,6 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT int32_t code; tEncodeSize(tEncodeSTqOffset, pOffset, len, code); if (code < 0) { - ASSERT(0); return -1; } void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); @@ -464,15 +463,22 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT SEncoder encoder; tEncoderInit(&encoder, abuf, len); tEncodeSTqOffset(&encoder, pOffset); + tEncoderClear(&encoder); // build param SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); + if (pParam == NULL) { + taosMemoryFree(buf); + return -1; + } pParam->params = pParamSet; pParam->pOffset = pOffset; // build send info SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { + taosMemoryFree(buf); + taosMemoryFree(pParam); return -1; } pMsgSendInfo->msgInfo = (SDataBuf){ @@ -547,6 +553,8 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { + tsem_destroy(&pParamSet->rspSem); + taosMemoryFree(pParamSet); goto FAIL; } goto HANDLE_RSP; @@ -565,6 +573,7 @@ HANDLE_RSP: tsem_wait(&pParamSet->rspSem); code = pParamSet->rspErr; tsem_destroy(&pParamSet->rspSem); + taosMemoryFree(pParamSet); return code; } else { code = 0; @@ -587,7 +596,14 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + if (async) { + if (automatic) { + tmq->commitCb(tmq, code, tmq->commitCbUserParam); + } else { + userCb(tmq, code, userParam); + } + } return -1; } @@ -642,16 +658,6 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t code = pParamSet->rspErr; tsem_destroy(&pParamSet->rspSem); taosMemoryFree(pParamSet); - } else { - code = 0; - } - - if (code != 0 && async) { - if (automatic) { - tmq->commitCb(tmq, code, tmq->commitCbUserParam); - } else { - userCb(tmq, code, userParam); - } } #if 0 @@ -722,6 +728,7 @@ void tmqSendHbReq(void* param, void* tmrId) { SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { taosMemoryFree(pReq); + goto OVER; } sendInfo->msgInfo = (SDataBuf){ .pData = pReq, @@ -871,8 +878,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("consumer %" PRId64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), - pTmq->groupId); + tscError("setting up new consumer failed since %s, consumer group %s", terrstr(), conf->groupId); return NULL; } @@ -1061,9 +1067,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = 0; FAIL: if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree); - if (code != 0 && buf) { - taosMemoryFree(buf); - } + taosMemoryFree(buf); + return code; } @@ -1101,7 +1106,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { - taosMemoryFree(pMsg->pData); tscWarn("msg discard from vgId:%d, epoch %d since out of memory", vgId, epoch); goto CREATE_MSG_FAIL; } From 51e5522ca442ad53d1b2653d8109bbaf10a4c95f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 50/63] fix: coverity issues CID: 399471 --- examples/c/tmq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 19adaad116..d3fc803c94 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -212,6 +212,7 @@ tmq_list_t* build_topic_list() { tmq_list_t* topicList = tmq_list_new(); int32_t code = tmq_list_append(topicList, "topicname"); if (code) { + tmq_list_destroy(topicList); return NULL; } return topicList; From 1a3b2fa4f2a3377df76a60f4bf6fb9b15d7fc219 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 51/63] fix: coverity issues CID: 399527 --- source/libs/scalar/src/filter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 0cfeeedde8..a04d5a3a3a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1214,7 +1214,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit *u, uint32_t *uidx) { SFilterFieldId left, right, *pright = &right; - int32_t type = FILTER_UNIT_DATA_TYPE(u); + uint8_t type = FILTER_UNIT_DATA_TYPE(u); uint16_t flag = 0; filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); @@ -2614,7 +2614,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum int32_t usize = (int32_t)taosArrayGetSize((SArray *)colInfo->info); for (int32_t n = 0; n < usize; ++n) { - SFilterUnit *u = taosArrayGetP((SArray *)colInfo->info, n); + SFilterUnit *u = (SFilterUnit *)taosArrayGetP((SArray *)colInfo->info, n); filterAddUnitFromUnit(info, &oinfo, u, &uidx); filterAddUnitToGroup(&ng, uidx); From bcae6e1f4e60bf0f69470e406f3bf5ce082db97c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 17:18:57 +0800 Subject: [PATCH 52/63] refactor(stream): optimize auto create child table --- source/dnode/vnode/src/tq/tqSink.c | 176 +++++++++++++++++------------ 1 file changed, 105 insertions(+), 71 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 76bfa72caa..366cd6268e 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -293,7 +293,6 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper; int32_t blockSz = taosArrayGetSize(pBlocks); - bool createTb = true; SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); if (!tagArray) { @@ -303,6 +302,7 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); for (int32_t i = 0; i < blockSz; i++) { + bool createTb = true; SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); if (pDataBlock->info.type == STREAM_DELETE_RESULT) { SBatchDeleteReq deleteReq = {0}; @@ -337,83 +337,115 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { tqDebug("failed to put delete req into write-queue since %s", terrstr()); } } else { - SVCreateTbReq createTbReq = {0}; - - // set const - createTbReq.flags = 0; - createTbReq.type = TSDB_CHILD_TABLE; - createTbReq.ctb.suid = suid; - - // set super table name - SName name = {0}; - tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); - - // set tag content - taosArrayClear(tagArray); - STagVal tagVal = { - .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .i64 = (int64_t)pDataBlock->info.groupId, - }; - taosArrayPush(tagArray, &tagVal); - createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); - - STag* pTag = NULL; - tTagNew(tagArray, 1, false, &pTag); - if (pTag == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(tagArray); - tdDestroySVCreateTbReq(&createTbReq); - return; - } - createTbReq.ctb.pTag = (uint8_t*)pTag; - - // set tag name - SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); - char tagNameStr[TSDB_COL_NAME_LEN] = {0}; - strcpy(tagNameStr, "group_id"); - taosArrayPush(tagName, tagNameStr); - createTbReq.ctb.tagName = tagName; - - // set table name + char* ctbName = NULL; + // set child table name if (pDataBlock->info.parTbName[0]) { - createTbReq.name = strdup(pDataBlock->info.parTbName); + ctbName = strdup(pDataBlock->info.parTbName); } else { - createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); } - int32_t schemaLen; - int32_t code; - tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - if (code < 0) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - return; - } + int32_t schemaLen = 0; + void* schemaStr = NULL; - // save schema str - void* schemaStr = taosMemoryMalloc(schemaLen); - if (schemaStr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - return; - } + int64_t uid = 0; + SMetaReader mr = {0}; + metaReaderInit(&mr, pVnode->pMeta, 0); + if (metaGetTableEntryByName(&mr, ctbName) < 0) { + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); - SEncoder encoder = {0}; - tEncoderInit(&encoder, schemaStr, schemaLen); - code = tEncodeSVCreateTbReq(&encoder, &createTbReq); - if (code < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); + SVCreateTbReq createTbReq = {0}; + + // set const + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + createTbReq.name = ctbName; + ctbName = NULL; + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, + }; + taosArrayPush(tagArray, &tagVal); + createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(tagArray); + tdDestroySVCreateTbReq(&createTbReq); + return; + } + createTbReq.ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + createTbReq.ctb.tagName = tagName; + + int32_t code; + tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + // set schema str + schemaStr = taosMemoryMalloc(schemaLen); + if (schemaStr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, schemaStr, schemaLen); + code = tEncodeSVCreateTbReq(&encoder, &createTbReq); + if (code < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + tEncoderClear(&encoder); + taosMemoryFree(schemaStr); + return; + } tEncoderClear(&encoder); - taosMemoryFree(schemaStr); - return; + tdDestroySVCreateTbReq(&createTbReq); + } else { + if (mr.me.type != TSDB_CHILD_TABLE) { + tqError("vgId:%d, failed to write into %s, since table type incorrect, type %d", TD_VID(pVnode), ctbName, + mr.me.type); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; + } + if (mr.me.ctbEntry.suid != suid) { + tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid: %ld, actual suid %ld", + TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; + } + + createTb = false; + uid = mr.me.uid; + metaReaderClear(&mr); + taosMemoryFreeClear(ctbName); } - tEncoderClear(&encoder); - tdDestroySVCreateTbReq(&createTbReq); int32_t cap = sizeof(SSubmitReq); @@ -445,9 +477,11 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { memcpy(blkSchema, schemaStr, schemaLen); blkHead->schemaLen = htonl(schemaLen); rowData = POINTER_SHIFT(blkSchema, schemaLen); + } else { + blkHead->uid = uid; } - taosMemoryFree(schemaStr); + taosMemoryFreeClear(schemaStr); for (int32_t j = 0; j < rows; j++) { SRowBuilder rb = {0}; From 89800c9d54c0efdaa642c3332f6c8102b9d0b890 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Oct 2022 18:26:16 +0800 Subject: [PATCH 53/63] chore: taostools sanitizer flag align with tdengine (#17407) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index f0a1ad83ad..73409a950a 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 2849aa4 + GIT_TAG 4d02980 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 16fb3525aad0e15a4711fb02188d96042e90ece8 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 17 Oct 2022 10:59:32 +0800 Subject: [PATCH 54/63] fix: coverity issues CID: 399527 --- source/common/src/tvariant.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index d0f4855549..9660a37f48 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -29,7 +29,7 @@ break; \ } \ if ((res) > (maxv)) { \ - *exti = -1; \ + *exti = 1; \ break; \ } \ assert(0); \ From 5d5c0b876c650fe038cebf36cbbd1bee10d9141a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 17 Oct 2022 18:42:07 +0800 Subject: [PATCH 55/63] fix: coverity scan of deadcode when print submit msg --- source/dnode/vnode/src/vnd/vnodeSvr.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 3556b47dd3..d0a00a8b6d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -802,15 +802,11 @@ static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, tInitSubmitBlkIter(msgIter, pBlock, &blkIter); if (blkIter.row == NULL) return 0; - if (!pSchema || (suid != msgIter->suid) || rv != TD_ROW_SVER(blkIter.row)) { - if (pSchema) { - taosMemoryFreeClear(pSchema); - } - pSchema = metaGetTbTSchema(pMeta, msgIter->suid, TD_ROW_SVER(blkIter.row), 1); // TODO: use the real schema - if (pSchema) { - suid = msgIter->suid; - rv = TD_ROW_SVER(blkIter.row); - } + + pSchema = metaGetTbTSchema(pMeta, msgIter->suid, TD_ROW_SVER(blkIter.row), 1); // TODO: use the real schema + if (pSchema) { + suid = msgIter->suid; + rv = TD_ROW_SVER(blkIter.row); } if (!pSchema) { printf("%s:%d no valid schema\n", tags, __LINE__); From 34f576b681357dc5a66bbc1096bef108968c8128 Mon Sep 17 00:00:00 2001 From: Zhiqiang Wang <1296468573@qq.com> Date: Mon, 17 Oct 2022 19:16:11 +0800 Subject: [PATCH 56/63] fix: show help install error (#17408) --- packaging/tools/make_install.sh | 83 ++++++++++++++++---------------- tools/shell/src/shellArguments.c | 2 +- 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index b56a5a203d..d0e94e3eb3 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -168,7 +168,7 @@ function install_bin() { ${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || : - [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : + [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : @@ -182,21 +182,21 @@ function install_bin() { ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : - [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : - [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || : - [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : - [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : - [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : - [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || : - [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : - [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor > /dev/null 2>&1 || : + [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} > /dev/null 2>&1 || : else ${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || : - [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : + [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : @@ -206,14 +206,14 @@ function install_bin() { ${csudo}cp -r ${script_dir}/remove.sh ${install_main_dir}/bin || : ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : - [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : - [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || : - [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : - [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : - [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : - [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || : + [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} > /dev/null 2>&1 || : fi } @@ -238,7 +238,7 @@ function install_jemalloc() { if [ -f "${binary_dir}/build/lib/libjemalloc.so.2" ]; then ${csudo}/usr/bin/install -c -d /usr/local/lib ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.so.2 /usr/local/lib - ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so > /dev/null 2>&1 ${csudo}/usr/bin/install -c -d /usr/local/lib [ -f ${binary_dir}/build/lib/libjemalloc.a ] && ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.a /usr/local/lib @@ -274,8 +274,8 @@ function install_avro() { if [ -f "${binary_dir}/build/$1/libavro.so.23.0.0" ] && [ -d /usr/local/$1 ]; then ${csudo}/usr/bin/install -c -d /usr/local/$1 ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.so.23.0.0 /usr/local/$1 - ${csudo}ln -sf libavro.so.23.0.0 /usr/local/$1/libavro.so.23 - ${csudo}ln -sf libavro.so.23 /usr/local/$1/libavro.so + ${csudo}ln -sf libavro.so.23.0.0 /usr/local/$1/libavro.so.23 > /dev/null 2>&1 + ${csudo}ln -sf libavro.so.23 /usr/local/$1/libavro.so > /dev/null 2>&1 ${csudo}/usr/bin/install -c -d /usr/local/$1 [ -f ${binary_dir}/build/$1/libavro.a ] && ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.a /usr/local/$1 @@ -304,11 +304,11 @@ function install_lib() { ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/libtaos.so.${verNumber} - ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 - ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 > /dev/null 2>&1 + ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so > /dev/null 2>&1 if [ -d "${lib64_link_dir}" ]; then - ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 - ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so + ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 > /dev/null 2>&1 + ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so > /dev/null 2>&1 fi if [ -f ${binary_dir}/build/lib/libtaosws.so ]; then @@ -316,23 +316,23 @@ function install_lib() { ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so ||: - ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : + ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so > /dev/null 2>&1 || : fi else ${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \ ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* ${csudo}ln -sf ${install_main_dir}/driver/libtaos.${verNumber}.dylib \ - ${lib_link_dir}/libtaos.1.dylib || : + ${lib_link_dir}/libtaos.1.dylib > /dev/null 2>&1 || : - ${csudo}ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib || : + ${csudo}ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib > /dev/null 2>&1 || : if [ -f ${binary_dir}/build/lib/libtaosws.dylib ]; then ${csudo}cp ${binary_dir}/build/lib/libtaosws.dylib \ ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.dylib ||: - ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib || : + ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib > /dev/null 2>&1 || : fi fi @@ -346,6 +346,7 @@ function install_lib() { } function install_header() { + ${csudo}mkdir -p ${inc_link_dir} ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h ||: ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ @@ -353,13 +354,13 @@ function install_header() { if [ -f ${binary_dir}/build/include/taosws.h ]; then ${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h ||: - ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h ||: + ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h > /dev/null 2>&1 ||: fi - ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h - ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h - ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h - ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h + ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h > /dev/null 2>&1 + ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h > /dev/null 2>&1 + ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h > /dev/null 2>&1 + ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h > /dev/null 2>&1 ${csudo}chmod 644 ${install_main_dir}/include/* } @@ -374,7 +375,7 @@ function install_config() { ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ ${cfg_install_dir}/${configFile}.${verNumber} ${csudo}ln -s ${cfg_install_dir}/${configFile} \ - ${install_main_dir}/cfg/${configFile} + ${install_main_dir}/cfg/${configFile} > /dev/null 2>&1 else ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ ${cfg_install_dir}/${configFile}.${verNumber} @@ -395,7 +396,7 @@ function install_taosadapter_config() { ${cfg_install_dir}/taosadapter.toml.${verNumber} || : [ -f ${cfg_install_dir}/taosadapter.toml ] && ${csudo}ln -s ${cfg_install_dir}/taosadapter.toml \ - ${install_main_dir}/cfg/taosadapter.toml || : + ${install_main_dir}/cfg/taosadapter.toml > /dev/null 2>&1 || : else if [ -f "${binary_dir}/test/cfg/taosadapter.toml" ]; then ${csudo}cp -f ${binary_dir}/test/cfg/taosadapter.toml \ @@ -408,12 +409,12 @@ function install_taosadapter_config() { function install_log() { ${csudo}rm -rf ${log_dir} || : ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} - ${csudo}ln -s ${log_dir} ${install_main_dir}/log + ${csudo}ln -s ${log_dir} ${install_main_dir}/log > /dev/null 2>&1 } function install_data() { ${csudo}mkdir -p ${data_dir} && ${csudo}chmod 777 ${data_dir} - ${csudo}ln -s ${data_dir} ${install_main_dir}/data + ${csudo}ln -s ${data_dir} ${install_main_dir}/data > /dev/null 2>&1 } function install_connector() { @@ -533,7 +534,7 @@ function install_taosadapter_service() { function install_service_on_launchctl() { ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : ${csudo}cp ${script_dir}/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist - ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist || : + ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : } function install_service() { diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index eedb6d7295..4d40de66bd 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -22,7 +22,7 @@ #define TAOS_CONSOLE_PROMPT_HEADER "taos> " #define TAOS_CONSOLE_PROMPT_CONTINUE " -> " -#define SHELL_HOST "The auth string to use when connecting to the server." +#define SHELL_HOST "TDengine server FQDN to connect. The default host is localhost." #define SHELL_PORT "The TCP/IP port number to use for the connection." #define SHELL_USER "The user name to use when connecting to the server." #define SHELL_PASSWORD "The password to use when connecting to the server." From c73535493d1d012ad8d5c8fd03f4f0106666a184 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 17 Oct 2022 19:32:13 +0800 Subject: [PATCH 57/63] test: add insert rows --- tests/system-test/7-tmq/tmqShow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index c0f33d9204..0691da6786 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -37,7 +37,7 @@ class TDTestCase: 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], 'ctbPrefix': 'ctb', - 'ctbNum': 10, + 'ctbNum': 100, 'rowsPerTbl': 4000, 'batchNum': 15, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 @@ -124,7 +124,8 @@ class TDTestCase: tdLog.info("async insert data") pThread = tmqCom.asyncInsertData(paraDict) - time.sleep(5) + tmqCom.getStartConsumeNotifyFromTmqsim(); + #time.sleep(5) tdLog.info("check show consumers") tdSql.query("show consumers") # tdLog.info(tdSql.queryResult) From 77c4c8fd88e5b40b95b7a2e3597ef2d8e6be3206 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 17 Oct 2022 17:44:41 +0800 Subject: [PATCH 58/63] test: fix failed case --- source/dnode/vnode/src/tq/tqSink.c | 6 +++++- tests/system-test/7-tmq/tmqAutoCreateTbl.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 366cd6268e..9bdcf04e89 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -352,6 +352,7 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { SMetaReader mr = {0}; metaReaderInit(&mr, pVnode->pMeta, 0); if (metaGetTableEntryByName(&mr, ctbName) < 0) { + metaReaderClear(&mr); tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); SVCreateTbReq createTbReq = {0}; @@ -444,6 +445,9 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { createTb = false; uid = mr.me.uid; metaReaderClear(&mr); + + tqDebug("vgId:%d, stream write, table %s, uid %ld already exist, skip create", TD_VID(pVnode), ctbName, uid); + taosMemoryFreeClear(ctbName); } @@ -478,7 +482,7 @@ void tqTableSink1(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { blkHead->schemaLen = htonl(schemaLen); rowData = POINTER_SHIFT(blkSchema, schemaLen); } else { - blkHead->uid = uid; + blkHead->uid = htobe64(uid); } taosMemoryFreeClear(schemaStr); diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index de95fef148..587472ed82 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -42,7 +42,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 500, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 3, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -87,7 +87,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 500, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} From cf9abe7258a4b50d8f037fb5e5fa9de1280e0f09 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 17 Oct 2022 19:48:36 +0800 Subject: [PATCH 59/63] fix: fix coverity check issues --- include/util/tdef.h | 3 +++ source/client/src/clientHb.c | 5 +++- source/dnode/mnode/impl/inc/mndQuery.h | 2 ++ source/dnode/mnode/impl/src/mndProfile.c | 2 +- source/dnode/mnode/impl/src/mndQuery.c | 7 +++++ source/dnode/vnode/src/vnd/vnodeQuery.c | 6 +++++ source/libs/catalog/inc/ctgRemote.h | 2 ++ source/libs/catalog/src/catalog.c | 16 ++++++++---- source/libs/catalog/src/ctgAsync.c | 15 +++-------- source/libs/catalog/src/ctgCache.c | 33 ++++++++++++------------ source/libs/catalog/src/ctgRemote.c | 6 +++++ source/libs/command/inc/commandInt.h | 4 +-- source/libs/command/src/command.c | 1 + source/libs/command/src/explain.c | 1 + source/libs/executor/src/dataDeleter.c | 2 ++ source/libs/executor/src/dataInserter.c | 1 + source/libs/qcom/src/queryUtil.c | 6 +++++ source/libs/qcom/src/querymsg.c | 6 +++-- source/libs/qworker/src/qworker.c | 5 ++-- source/libs/scheduler/src/schJob.c | 1 - source/libs/scheduler/src/schRemote.c | 8 +++--- source/libs/scheduler/src/schTask.c | 24 +++++++++++++---- source/libs/scheduler/src/schUtil.c | 2 +- 23 files changed, 103 insertions(+), 55 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index e6d28ae341..d9bb558b74 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -239,6 +239,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 +#define TSDB_MAX_COL_TAG_NUM (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS) #define TSDB_MAX_JSON_TAG_LEN 16384 #define TSDB_MAX_JSON_KEY_LEN 256 @@ -496,6 +497,8 @@ enum { #define MAX_NUM_STR_SIZE 40 +#define MAX_META_MSG_IN_BATCH 1048576 + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 096e2cdac8..aa725a4988 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -173,7 +173,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; - tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, + tscTrace("conn %d hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, pTscObj->pAppInfo->totalDnodes); if (pRsp->query->killRid) { @@ -440,6 +440,7 @@ int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S } if (userNum <= 0) { + taosMemoryFree(users); return TSDB_CODE_SUCCESS; } @@ -476,6 +477,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl } if (dbNum <= 0) { + taosMemoryFree(dbs); return TSDB_CODE_SUCCESS; } @@ -514,6 +516,7 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC } if (stbNum <= 0) { + taosMemoryFree(stbs); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/mnode/impl/inc/mndQuery.h b/source/dnode/mnode/impl/inc/mndQuery.h index 4b359c0a14..69ec54f330 100644 --- a/source/dnode/mnode/impl/inc/mndQuery.h +++ b/source/dnode/mnode/impl/inc/mndQuery.h @@ -22,6 +22,8 @@ extern "C" { #endif +#define MAX_META_MSG_IN_BATCH 1048576 + int32_t mndInitQuery(SMnode *pMnode); void mndCleanupQuery(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 1909b168bd..d42b66455f 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -369,7 +369,7 @@ static void mndReleaseApp(SMnode *pMnode, SAppObj *pApp) { taosCacheRelease(pMgmt->appCache, (void **)&pApp, false); } -void *mndGetNextApp(SMnode *pMnode, SCacheIter *pIter) { +SAppObj *mndGetNextApp(SMnode *pMnode, SCacheIter *pIter) { SAppObj *pApp = NULL; bool hasNext = taosCacheIterNext(pIter); if (hasNext) { diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index f185cdeaec..2e01fadbae 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -77,6 +77,12 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { void *pRsp = NULL; SMnode *pMnode = pMsg->info.node; + if (msgNum >= MAX_META_MSG_IN_BATCH) { + code = TSDB_CODE_INVALID_MSG; + mError("too many msgs %d in mnode batch meta req", msgNum); + goto _exit; + } + SArray *batchRsp = taosArrayInit(msgNum, sizeof(SBatchRsp)); if (NULL == batchRsp) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -106,6 +112,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { if (fp == NULL) { mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index e18379fe8a..8c1f858cbb 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -272,6 +272,12 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { SRpcMsg rspMsg = {0}; void *pRsp = NULL; + if (msgNum >= MAX_META_MSG_IN_BATCH) { + code = TSDB_CODE_INVALID_MSG; + qError("too many msgs %d in vnode batch meta req", msgNum); + goto _exit; + } + SArray *batchRsp = taosArrayInit(msgNum, sizeof(SBatchRsp)); if (NULL == batchRsp) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/catalog/inc/ctgRemote.h b/source/libs/catalog/inc/ctgRemote.h index 8edd3b3880..746344b7ed 100644 --- a/source/libs/catalog/inc/ctgRemote.h +++ b/source/libs/catalog/inc/ctgRemote.h @@ -20,6 +20,8 @@ extern "C" { #endif +#define CTG_MAX_REQ_IN_BATCH 1048576 + #ifdef __cplusplus } #endif diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b027dbcfa5..b960103d94 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -173,15 +173,19 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput)); } - CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, output, syncReq)); + code = ctgUpdateTbMetaEnqueue(pCtg, output, syncReq); + output = NULL; + CTG_ERR_JRET(code); return TSDB_CODE_SUCCESS; _return: - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); - + if (output) { + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); + } + CTG_RET(code); } @@ -290,7 +294,9 @@ int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp* rspMsg, bool syncOp) { CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, rspMsg->tableType == TSDB_SUPER_TABLE, &output->tbMeta)); } - CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, output, syncOp)); + code = ctgUpdateTbMetaEnqueue(pCtg, output, syncOp); + output = NULL; + CTG_ERR_JRET(code); return TSDB_CODE_SUCCESS; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 1305540a50..1d2e3640a1 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -71,7 +71,7 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", pJob->queryId, taskIdx, - ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); + ctgTaskTypeStr(task.type), (int32_t)taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); return TSDB_CODE_SUCCESS; } @@ -202,7 +202,7 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", pJob->queryId, taskIdx, - ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); + ctgTaskTypeStr(task.type), (int32_t)taosArrayGetSize(ctx->pNames), pJob->tbHashNum); return TSDB_CODE_SUCCESS; } @@ -1056,7 +1056,6 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; @@ -1223,7 +1222,6 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; @@ -1309,7 +1307,6 @@ int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } _return: @@ -1346,7 +1343,6 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } _return: @@ -1382,7 +1378,6 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { @@ -1672,7 +1667,7 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { int32_t baseResIdx = 0; for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); - ctgDebug("start to check tb metas in db %s, tbNum %d", pReq->dbFName, taosArrayGetSize(pReq->pTables)); + ctgDebug("start to check tb metas in db %s, tbNum %d", pReq->dbFName, (int32_t)taosArrayGetSize(pReq->pTables)); CTG_ERR_RET(ctgGetTbMetasFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); } @@ -2010,10 +2005,6 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask* pTask) { _return: - if (dbCache) { - ctgReleaseVgInfoToCache(pCtg, dbCache); - } - CTG_RET(code); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 3a774afdcc..89571ebf55 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -597,6 +597,8 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + taosMemoryFree(operation->data); + taosMemoryFree(operation); CTG_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -648,6 +650,7 @@ int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) SCtgDropDBMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDBMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDBMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -668,7 +671,6 @@ int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -681,6 +683,7 @@ int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) SCtgDropDbVgroupMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDbVgroupMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDbVgroupMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -700,7 +703,6 @@ int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -714,6 +716,7 @@ int32_t ctgDropStbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SCtgDropStbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropStbMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropStbMetaMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -731,7 +734,6 @@ int32_t ctgDropStbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -744,6 +746,7 @@ int32_t ctgDropTbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SCtgDropTblMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTblMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTblMetaMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -760,7 +763,6 @@ int32_t ctgDropTbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -773,6 +775,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); + taosMemoryFree(op); ctgFreeVgInfo(dbInfo); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -796,8 +799,6 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId _return: ctgFreeVgInfo(dbInfo); - taosMemoryFreeClear(op->data); - taosMemoryFreeClear(op); CTG_RET(code); } @@ -810,6 +811,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool sy SCtgUpdateTbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbMetaMsg)); + taosMemoryFree(op); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -834,8 +836,6 @@ _return: taosMemoryFree(output); } - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -847,6 +847,7 @@ int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEp SCtgUpdateEpsetMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateEpsetMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateEpsetMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -863,8 +864,6 @@ int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEp _return: - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -877,6 +876,7 @@ int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -892,7 +892,6 @@ int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp _return: tFreeSGetUserAuthRsp(pAuth); - taosMemoryFreeClear(msg); CTG_RET(code); } @@ -906,6 +905,7 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog *pCtg, STableIndex **pIndex, bool syncO SCtgUpdateTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbIndexMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbIndexMsg)); + taosMemoryFree(op); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -923,7 +923,6 @@ _return: taosArrayDestroyEx((*pIndex)->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(*pIndex); - taosMemoryFreeClear(msg); CTG_RET(code); } @@ -937,6 +936,7 @@ int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { SCtgDropTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTbIndexMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbIndexMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -952,8 +952,6 @@ int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { _return: - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -968,6 +966,7 @@ int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool freeCtg, bool stopQueue, bool SCtgClearCacheMsg *msg = taosMemoryMalloc(sizeof(SCtgClearCacheMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgClearCacheMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -981,8 +980,6 @@ int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool freeCtg, bool stopQueue, bool _return: - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -1483,7 +1480,9 @@ int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool sync int32_t code = 0; CTG_ERR_RET(ctgCloneMetaOutput(pOut, &pOutput)); - CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, pOutput, syncReq)); + code = ctgUpdateTbMetaEnqueue(pCtg, pOutput, syncReq); + pOutput = NULL; + CTG_ERR_JRET(code); return TSDB_CODE_SUCCESS; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 744d6eab06..23bccb0835 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -19,6 +19,7 @@ #include "tname.h" #include "tref.h" #include "trpc.h" +#include "ctgRemote.h" int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; @@ -579,6 +580,11 @@ int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg) { int32_t num = taosArrayGetSize(pBatch->pMsgs); SBatchReq* pBatchReq = (SBatchReq*)(*msg); + if (num >= CTG_MAX_REQ_IN_BATCH) { + qError("too many msgs %d in one batch request", num); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + pBatchReq->header.vgId = htonl(vgId); pBatchReq->msgNum = htonl(num); offset += sizeof(SBatchReq); diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index ad677a81ff..ce4c4cf09b 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -89,8 +89,8 @@ extern "C" { #define EXPLAIN_STRING_TYPE_FORMAT "%s" #define EXPLAIN_INPUT_ORDER_FORMAT "input_order=%s" #define EXPLAIN_OUTPUT_ORDER_TYPE_FORMAT "output_order=%s" -#define EXPLAIN_OFFSET_FORMAT "offset=%d" -#define EXPLAIN_SOFFSET_FORMAT "soffset=%d" +#define EXPLAIN_OFFSET_FORMAT "offset=%" PRId64 +#define EXPLAIN_SOFFSET_FORMAT "soffset=%" PRId64 #define EXPLAIN_PARTITIONS_FORMAT "partitions=%d" #define COMMAND_RESET_LOG "resetLog" diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 18d839e109..a9134de537 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -485,6 +485,7 @@ static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRs SSDataBlock* pBlock = buildCreateTbResultDataBlock(); int32_t code = setCreateTBResultIntoDataBlock(pBlock, pStmt->pDbCfg, pStmt->tableName, pStmt->pTableCfg); if (code) { + blockDataDestroy(pBlock); return code; } return buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp); diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index c5de44e1f6..e0a4218243 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1598,6 +1598,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); + blockDataDestroy(pBlock); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 1ade4b89e6..55978855d1 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -267,6 +267,8 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData taosThreadMutexInit(&deleter->mutex, NULL); if (NULL == deleter->pDataBlocks) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + destroyDataSinker((SDataSinkHandle*)deleter); + taosMemoryFree(deleter); return TSDB_CODE_QRY_OUT_OF_MEMORY; } *pHandle = deleter; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 6a43ea40c6..18923ffb67 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -323,6 +323,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat int32_t code = tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); if (code) { + destroyDataSinker((SDataSinkHandle*)pInserterNode); return code; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 8162b922ce..a917397d02 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -424,6 +424,12 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } + if ((pSrc->tableInfo.numOfColumns + pSrc->tableInfo.numOfTags) > TSDB_MAX_COL_TAG_NUM) { + *pDst = NULL; + qError("too many column and tag num:%d,%d", pSrc->tableInfo.numOfColumns, pSrc->tableInfo.numOfTags); + return TSDB_CODE_INVALID_PARA; + } + int32_t metaSize = sizeof(STableMeta) + (pSrc->tableInfo.numOfColumns + pSrc->tableInfo.numOfTags) * sizeof(SSchema); *pDst = taosMemoryMalloc(metaSize); if (NULL == *pDst) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 80ba356e8f..a6f26088de 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -271,8 +271,8 @@ int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t SBuildTableInput *pInput = input; STableCfgReq cfgReq = {0}; cfgReq.header.vgId = pInput->vgId; - strcpy(cfgReq.dbFName, pInput->dbFName); - strcpy(cfgReq.tbName, pInput->tbName); + strncpy(cfgReq.dbFName, pInput->dbFName, sizeof(cfgReq.dbFName)); + strncpy(cfgReq.tbName, pInput->tbName, sizeof(cfgReq.tbName)); int32_t bufLen = tSerializeSTableCfgReq(NULL, 0, &cfgReq); void *pBuf = (*mallcFp)(bufLen); @@ -615,6 +615,8 @@ int32_t queryProcessGetTbCfgRsp(void *output, char *msg, int32_t msgSize) { STableCfgRsp *out = taosMemoryCalloc(1, sizeof(STableCfgRsp)); if (tDeserializeSTableCfgRsp(msg, msgSize, out) != 0) { qError("tDeserializeSTableCfgRsp failed, msgSize:%d", msgSize); + tFreeSTableCfgRsp(out); + taosMemoryFree(out); return TSDB_CODE_INVALID_MSG; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index d2c16a1cb5..0ff40f3b9a 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -258,7 +258,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); if (len < 0) { - QW_TASK_ELOG("invalid length from dsGetDataLength, length:%d", len); + QW_TASK_ELOG("invalid length from dsGetDataLength, length:%" PRId64, len); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -292,7 +292,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, } // Got data from sink - QW_TASK_DLOG("there are data in sink, dataLength:%d", len); + QW_TASK_DLOG("there are data in sink, dataLength:%" PRId64, len); *dataLen += len; @@ -408,7 +408,6 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu // QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); - break; } QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC)); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index bb8451ba37..8fcfdce2b2 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -122,7 +122,6 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_DROP: SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); - break; default: SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus)); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1b59c06140..c26ae4e646 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -169,7 +169,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) { SCH_LOCK(SCH_WRITE, &pJob->resLock); if (NULL == pJob->execRes.res) { - pJob->execRes.res = taosArrayInit(batchRsp.nRsps, POINTER_BYTES); + pJob->execRes.res = (void*)taosArrayInit(batchRsp.nRsps, POINTER_BYTES); pJob->execRes.msgType = TDMT_VND_CREATE_TABLE; } @@ -386,7 +386,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa // NEVER REACH HERE SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:0x%" PRIx64, pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); - break; } case TDMT_SCH_LINK_BROKEN: SCH_TASK_ELOG("link broken received, error:%x - %s", rspCode, tstrerror(rspCode)); @@ -981,7 +980,7 @@ _return: } int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { - uint32_t msgSize = 0; + int32_t msgSize = 0; void *msg = NULL; int32_t code = 0; bool isCandidateAddr = false; @@ -1133,12 +1132,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, default: SCH_TASK_ELOG("unknown msg type to send, msgType:%d", msgType); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); - break; } #if 1 SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; - code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); + code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); msg = NULL; SCH_ERR_JRET(code); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 33bf50d853..c641b88152 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -904,7 +904,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { } else if (tsQueryPlannerTrace) { char *msg = NULL; int32_t msgLen = 0; - qSubPlanToString(plan, &msg, &msgLen); + SCH_ERR_RET(qSubPlanToString(plan, &msg, &msgLen)); SCH_TASK_DLOGL("physical plan len:%d, %s", msgLen, msg); taosMemoryFree(msg); } @@ -926,6 +926,7 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { } SArray *explainRes = NULL; + int32_t code = 0; SQWMsg qwMsg = {0}; qwMsg.msgInfo.taskType = TASK_TYPE_TEMP; qwMsg.msgInfo.explain = SCH_IS_EXPLAIN_JOB(pJob); @@ -938,14 +939,21 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); } - SCH_ERR_RET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, + SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); + explainRes = NULL; } - SCH_RET(schProcessOnTaskSuccess(pJob, pTask)); + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + +_return: + + taosArrayDestroy(explainRes); + + SCH_RET(code); } int32_t schLaunchTaskImpl(void *param) { @@ -1097,22 +1105,28 @@ int32_t schExecRemoteFetch(SSchJob *pJob, SSchTask *pTask) { int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { void *pRsp = NULL; + int32_t code = 0; SArray *explainRes = NULL; if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); } - SCH_ERR_RET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, + SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); + explainRes = NULL; } SCH_ERR_RET(schProcessFetchRsp(pJob, pTask, pRsp, TSDB_CODE_SUCCESS)); - return TSDB_CODE_SUCCESS; +_return: + + taosArrayDestroy(explainRes); + + SCH_RET(code); } // Note: no more error processing, handled in function internal diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 7099dbb2c0..e20ed9d58e 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -242,7 +242,7 @@ uint64_t schGenUUID(void) { if (hashId == 0) { char uid[64] = {0}; - int32_t code = taosGetSystemUUID(uid, tListLen(uid)); + int32_t code = taosGetSystemUUID(uid, tListLen(uid) - 1); if (code != TSDB_CODE_SUCCESS) { qError("Failed to get the system uid, reason:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } else { From 5ff8bb5bc0b8af92bd0c62b53646b50ac60cb145 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 17 Oct 2022 19:56:59 +0800 Subject: [PATCH 60/63] fix invalid packet --- source/libs/transport/src/transComm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index f843c8559c..4ae1a1083a 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -128,10 +128,10 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) { static const int HEADSIZE = sizeof(STransMsgHead); SConnBuffer* p = connBuf; - if (p->left != 0) { + if (p->left != 0 || p->total <= 0) { return -1; } - int total = connBuf->total; + int total = p->total; if (total >= HEADSIZE && !p->invalid) { *buf = taosMemoryCalloc(1, total); memcpy(*buf, p->buf, total); From 34c7cda3566ffe675d79fd38072929cee907f054 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 17 Oct 2022 20:25:11 +0800 Subject: [PATCH 61/63] test: change sleep with commit signal --- tests/system-test/7-tmq/subscribeDb4.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index 63a59c0dc3..d253c34a53 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -41,7 +41,7 @@ class TDTestCase: ifcheckdata = 0 ifManualCommit = 1 groupId = 'group.id:cgrp1' - autoCommit = 'enable.auto.commit:false' + autoCommit = 'enable.auto.commit:true' autoCommitInterval = 'auto.commit.interval.ms:1000' autoOffset = 'auto.offset.reset:earliest' @@ -87,8 +87,9 @@ class TDTestCase: tdLog.info("start consume processor") tmqCom.startTmqSimProcess(self.pollDelay,self.paraDict["dbName"],self.showMsg, self.showRow,self.cdbName) - tdLog.info("After waiting for a period of time, drop one stable") - time.sleep(3) + tdLog.info("After waiting for a commit notify, drop one stable") + #time.sleep(3) + tmqCom.getStartCommitNotifyFromTmqsim() tdSql.execute("drop table %s.%s" %(self.paraDict['dbName'], self.paraDict['stbName'])) tdLog.info("wait result from consumer, then check it") From 951857430ef90c19d929644042a4123ad46b3805 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Oct 2022 10:21:19 +0800 Subject: [PATCH 62/63] test: comment out unstable case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 39adfc9d53..4c0ef5a527 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -248,7 +248,7 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 - python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 From bb59553e39312176f86b353a5161d006b91f380a Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Tue, 18 Oct 2022 11:23:24 +0800 Subject: [PATCH 63/63] Update 01-arch.md --- docs/en/21-tdinternal/01-arch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md index a938084eb1..697ecb98a1 100644 --- a/docs/en/21-tdinternal/01-arch.md +++ b/docs/en/21-tdinternal/01-arch.md @@ -222,12 +222,12 @@ TDengine provides a variety of query processing functions for tables and STables ### Query Process -1. TDEngine client driver `taosc` parses the SQL statement and generates an abstract syntax tree (AST), then check and verify the AST according to metadata. During this stage, the metadata management module in `taosc` (Catalog) requests the metadata of the involved database and table from mnode and vnode. +1. TDengine client driver `taosc` parses the SQL statement and generates an abstract syntax tree (AST), then checks and verifies the AST according to metadata. During this stage, the metadata management module in `taosc` (Catalog) requests the metadata of the involved database and table from mnode and vnode. 2. After the verification passes, `taosc` generates distributed query plan and optimizes the plan. 3. `taosc` schedules the tasks according to configured query policy, a query sub-task may be scheduled to a vnode or qnode according to data relative and system load. Please be noted that both vnode and qnode are logic execution unit, the physical execution node is dnode (data node). 4. When a dnode receives a query request, it identifies which vnode or qnode this query request is targeted, and forwards the request to the query execution queue of the identified vnode or qnode. 5. The query execution thread of the vnode or qnode establishes fundamental query execution context, and executes the query, and notifies the client once obtaining a part of result data. -6. TDengine client driver `taosc` will initiates next level query tasks or obtain the result simply. +6. TDengine client driver `taosc` will initiate next level query tasks or obtain the result simply. ### Aggregation by Time Axis, Downsampling, Interpolation