From 1441684c03f5a6e6bab5749af706c8d300fe0024 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 17 Nov 2022 09:29:27 +0800 Subject: [PATCH 01/95] cleanup: remove unused pager open --- source/libs/tdb/src/db/tdbPager.c | 82 ++++++------------------------- 1 file changed, 16 insertions(+), 66 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index abbad06515..c4346f62f4 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -28,12 +28,12 @@ typedef struct { TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); struct hashset_st { - size_t nbits; - size_t mask; - size_t capacity; + size_t nbits; + size_t mask; + size_t capacity; size_t *items; - size_t nitems; - double load_factor; + size_t nitems; + double load_factor; }; static const unsigned int prime = 39; @@ -68,11 +68,11 @@ void hashset_destroy(hashset_t set) { } int hashset_add_member(hashset_t set, void *item) { - size_t value = (size_t) item; + size_t value = (size_t)item; size_t h; if (value == 0) { - return -1; + return -1; } for (h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { @@ -103,7 +103,7 @@ int hashset_add(hashset_t set, void *item) { set->nitems = 0; for (size_t i = 0; i < old_capacity; ++i) { - hashset_add_member(set, (void*)old_items[i]); + hashset_add_member(set, (void *)old_items[i]); } tdbOsFree(old_items); } @@ -112,7 +112,7 @@ int hashset_add(hashset_t set, void *item) { } int hashset_remove(hashset_t set, void *item) { - size_t value = (size_t) item; + size_t value = (size_t)item; for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { if (set->items[h] == value) { @@ -126,7 +126,7 @@ int hashset_remove(hashset_t set, void *item) { } int hashset_contains(hashset_t set, void *item) { - size_t value = (size_t) item; + size_t value = (size_t)item; for (size_t h = set->mask & (prime * value); set->items[h] != 0; h = set->mask & (h + prime2)) { if (set->items[h] == value) { @@ -226,58 +226,7 @@ int tdbPagerClose(SPager *pPager) { } return 0; } -/* -int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) { - SPgno pgno; - SPage *pPage; - int ret; - if (pPager->dbOrigSize > 0) { - pgno = 1; - } else { - pgno = 0; - } - - { - // TODO: try to search the main DB to get the page number - // pgno = 0; - } - - if (pgno == 0 && toCreate) { - // allocate a new child page - TXN txn; - tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); - - pPager->inTran = 1; - - SBtreeInitPageArg zArg; - zArg.flags = 0x1 | 0x2; // root leaf node; - zArg.pBt = pBt; - ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, &txn); - if (ret < 0) { - return -1; - } - - // ret = tdbPagerAllocPage(pPager, &pPage, &pgno); - // if (ret < 0) { - // return -1; - //} - - // TODO: Need to zero the page - - ret = tdbPagerWrite(pPager, pPage); - if (ret < 0) { - tdbError("failed to write page since %s", terrstr()); - return -1; - } - - tdbTxnClose(&txn); - } - - *ppgno = pgno; - return 0; -} -*/ int tdbPagerWrite(SPager *pPager, SPage *pPage) { int ret; SPage **ppPage; @@ -319,7 +268,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage); // Write page to journal if neccessary - if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize && (pPager->jPageSet == NULL || !hashset_contains(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))))) { + if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize && + (pPager->jPageSet == NULL || !hashset_contains(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { tdbError("failed to write page to journal since %s", tstrerror(terrno)); @@ -327,7 +277,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { } if (pPager->jPageSet) { - hashset_add(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + hashset_add(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } } @@ -391,7 +341,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); if (pPager->jPageSet) { - hashset_remove(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } tdbPCacheRelease(pPager->pCache, pPage, pTxn); } @@ -503,7 +453,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { return -1; } - u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); + u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); if (pageBuf == NULL) { return -1; } @@ -560,7 +510,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); - hashset_remove(pPager->jPageSet, (void*)((long)TDB_PAGE_PGNO(pPage))); + hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } From 78335f83d2381d10e200e420d22145572dc9184c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 17 Nov 2022 11:54:07 +0800 Subject: [PATCH 02/95] tdb/api: migrate txn related api changes of meta, sma, tq, and stream meta/state --- include/libs/stream/streamState.h | 2 +- include/libs/stream/tstream.h | 2 +- source/dnode/vnode/src/inc/meta.h | 4 +- source/dnode/vnode/src/meta/metaCommit.c | 21 ++-- source/dnode/vnode/src/meta/metaSma.c | 8 +- source/dnode/vnode/src/meta/metaTable.c | 66 ++++++------- source/dnode/vnode/src/tq/tqMeta.c | 61 +++++------- source/dnode/vnode/src/tq/tqSnapshot.c | 14 +-- source/dnode/vnode/src/tq/tqStreamStateSnap.c | 13 ++- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 13 ++- source/libs/stream/src/streamMeta.c | 39 ++++---- source/libs/stream/src/streamState.c | 54 +++++------ source/libs/tdb/inc/tdb.h | 3 +- source/libs/tdb/src/db/tdbBtree.c | 47 ++++++--- source/libs/tdb/src/db/tdbDb.c | 21 +++- source/libs/tdb/src/db/tdbTable.c | 2 +- source/libs/tdb/src/db/tdbTxn.c | 8 +- source/libs/tdb/src/inc/tdbInt.h | 14 +-- source/libs/tdb/test/tdbExOVFLTest.cpp | 54 +++++------ source/libs/tdb/test/tdbTest.cpp | 96 ++++++++----------- 20 files changed, 274 insertions(+), 268 deletions(-) diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 9443df5e14..13dc0be75c 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -35,7 +35,7 @@ typedef struct { TTB* pFuncStateDb; TTB* pFillStateDb; // todo refactor TTB* pSessionStateDb; - TXN txn; + TXN* txn; int32_t number; } SStreamState; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index ecd1b6f916..7ca299d312 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -562,7 +562,7 @@ typedef struct SStreamMeta { SHashObj* pTasks; SHashObj* pRecoverStatus; void* ahandle; - TXN txn; + TXN* txn; FTaskExpand* expandFunc; int32_t vgId; SRWLatch lock; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 2ceae91f7c..11efd63d10 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -70,7 +70,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid); int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo); int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid); int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); -void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t delta); +void metaUpdateStbStats(SMeta* pMeta, int64_t uid, int64_t delta); struct SMeta { TdThreadRwlock lock; @@ -78,7 +78,7 @@ struct SMeta { char* path; SVnode* pVnode; TDB* pEnv; - TXN txn; + TXN* txn; TTB* pTbDb; TTB* pSkmDb; TTB* pUidIdx; diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index 0be0c3e407..c46c219ac3 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -20,12 +20,19 @@ static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVB // begin a meta txn int metaBegin(SMeta *pMeta, int8_t fromSys) { + void *(*xMalloc)(void *, size_t); + void (*xFree)(void *, void *); + void *xArg = NULL; + if (fromSys) { - tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + xMalloc = tdbDefaultMalloc; + xFree = tdbDefaultFree; } else { - tdbTxnOpen(&pMeta->txn, 0, metaMalloc, metaFree, pMeta->pVnode->inUse, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + xMalloc = metaMalloc; + xFree = metaFree; + xArg = pMeta->pVnode->inUse; } - if (tdbBegin(pMeta->pEnv, &pMeta->txn) < 0) { + if (tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } @@ -33,9 +40,9 @@ int metaBegin(SMeta *pMeta, int8_t fromSys) { } // commit the meta txn -int metaCommit(SMeta *pMeta) { return tdbCommit(pMeta->pEnv, &pMeta->txn); } -int metaFinishCommit(SMeta *pMeta) { return tdbPostCommit(pMeta->pEnv, &pMeta->txn); } -int metaPrepareAsyncCommit(SMeta *pMeta) { return tdbPrepareAsyncCommit(pMeta->pEnv, &pMeta->txn); } +int metaCommit(SMeta *pMeta) { return tdbCommit(pMeta->pEnv, pMeta->txn); } +int metaFinishCommit(SMeta *pMeta) { return tdbPostCommit(pMeta->pEnv, pMeta->txn); } +int metaPrepareAsyncCommit(SMeta *pMeta) { return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); } // abort the meta txn -int metaAbort(SMeta *pMeta) { return tdbAbort(pMeta->pEnv, &pMeta->txn); } +int metaAbort(SMeta *pMeta) { return tdbAbort(pMeta->pEnv, pMeta->txn); } diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 3ada7d1814..2e3cbb97cf 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -117,7 +117,7 @@ static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, pMeta->txn) < 0) { goto _err; } @@ -131,17 +131,17 @@ _err: static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { SUidIdxVal uidIdxVal = {.suid = pME->smaEntry.tsma->indexUid, .version = pME->version, .skmVer = 0}; - return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), &pMeta->txn); + return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); } static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) { SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid}; - return tdbTbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, pMeta->txn); } static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 5921adfbfa..16302350ba 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -261,7 +261,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb // drop all child tables TBC *pCtbIdxc = NULL; - tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); if (rc < 0) { tdbTbcClose(pCtbIdxc); @@ -295,10 +295,10 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb _drop_super_table: tdbTbGet(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pData, &nData); tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = ((SUidIdxVal *)pData)[0].version, .uid = pReq->suid}, - sizeof(STbDbKey), &pMeta->txn); - tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pMeta->txn); - tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn); - tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn); + sizeof(STbDbKey), pMeta->txn); + tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, pMeta->txn); + tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); + tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); metaULock(pMeta); @@ -321,7 +321,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t ret; int32_t c = -2; - tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); if (ret < 0 || c) { tdbTbcClose(pUidIdxc); @@ -340,7 +340,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { oversion = ((SUidIdxVal *)pData)[0].version; - tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(ret == 0 && c == 0); @@ -589,7 +589,7 @@ static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { STtlIdxKey ttlKey = {0}; metaBuildTtlIdxKey(&ttlKey, pME); if (ttlKey.dtime == 0) return 0; - return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), &pMeta->txn); + return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), pMeta->txn); } static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { @@ -651,7 +651,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid, &pTagIdxKey, &nTagIdxKey) == 0) { - tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, &pMeta->txn); + tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); } metaDestroyTagIdxKey(pTagIdxKey); } @@ -661,9 +661,9 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { } } - tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pMeta->txn); - tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, &pMeta->txn); - tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn); + tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), pMeta->txn); + tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, pMeta->txn); + tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), pMeta->txn); if (e.type == TSDB_CHILD_TABLE || e.type == TSDB_NORMAL_TABLE) metaDeleteCtimeIdx(pMeta, &e); if (e.type == TSDB_NORMAL_TABLE) metaDeleteNcolIdx(pMeta, &e); @@ -671,7 +671,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e); if (e.type == TSDB_CHILD_TABLE) { - tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), &pMeta->txn); + tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), pMeta->txn); --pMeta->pVnode->config.vndStats.numOfCTables; @@ -682,7 +682,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { --pMeta->pVnode->config.vndStats.numOfNTables; pMeta->pVnode->config.vndStats.numOfNTimeSeries -= e.ntbEntry.schemaRow.nCols - 1; } else if (e.type == TSDB_SUPER_TABLE) { - tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), &pMeta->txn); + tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), pMeta->txn); // drop schema.db (todo) metaStatsCacheDrop(pMeta, uid); @@ -702,7 +702,7 @@ int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) { return 0; } - return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn); } int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -710,14 +710,14 @@ int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) { return 0; } - return tdbTbDelete(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), &pMeta->txn); + return tdbTbDelete(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), pMeta->txn); } int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { SNcolIdxKey ncolKey = {0}; if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { return 0; } - return tdbTbInsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn); } int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -725,7 +725,7 @@ int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { return 0; } - return tdbTbDelete(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), &pMeta->txn); + return tdbTbDelete(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), pMeta->txn); } static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) { @@ -760,7 +760,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // search uid index TBC *pUidIdxc = NULL; - tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); @@ -770,7 +770,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // search table.db TBC *pTbDbc = NULL; - tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); @@ -951,7 +951,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA // search uid index TBC *pUidIdxc = NULL; - tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); @@ -964,7 +964,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA SDecoder dc2 = {0}; /* get ctbEntry */ - tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); @@ -1062,7 +1062,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ASSERT(ctbEntry.ctbEntry.pTags); SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, - ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); + ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn); metaULock(pMeta); @@ -1110,7 +1110,7 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p // search uid index TBC *pUidIdxc = NULL; - tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); @@ -1120,7 +1120,7 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p // search table.db TBC *pTbDbc = NULL; - tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); @@ -1227,7 +1227,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, pMeta->txn) < 0) { goto _err; } @@ -1250,29 +1250,29 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { SUidIdxVal uidIdxVal = {.suid = info.suid, .version = info.version, .skmVer = info.skmVer}; - return tdbTbUpsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), &pMeta->txn); + return tdbTbUpsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), pMeta->txn); } static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); } static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { STtlIdxKey ttlKey = {0}; metaBuildTtlIdxKey(&ttlKey, pME); if (ttlKey.dtime == 0) return 0; - return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, - ((STag *)(pME->ctbEntry.pTags))->len, &pMeta->txn); + ((STag *)(pME->ctbEntry.pTags))->len, pMeta->txn); } int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, @@ -1364,7 +1364,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { ret = -1; goto end; } - tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); + tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); } end: metaDestroyTagIdxKey(pTagIdxKey); @@ -1411,7 +1411,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderInit(&coder, pVal, vLen); tEncodeSSchemaWrapper(&coder, pSW); - if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, pMeta->txn) < 0) { rcode = -1; goto _exit; } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index a15d19fbe1..c4098f68e6 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -108,24 +108,22 @@ int32_t tqMetaClose(STQ* pTq) { } int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen) { - TXN txn; - if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + TXN* txn; + + if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < + 0) { return -1; } - if (tdbBegin(pTq->pMetaDB, &txn) < 0) { + if (tdbTbUpsert(pTq->pCheckStore, key, strlen(key), value, vLen, txn) < 0) { return -1; } - if (tdbTbUpsert(pTq->pCheckStore, key, strlen(key), value, vLen, &txn) < 0) { + if (tdbCommit(pTq->pMetaDB, txn) < 0) { return -1; } - if (tdbCommit(pTq->pMetaDB, &txn) < 0) { - return -1; - } - - if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { return -1; } @@ -133,25 +131,22 @@ int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_ } int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) { - TXN txn; + TXN* txn; - if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < + 0) { ASSERT(0); } - if (tdbBegin(pTq->pMetaDB, &txn) < 0) { - ASSERT(0); - } - - if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), &txn) < 0) { + if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), txn) < 0) { /*ASSERT(0);*/ } - if (tdbCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } - if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } @@ -219,25 +214,22 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { ASSERT(0); } - TXN txn; + TXN* txn; - if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < + 0) { ASSERT(0); } - if (tdbBegin(pTq->pMetaDB, &txn) < 0) { + if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, txn) < 0) { ASSERT(0); } - if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, &txn) < 0) { + if (tdbCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } - if (tdbCommit(pTq->pMetaDB, &txn) < 0) { - ASSERT(0); - } - - if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } @@ -247,25 +239,22 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { } int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { - TXN txn; + TXN* txn; - if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < + 0) { ASSERT(0); } - if (tdbBegin(pTq->pMetaDB, &txn) < 0) { - ASSERT(0); - } - - if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), &txn) < 0) { + if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), txn) < 0) { /*ASSERT(0);*/ } - if (tdbCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } - if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index b68763867e..d811d943ed 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -129,7 +129,7 @@ struct STqSnapWriter { STQ* pTq; int64_t sver; int64_t ever; - TXN txn; + TXN* txn; }; int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) { @@ -146,8 +146,10 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p pWriter->sver = sver; pWriter->ever = ever; - if (tdbTxnOpen(&pWriter->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { - ASSERT(0); + if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + code = -1; + taosMemoryFree(pWriter); + goto _err; } *ppWriter = pWriter; @@ -165,11 +167,11 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { STQ* pTq = pWriter->pTq; if (rollback) { - tdbAbort(pWriter->pTq->pMetaDB, &pWriter->txn); + tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn); } else { - code = tdbCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; - code = tdbPostCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; } diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 08d5931bc3..b1f00bdf74 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -129,7 +129,7 @@ struct STqSnapWriter { STQ* pTq; int64_t sver; int64_t ever; - TXN txn; + TXN* txn; }; int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) { @@ -146,8 +146,10 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p pWriter->sver = sver; pWriter->ever = ever; - if (tdbTxnOpen(&pWriter->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { - ASSERT(0); + if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + code = -1; + taosMemoryFree(pWriter); + goto _err; } *ppWriter = pWriter; @@ -165,11 +167,12 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { STQ* pTq = pWriter->pTq; if (rollback) { + tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn); ASSERT(0); } else { - code = tdbCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; - code = tdbPostCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; } diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index 31e44a5b6d..305378bc93 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -129,7 +129,7 @@ struct STqSnapWriter { STQ* pTq; int64_t sver; int64_t ever; - TXN txn; + TXN* txn; }; int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) { @@ -146,8 +146,10 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p pWriter->sver = sver; pWriter->ever = ever; - if (tdbTxnOpen(&pWriter->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { - ASSERT(0); + if (tdbBegin(pTq->pMetaStore, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + code = -1; + taosMemoryFree(pWriter); + goto _err; } *ppWriter = pWriter; @@ -165,11 +167,12 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { STQ* pTq = pWriter->pTq; if (rollback) { + tdbAbort(pWriter->pTq->pMetaStore, pWriter->txn); ASSERT(0); } else { - code = tdbCommit(pWriter->pTq->pMetaStore, &pWriter->txn); + code = tdbCommit(pWriter->pTq->pMetaStore, pWriter->txn); if (code) goto _err; - code = tdbPostCommit(pWriter->pTq->pMetaStore, &pWriter->txn); + code = tdbPostCommit(pWriter->pTq->pMetaStore, pWriter->txn); if (code) goto _err; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a864814a74..edc16b6062 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -69,8 +69,8 @@ _err: } void streamMetaClose(SStreamMeta* pMeta) { - tdbCommit(pMeta->db, &pMeta->txn); - tdbPostCommit(pMeta->db, &pMeta->txn); + tdbCommit(pMeta->db, pMeta->txn); + tdbPostCommit(pMeta->db, pMeta->txn); tdbTbClose(pMeta->pTaskDb); tdbTbClose(pMeta->pCheckpointDb); tdbClose(pMeta->db); @@ -115,7 +115,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, goto FAIL; } - if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, &pMeta->txn) < 0) { + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) { taosHashRemove(pMeta->pTasks, &pTask->taskId, sizeof(int32_t)); ASSERT(0); goto FAIL; @@ -152,7 +152,7 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { tEncodeSStreamTask(&encoder, pTask); tEncoderClear(&encoder); - if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), buf, len, &pMeta->txn) < 0) { + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) { ASSERT(0); return -1; } @@ -226,7 +226,7 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING); - if (tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), &pMeta->txn) < 0) { + if (tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn) < 0) { /*return -1;*/ } @@ -249,42 +249,35 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { } int32_t streamMetaBegin(SStreamMeta* pMeta) { - if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { - return -1; - } - - if (tdbBegin(pMeta->db, &pMeta->txn) < 0) { + if (tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } return 0; } int32_t streamMetaCommit(SStreamMeta* pMeta) { - if (tdbCommit(pMeta->db, &pMeta->txn) < 0) { + if (tdbCommit(pMeta->db, pMeta->txn) < 0) { return -1; } - memset(&pMeta->txn, 0, sizeof(TXN)); - if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { + if (tdbPostCommit(pMeta->db, pMeta->txn) < 0) { return -1; } - if (tdbBegin(pMeta->db, &pMeta->txn) < 0) { + + if (tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } return 0; } int32_t streamMetaAbort(SStreamMeta* pMeta) { - if (tdbAbort(pMeta->db, &pMeta->txn) < 0) { + if (tdbAbort(pMeta->db, pMeta->txn) < 0) { return -1; } - memset(&pMeta->txn, 0, sizeof(TXN)); - if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { - return -1; - } - if (tdbBegin(pMeta->db, &pMeta->txn) < 0) { + + if (tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } return 0; diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index aefe30116b..a20efc68ab 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -163,8 +163,8 @@ _err: } void streamStateClose(SStreamState* pState) { - tdbCommit(pState->db, &pState->txn); - tdbPostCommit(pState->db, &pState->txn); + tdbCommit(pState->db, pState->txn); + tdbPostCommit(pState->db, pState->txn); tdbTbClose(pState->pStateDb); tdbTbClose(pState->pFuncStateDb); tdbTbClose(pState->pFillStateDb); @@ -175,71 +175,61 @@ void streamStateClose(SStreamState* pState) { } int32_t streamStateBegin(SStreamState* pState) { - if (tdbTxnOpen(&pState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { - return -1; - } - - if (tdbBegin(pState->db, &pState->txn) < 0) { - tdbTxnClose(&pState->txn); + if (tdbBegin(pState->db, &pState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + tdbAbort(pState->db, pState->txn); return -1; } return 0; } int32_t streamStateCommit(SStreamState* pState) { - if (tdbCommit(pState->db, &pState->txn) < 0) { + if (tdbCommit(pState->db, pState->txn) < 0) { return -1; } - if (tdbPostCommit(pState->db, &pState->txn) < 0) { + if (tdbPostCommit(pState->db, pState->txn) < 0) { return -1; } - memset(&pState->txn, 0, sizeof(TXN)); - if (tdbTxnOpen(&pState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { - return -1; - } - if (tdbBegin(pState->db, &pState->txn) < 0) { + + if (tdbBegin(pState->db, &pState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } return 0; } int32_t streamStateAbort(SStreamState* pState) { - if (tdbAbort(pState->db, &pState->txn) < 0) { + if (tdbAbort(pState->db, pState->txn) < 0) { return -1; } - memset(&pState->txn, 0, sizeof(TXN)); - if (tdbTxnOpen(&pState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { - return -1; - } - if (tdbBegin(pState->db, &pState->txn) < 0) { + + if (tdbBegin(pState->db, &pState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } return 0; } int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen) { - return tdbTbUpsert(pState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, &pState->txn); + return tdbTbUpsert(pState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, pState->txn); } int32_t streamStateFuncGet(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen) { return tdbTbGet(pState->pFuncStateDb, key, sizeof(STupleKey), pVal, pVLen); } int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) { - return tdbTbDelete(pState->pFuncStateDb, key, sizeof(STupleKey), &pState->txn); + return tdbTbDelete(pState->pFuncStateDb, key, sizeof(STupleKey), pState->txn); } // todo refactor int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbUpsert(pState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, &pState->txn); + return tdbTbUpsert(pState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, pState->txn); } // todo refactor int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { - return tdbTbUpsert(pState->pFillStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); + return tdbTbUpsert(pState->pFillStateDb, key, sizeof(SWinKey), value, vLen, pState->txn); } // todo refactor @@ -256,7 +246,7 @@ int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal // todo refactor int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbDelete(pState->pStateDb, &sKey, sizeof(SStateKey), &pState->txn); + return tdbTbDelete(pState->pStateDb, &sKey, sizeof(SStateKey), pState->txn); } int32_t streamStateClear(SStreamState* pState) { @@ -280,7 +270,7 @@ void streamStateSetNumber(SStreamState* pState, int32_t number) { pState->number // todo refactor int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) { - return tdbTbDelete(pState->pFillStateDb, key, sizeof(SWinKey), &pState->txn); + return tdbTbDelete(pState->pFillStateDb, key, sizeof(SWinKey), pState->txn); } int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { @@ -512,7 +502,7 @@ void streamFreeVal(void* val) { tdbFree(val); } int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbUpsert(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, &pState->txn); + return tdbTbUpsert(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, pState->txn); } int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { @@ -535,7 +525,7 @@ int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVa int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbDelete(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), &pState->txn); + return tdbTbDelete(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), pState->txn); } SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key) { diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 7ab2bc3995..69c8c3b087 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -33,7 +33,8 @@ typedef struct STxn TXN; // TDB int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback); int32_t tdbClose(TDB *pDb); -int32_t tdbBegin(TDB *pDb, TXN *pTxn); +int32_t tdbBegin(TDB *pDb, TXN **pTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, + int flags); int32_t tdbCommit(TDB *pDb, TXN *pTxn); int32_t tdbPostCommit(TDB *pDb, TXN *pTxn); int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e3860f85c6..3710ccb106 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -69,7 +69,7 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * static int tdbBtcMoveDownward(SBTC *pBtc); static int tdbBtcMoveUpward(SBTC *pBtc); -int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, +int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, TDB *pEnv, SBTree **ppBt) { SBTree *pBt; int ret; @@ -106,22 +106,26 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg if (pgno == 0) { // fetch page & insert into main db SPage *pPage; - TXN txn; - tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + TXN *txn; - pPager->inTran = 1; + ret = tdbBegin(pEnv, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + if (ret < 0) { + return -1; + } SBtreeInitPageArg zArg; zArg.flags = 0x1 | 0x2; // root leaf node; zArg.pBt = pBt; - ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, &txn); + ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, txn); if (ret < 0) { + tdbAbort(pEnv, txn); return -1; } ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { tdbError("failed to write page since %s", terrstr()); + tdbAbort(pEnv, txn); return -1; } @@ -130,18 +134,18 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg pBt->info.nLevel = 1; pBt->info.nData = 0; pBt->tbname = (char *)tbname; - // ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pgno, sizeof(SPgno), &txn); - ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pBt->info, sizeof(pBt->info), &txn); + + ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pBt->info, sizeof(pBt->info), txn); if (ret < 0) { + tdbAbort(pEnv, txn); return -1; } } - // tdbUnrefPage(pPage); - tdbPCacheRelease(pPager->pCache, pPage, &txn); - tdbCommit(pPager->pEnv, &txn); - tdbPostCommit(pPager->pEnv, &txn); - tdbTxnClose(&txn); + tdbPCacheRelease(pPager->pCache, pPage, txn); + + tdbCommit(pPager->pEnv, txn); + tdbPostCommit(pPager->pEnv, txn); } ASSERT(pgno != 0); @@ -1535,10 +1539,21 @@ int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn) { memset(&pBtc->coder, 0, sizeof(SCellDecoder)); if (pTxn == NULL) { - pBtc->pTxn = &pBtc->txn; - tdbTxnOpen(pBtc->pTxn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); + TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); + if (!pTxn) { + return -1; + } + + if (tdbTxnOpen(pTxn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + tdbOsFree(pTxn); + return -1; + } + + pBtc->pTxn = pTxn; + pBtc->freeTxn = 1; } else { pBtc->pTxn = pTxn; + pBtc->freeTxn = 0; } return 0; @@ -2232,6 +2247,10 @@ int tdbBtcClose(SBTC *pBtc) { tdbFree(pBtc->coder.pVal); } + if (pBtc->freeTxn) { + tdbTxnClose(pBtc->pTxn); + } + return 0; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 5aff5b7bb2..bda06aed39 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -99,19 +99,34 @@ int tdbClose(TDB *pDb) { int32_t tdbAlter(TDB *pDb, int pages) { return tdbPCacheAlter(pDb->pCache, pages); } -int32_t tdbBegin(TDB *pDb, TXN *pTxn) { +int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, + int flags) { SPager *pPager; int ret; + int64_t txnId = 1; + + TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); + if (!pTxn) { + return -1; + } + + if (tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags) < 0) { + tdbOsFree(pTxn); + return -1; + } for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerBegin(pPager, pTxn); if (ret < 0) { tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); + tdbTxnClose(pTxn); return -1; } } + *ppTxn = pTxn; + return 0; } @@ -144,6 +159,8 @@ int32_t tdbPostCommit(TDB *pDb, TXN *pTxn) { } } + tdbTxnClose(pTxn); + return 0; } @@ -176,6 +193,8 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { } } + tdbTxnClose(pTxn); + return 0; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 8b029b06d6..636c4fd997 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -108,7 +108,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF ASSERT(pPager != NULL); // pTb->pBt - ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, &(pTb->pBt)); + ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt)); if (ret < 0) { tdbOsFree(pTb); return -1; diff --git a/source/libs/tdb/src/db/tdbTxn.c b/source/libs/tdb/src/db/tdbTxn.c index f173d89779..77c87d18f2 100644 --- a/source/libs/tdb/src/db/tdbTxn.c +++ b/source/libs/tdb/src/db/tdbTxn.c @@ -28,4 +28,10 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void return 0; } -int tdbTxnClose(TXN *pTxn) { return 0; } \ No newline at end of file +int tdbTxnClose(TXN *pTxn) { + if (pTxn) { + tdbOsFree(pTxn); + } + + return 0; +} diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 731b1927e7..8e04383b95 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -147,11 +147,11 @@ struct SBTC { SPage *pgStack[BTREE_MAX_DEPTH + 1]; SCellDecoder coder; TXN *pTxn; - TXN txn; + i8 freeTxn; }; // SBTree -int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, +int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, TDB *pEnv, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); @@ -396,12 +396,12 @@ struct SPager { SPCache *pCache; SPgno dbFileSize; SPgno dbOrigSize; - //SPage *pDirty; + // SPage *pDirty; hashset_t jPageSet; - SRBTree rbt; - u8 inTran; - SPager *pNext; // used by TDB - SPager *pHashNext; // used by TDB + SRBTree rbt; + u8 inTran; + SPager *pNext; // used by TDB + SPager *pHashNext; // used by TDB #ifdef USE_MAINDB TDB *pEnv; #endif diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp index 305e91f62c..f4f09b20b8 100644 --- a/source/libs/tdb/test/tdbExOVFLTest.cpp +++ b/source/libs/tdb/test/tdbExOVFLTest.cpp @@ -170,11 +170,9 @@ static void insertOfp(void) { SPoolMem *pPool = openPool(); // start a transaction - TXN txn; - int64_t txnid = 0; - ++txnid; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); // generate value payload // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) @@ -185,12 +183,12 @@ static void insertOfp(void) { // insert the generated big data // char const *key = "key1"; char const *key = "key123456789"; - ret = tdbTbInsert(pDb, key, strlen(key), val, valLen, &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, valLen, txn); GTEST_ASSERT_EQ(ret, 0); // commit current transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); } // TEST(TdbOVFLPagesTest, DISABLED_TbInsertTest) { @@ -258,11 +256,9 @@ TEST(TdbOVFLPagesTest, TbDeleteTest) { SPoolMem *pPool = openPool(); // start a transaction - TXN txn; - int64_t txnid = 0; - ++txnid; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); // generate value payload // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) @@ -271,7 +267,7 @@ TEST(TdbOVFLPagesTest, TbDeleteTest) { generateBigVal(val, valLen); { // insert the generated big data - ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, &txn); + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); GTEST_ASSERT_EQ(ret, 0); } @@ -297,7 +293,7 @@ tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_REA tdbBegin(pEnv, &txn); */ { // upsert the data - ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), &txn); + ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -316,7 +312,7 @@ tdbBegin(pEnv, &txn); } { // delete the data - ret = tdbTbDelete(pDb, "key1", strlen("key1"), &txn); + ret = tdbTbDelete(pDb, "key1", strlen("key1"), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -335,8 +331,8 @@ tdbBegin(pEnv, &txn); } // commit current transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); } // TEST(tdb_test, DISABLED_simple_insert1) { @@ -346,7 +342,7 @@ TEST(tdb_test, simple_insert1) { TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1; - TXN txn; + TXN *txn; int const pageSize = 4096; taosRemoveDir("tdb"); @@ -365,16 +361,13 @@ TEST(tdb_test, simple_insert1) { // char val[(4083 - 4 - 3 - 2)]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) int64_t poolLimit = 4096; // 1M pool limit - int64_t txnid = 0; SPoolMem *pPool; // open the pool pPool = openPool(); // start a transaction - txnid++; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key0"); @@ -393,26 +386,25 @@ TEST(tdb_test, simple_insert1) { val[i] = c; } - ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, &txn); + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one if (pPool->size >= poolLimit) { // commit current transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); // start a new transaction clearPool(pPool); - txnid++; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); } } // commit the transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); { // Query the data void *pVal = NULL; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index f3a301cf5b..54e80a0009 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -125,7 +125,7 @@ TEST(tdb_test, DISABLED_simple_insert1) { TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; - TXN txn; + TXN *txn; taosRemoveDir("tdb"); @@ -142,40 +142,35 @@ TEST(tdb_test, DISABLED_simple_insert1) { char key[64]; char val[64]; int64_t poolLimit = 4096; // 1M pool limit - int64_t txnid = 0; SPoolMem *pPool; // open the pool pPool = openPool(); // start a transaction - txnid++; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one if (pPool->size >= poolLimit) { // commit current transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); // start a new transaction clearPool(pPool); - txnid++; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); } } // commit the transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); { // Query the data void *pVal = NULL; @@ -245,7 +240,7 @@ TEST(tdb_test, DISABLED_simple_insert2) { TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; - TXN txn; + TXN *txn; taosRemoveDir("tdb"); @@ -261,21 +256,18 @@ TEST(tdb_test, DISABLED_simple_insert2) { { char key[64]; char val[64]; - int64_t txnid = 0; SPoolMem *pPool; // open the pool pPool = openPool(); // start a transaction - txnid++; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -312,8 +304,8 @@ TEST(tdb_test, DISABLED_simple_insert2) { } // commit the transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); ret = tdbTbDrop(pDb); GTEST_ASSERT_EQ(ret, 0); @@ -331,7 +323,7 @@ TEST(tdb_test, DISABLED_simple_delete1) { TTB *pDb; char key[128]; char data[128]; - TXN txn; + TXN *txn; TDB *pEnv; SPoolMem *pPool; void *pKey = NULL; @@ -353,14 +345,13 @@ TEST(tdb_test, DISABLED_simple_delete1) { ret = tdbTbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb, 0); GTEST_ASSERT_EQ(ret, 0); - tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); // loop to insert batch data for (int iData = 0; iData < nKV; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -378,7 +369,7 @@ TEST(tdb_test, DISABLED_simple_delete1) { for (int iData = nKV - 1; iData > 30; iData--) { sprintf(key, "key%d", iData); - ret = tdbTbDelete(pDb, key, strlen(key), &txn); + ret = tdbTbDelete(pDb, key, strlen(key), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -413,7 +404,8 @@ TEST(tdb_test, DISABLED_simple_delete1) { tdbTbcClose(pDbc); - tdbCommit(pEnv, &txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); closePool(pPool); @@ -430,7 +422,7 @@ TEST(tdb_test, DISABLED_simple_upsert1) { char data[64]; void *pData = NULL; SPoolMem *pPool; - TXN txn; + TXN *txn; taosRemoveDir("tdb"); @@ -444,13 +436,12 @@ TEST(tdb_test, DISABLED_simple_upsert1) { pPool = openPool(); // insert some data - tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -467,11 +458,12 @@ TEST(tdb_test, DISABLED_simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d-u", iData); - ret = tdbTbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbUpsert(pDb, key, strlen(key), data, strlen(data), txn); GTEST_ASSERT_EQ(ret, 0); } - tdbCommit(pEnv, &txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); // query the data for (int iData = 0; iData < nData; iData++) { @@ -492,7 +484,7 @@ TEST(tdb_test, multi_thread_query) { TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; - TXN txn; + TXN *txn; taosRemoveDir("tdb"); @@ -508,26 +500,18 @@ TEST(tdb_test, multi_thread_query) { char key[64]; char val[64]; int64_t poolLimit = 4096 * 20; // 1M pool limit - int64_t txnid = 0; SPoolMem *pPool; // open the pool pPool = openPool(); // start a transaction - txnid++; - txn.flags = TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED; - txn.txnId = -1; - txn.xMalloc = poolMalloc; - txn.xFree = poolFree; - txn.xArg = pPool; - // tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, ); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -578,7 +562,7 @@ TEST(tdb_test, multi_thread_query) { std::vector threads; for (int i = 0; i < nThreads; i++) { if (i == 0) { - threads.push_back(std::thread(tdbCommit, pEnv, &txn)); + threads.push_back(std::thread(tdbCommit, pEnv, txn)); } else { threads.push_back(std::thread(f, pDb, nData)); } @@ -589,8 +573,8 @@ TEST(tdb_test, multi_thread_query) { } // commit the transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); // Close a database tdbTbClose(pDb); @@ -621,17 +605,12 @@ TEST(tdb_test, DISABLED_multi_thread1) { GTEST_ASSERT_EQ(ret, 0); auto insert = [](TDB *pDb, TTB *pTb, int nData, int *stop, std::shared_timed_mutex *mu) { - TXN txn = {0}; + TXN *txn = NULL; char key[128]; char val[128]; SPoolMem *pPool = openPool(); - txn.flags = TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED; - txn.txnId = -1; - txn.xMalloc = poolMalloc; - txn.xFree = poolFree; - txn.xArg = pPool; - tdbBegin(pDb, &txn); + tdbBegin(pDb, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); @@ -644,14 +623,17 @@ TEST(tdb_test, DISABLED_multi_thread1) { } if (pPool->size > 1024 * 1024) { - tdbCommit(pDb, &txn); + tdbCommit(pDb, txn); + tdbPostCommit(pDb, txn); clearPool(pPool); - tdbBegin(pDb, &txn); + tdbBegin(pDb, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); } } - tdbCommit(pDb, &txn); + tdbCommit(pDb, txn); + tdbPostCommit(pDb, txn); + closePool(pPool); *stop = 1; From 2c771153ef4a911ba36cfb69c8513e9bb41fecfe Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 28 Nov 2022 16:27:20 +0800 Subject: [PATCH 03/95] fix(streamState): use new txn related api --- source/libs/stream/src/streamState.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 5ed06e5eea..af1d738de0 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -192,9 +192,9 @@ void streamStateClose(SStreamState* pState) { } int32_t streamStateBegin(SStreamState* pState) { - if (tdbBegin(pState->db, &pState->pTdbState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - tdbAbort(pState->db, pState->txn); + tdbAbort(pState->pTdbState->db, pState->pTdbState->txn); return -1; } return 0; @@ -220,7 +220,7 @@ int32_t streamStateAbort(SStreamState* pState) { return -1; } - if (tdbBegin(pState->pTdbState->db, &pState->ptdbstate->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } @@ -823,7 +823,7 @@ _end: int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, - &pState->pTdbState->txn); + pState->pTdbState->txn); return 0; } From 7a7d7f7d00be0c0d44437e00a52deed2253b16f4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 28 Nov 2022 17:31:11 +0800 Subject: [PATCH 04/95] tdb/write: new tdbPWriteFile api for commit and prep async to maindb --- include/os/osFile.h | 1 + source/libs/tdb/src/inc/tdbOs.h | 1 + source/os/src/osFile.c | 22 ++++++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/include/os/osFile.h b/include/os/osFile.h index f6759d19a7..ae77e0f27a 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -88,6 +88,7 @@ int32_t taosFsyncFile(TdFilePtr pFile); int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count); int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset); int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); +int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset); void taosFprintfFile(TdFilePtr pFile, const char *format, ...); int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf); diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index b5dd27052c..3419034dd1 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -52,6 +52,7 @@ typedef TdFilePtr tdb_fd_t; #define tdbOsRead taosReadFile #define tdbOsPRead taosPReadFile #define tdbOsWrite taosWriteFile +#define tdbOsPWrite taosPWriteFile #define tdbOsFSync taosFsyncFile #define tdbOsLSeek taosLSeekFile #define tdbOsRemove remove diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index c3283ffe84..68365be481 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -491,6 +491,28 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { return count; } +int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) { + if (pFile == NULL) { + return 0; + } +#if FILE_WITH_LOCK + taosThreadRwlockWrlock(&(pFile->rwlock)); +#endif + assert(pFile->fd >= 0); // Please check if you have closed the file. +#ifdef WINDOWS + size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR); + _lseeki64(pFile->fd, offset, SEEK_SET); + int64_t ret = _write(pFile->fd, buf, count); + _lseeki64(pFile->fd, pos, SEEK_SET); +#else + int64_t ret = pwrite(pFile->fd, buf, count, offset); +#endif +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return ret; +} + int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); From 50318f6f9663ea786330c72ed0c9d4a776ea6da3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 28 Nov 2022 17:33:11 +0800 Subject: [PATCH 05/95] tdb/pager: use tdb pwrite page to db instead of write --- source/libs/tdb/src/db/tdbPager.c | 33 ++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 35992f9b55..71df83d7a9 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -142,7 +142,7 @@ int hashset_contains(hashset_t set, void *item) { static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *, int), void *arg, u8 loadPage); static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage); -static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage); +static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage); static FORCE_INLINE int32_t pageCmpFn(const SRBTreeNode *lhs, const SRBTreeNode *rhs) { SPage *pPageL = (SPage *)(((uint8_t *)lhs) - offsetof(SPage, node)); @@ -326,7 +326,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage = (SPage *)pNode; ASSERT(pPage->nOverflow == 0); - ret = tdbPagerWritePageToDB(pPager, pPage); + ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); return -1; @@ -403,7 +403,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { while ((pNode = tRBTreeIterNext(&iter)) != NULL) { pPage = (SPage *)pNode; if (pPage->isLocal) continue; - ret = tdbPagerWritePageToDB(pPager, pPage); + ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); return -1; @@ -553,7 +553,7 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) { if (pgno > maxPgno) { maxPgno = pgno; } - ret = tdbPagerWritePageToDB(pPager, pPage); + ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); return -1; @@ -769,13 +769,6 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { return 0; } /* -struct TdFile { - TdThreadRwlock rwlock; - int refId; - int fd; - FILE *fp; -} TdFile; -*/ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { i64 offset; int ret; @@ -795,7 +788,23 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { return -1; } - // pwrite(pPager->fd->fd, pPage->pData, pPage->pageSize, offset); + return 0; +} +*/ +static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) { + i64 offset; + int ret; + + offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); + + ret = tdbOsPWrite(pPager->fd, pPage->pData, pPage->pageSize, offset); + if (ret < 0) { + tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, + pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + return 0; } From a780305e1024e84a2ff31de0b665fe12f4b1e8df Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 28 Nov 2022 17:36:33 +0800 Subject: [PATCH 06/95] tdb/begin: use txn id in mem --- source/libs/tdb/src/db/tdbDb.c | 2 +- source/libs/tdb/src/inc/tdbInt.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index bda06aed39..bccea07269 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -103,7 +103,7 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void ( int flags) { SPager *pPager; int ret; - int64_t txnId = 1; + int64_t txnId = ++pDb->txnId; TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); if (!pTxn) { diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 8e04383b95..6578041225 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -382,6 +382,7 @@ struct STDB { #ifdef USE_MAINDB TTB *pMainDb; #endif + int64_t txnId; }; typedef struct hashset_st *hashset_t; From e090cd9dd9ffac6dfe915b180a5daccf0d9a9618 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 29 Nov 2022 10:18:12 +0800 Subject: [PATCH 07/95] tdb/pager: move journal to txn --- source/libs/tdb/inc/tdb.h | 8 +- source/libs/tdb/src/db/tdbPage.c | 5 +- source/libs/tdb/src/db/tdbPager.c | 119 ++++++++++++++++++------------ source/libs/tdb/src/inc/tdbInt.h | 13 ++-- 4 files changed, 87 insertions(+), 58 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 69c8c3b087..5f9de017f0 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -17,6 +17,7 @@ #define _TD_TDB_H_ #include "os.h" +#include "tdbOs.h" #ifdef __cplusplus extern "C" { @@ -78,12 +79,17 @@ int32_t tdbTxnClose(TXN *pTxn); // other void tdbFree(void *); +typedef struct hashset_st *hashset_t; + struct STxn { int flags; int64_t txnId; void *(*xMalloc)(void *, size_t); void (*xFree)(void *, void *); - void *xArg; + void *xArg; + tdb_fd_t jfd; + hashset_t jPageSet; + int preped; }; // error code diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index 016ad65cf3..0653b6fc36 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -69,14 +69,15 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = pPage; - tdbDebug("page/create: %p %p", pPage, xMalloc); + tdbTrace("page/create: %p/%d %p", pPage, pPage->id, xMalloc); return 0; } int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { u8 *ptr; - tdbDebug("page/destroy: %p %p", pPage, xFree); + tdbTrace("page/destroy: %p/%d %p", pPage, pPage->id, xFree); + ASSERT(!pPage->isDirty); ASSERT(xFree); for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 71df83d7a9..ebad93eb7f 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -219,9 +219,11 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { int tdbPagerClose(SPager *pPager) { if (pPager) { + /* if (pPager->inTran) { tdbOsClose(pPager->jfd); } + */ tdbOsClose(pPager->fd); tdbOsFree(pPager); } @@ -232,16 +234,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { int ret; SPage **ppPage; - ASSERT(pPager->inTran); -#if 0 - if (pPager->inTran == 0) { - ret = tdbPagerBegin(pPager); - if (ret < 0) { - return -1; - } - } -#endif - + // ASSERT(pPager->inTran); if (pPage->isDirty) return 0; // ref page one more time so the page will not be release @@ -271,15 +264,16 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { // Write page to journal if neccessary if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize && - (pPager->jPageSet == NULL || !hashset_contains(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) { + (pPager->pActiveTxn->jPageSet == NULL || + !hashset_contains(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { tdbError("failed to write page to journal since %s", tstrerror(terrno)); return -1; } - if (pPager->jPageSet) { - hashset_add(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); + if (pPager->pActiveTxn->jPageSet) { + hashset_add(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } } @@ -287,23 +281,28 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { } int tdbPagerBegin(SPager *pPager, TXN *pTxn) { + /* if (pPager->inTran) { return 0; } - + */ // Open the journal - pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); - if (TDB_FD_INVALID(pPager->jfd)) { + char jTxnFileName[TDB_FILENAME_LEN]; + sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); + if (TDB_FD_INVALID(pTxn->jfd)) { tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pPager->jPageSet = hashset_create(); + pTxn->jPageSet = hashset_create(); + ASSERT(pPager->pActiveTxn->preped == 1); + pPager->pActiveTxn = pTxn; // TODO: write the size of the file - + /* pPager->inTran = 1; - + */ return 0; } @@ -312,9 +311,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { int ret; // sync the journal file - ret = tdbOsFSync(pPager->jfd); + ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { - tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -344,8 +343,8 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); - if (pPager->jPageSet) { - hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); + if (pTxn->jPageSet) { + hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } tdbPCacheRelease(pPager->pCache, pPage, pTxn); } @@ -364,35 +363,39 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { } int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { + char jTxnFileName[TDB_FILENAME_LEN]; + sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + // remove the journal file - if (tdbOsClose(pPager->jfd) < 0) { - tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + if (tdbOsClose(pTxn->jfd) < 0) { + tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { - tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName); + if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (pPager->jPageSet) { - hashset_destroy(pPager->jPageSet); + if (pTxn->jPageSet) { + hashset_destroy(pTxn->jPageSet); } - pPager->inTran = 0; + // pPager->inTran = 0; return 0; } int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { SPage *pPage; + SPgno maxPgno = pPager->dbOrigSize; int ret; // sync the journal file - ret = tdbOsFSync(pPager->jfd); + ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { - tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -403,6 +406,11 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { while ((pNode = tRBTreeIterNext(&iter)) != NULL) { pPage = (SPage *)pNode; if (pPage->isLocal) continue; + + SPgno pgno = TDB_PAGE_PGNO(pPage); + if (pgno > maxPgno) { + maxPgno = pgno; + } ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); @@ -411,7 +419,8 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { } tdbTrace("tdbttl commit:%p, %d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize); - pPager->dbOrigSize = pPager->dbFileSize; + pPager->dbOrigSize = maxPgno; + // pPager->dbOrigSize = pPager->dbFileSize; // release the page iter = tRBTreeIterCreate(&pPager->rbt, 1); @@ -423,6 +432,8 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } + + pTxn->preped = 1; /* tdbTrace("reset dirty tree: %p", &pPager->rbt); tRBTreeCreate(&pPager->rbt, pageCmpFn); @@ -444,15 +455,15 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { SPgno journalSize = 0; int ret; - // 0, sync the journal file - ret = tdbOsFSync(pPager->jfd); + // sync the journal file + ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { - tdbError("failed to fsync jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - tdb_fd_t jfd = pPager->jfd; + tdb_fd_t jfd = pTxn->jfd; ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); if (ret < 0) { @@ -516,7 +527,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); - hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); + hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } @@ -524,11 +535,24 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { tRBTreeCreate(&pPager->rbt, pageCmpFn); // 4, remove the journal file - tdbOsClose(pPager->jfd); - (void)tdbOsRemove(pPager->jFileName); - hashset_destroy(pPager->jPageSet); + if (tdbOsClose(pTxn->jfd) < 0) { + tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } - pPager->inTran = 0; + char jTxnFileName[TDB_FILENAME_LEN]; + sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + + if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + hashset_destroy(pTxn->jPageSet); + + // pPager->inTran = 0; return 0; } @@ -751,17 +775,18 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { pgno = TDB_PAGE_PGNO(pPage); - ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno)); + ret = tdbOsWrite(pPager->pActiveTxn->jfd, &pgno, sizeof(pgno)); if (ret < 0) { - tdbError("failed to write pgno due to %s. file:%s, pgno:%u", strerror(errno), pPager->jFileName, pgno); + tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(errno), pPager->jFileName, + pgno, pPager->pActiveTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize); + ret = tdbOsWrite(pPager->pActiveTxn->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { - tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, - (long)pPage->pageSize); + tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(errno), + pPager->jFileName, pPage->pageSize, pPager->pActiveTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 6578041225..c46ab68689 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -385,24 +385,21 @@ struct STDB { int64_t txnId; }; -typedef struct hashset_st *hashset_t; - struct SPager { char *dbFileName; char *jFileName; int pageSize; uint8_t fid[TDB_FILE_ID_LEN]; tdb_fd_t fd; - tdb_fd_t jfd; SPCache *pCache; SPgno dbFileSize; SPgno dbOrigSize; // SPage *pDirty; - hashset_t jPageSet; - SRBTree rbt; - u8 inTran; - SPager *pNext; // used by TDB - SPager *pHashNext; // used by TDB + SRBTree rbt; + // u8 inTran; + TXN *pActiveTxn; + SPager *pNext; // used by TDB + SPager *pHashNext; // used by TDB #ifdef USE_MAINDB TDB *pEnv; #endif From 7902b7aaac10c75769a33e1c0245ffaccd698523 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 29 Nov 2022 10:46:04 +0800 Subject: [PATCH 08/95] fix/tdb: remove UAF preped --- source/libs/tdb/inc/tdb.h | 1 - source/libs/tdb/src/db/tdbPager.c | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 5f9de017f0..c728e29641 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -89,7 +89,6 @@ struct STxn { void *xArg; tdb_fd_t jfd; hashset_t jPageSet; - int preped; }; // error code diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index ebad93eb7f..082fec4902 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -297,7 +297,7 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { } pTxn->jPageSet = hashset_create(); - ASSERT(pPager->pActiveTxn->preped == 1); + pPager->pActiveTxn = pTxn; // TODO: write the size of the file /* @@ -433,7 +433,6 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { tdbPCacheRelease(pPager->pCache, pPage, pTxn); } - pTxn->preped = 1; /* tdbTrace("reset dirty tree: %p", &pPager->rbt); tRBTreeCreate(&pPager->rbt, pageCmpFn); From 797af5373ddca04177a64da4c78ebcab517e3b7c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Nov 2022 10:59:40 +0800 Subject: [PATCH 09/95] enh(query): improve the multi-way merge performance. --- source/libs/executor/src/exchangeoperator.c | 7 +--- source/libs/executor/src/sortoperator.c | 5 ++- source/libs/executor/src/tsort.c | 46 +++++++++++---------- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index b2ddff45a4..001d668749 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -209,7 +209,7 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { } } -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { +static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -318,7 +318,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); + createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); return pOperator; _error: @@ -581,13 +581,10 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; pOperator->cost.openCost = taosGetTimestampUs() - startTs; - - tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - tsem_post(&pExchangeInfo->ready); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index ec754f31b0..2f504cee3d 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -559,7 +559,7 @@ typedef struct SMultiwayMergeOperatorInfo { STupleHandle* prefetchedTuple; } SMultiwayMergeOperatorInfo; -int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { +int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) { SMultiwayMergeOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -580,6 +580,7 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = pOperator->pDownstream[i]; ps->onlyRef = true; + tsortAddSource(pInfo->pSortHandle, ps); } @@ -792,7 +793,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result. setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, + pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL, destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo); code = appendDownstream(pOperator, downStreams, numStreams); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 3f91142708..4db76b82f5 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -34,14 +34,12 @@ struct SSortHandle { int32_t pageSize; int32_t numOfPages; SDiskbasedBuf* pBuf; - - SArray* pSortInfo; - SArray* pOrderedSource; - - int32_t loops; - uint64_t sortElapsed; - int64_t startTs; - uint64_t totalElapsed; + SArray* pSortInfo; + SArray* pOrderedSource; + int32_t loops; + uint64_t sortElapsed; + int64_t startTs; + uint64_t totalElapsed; int32_t sourceId; SSDataBlock* pDataBlock; @@ -99,9 +97,9 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page } static int32_t sortComparCleanup(SMsortComparParam* cmpParam) { + // NOTICE: pSource may be, if it is SORT_MULTISOURCE_MERGE for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SSortSource* pSource = - cmpParam->pSources[i]; // NOTICE: pSource may be SGenericSource *, if it is SORT_MULTISOURCE_MERGE + SSortSource* pSource = cmpParam->pSources[i]; blockDataDestroy(pSource->src.pBlock); taosMemoryFreeClear(pSource); } @@ -228,15 +226,15 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { return doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId, pPageIdList); } -static void setCurrentSourceIsDone(SSortSource* pSource, SSortHandle* pHandle) { +static void setCurrentSourceDone(SSortSource* pSource, SSortHandle* pHandle) { pSource->src.rowIndex = -1; ++pHandle->numOfCompletedSources; } -static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex, +static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32_t startIndex, int32_t endIndex, SSortHandle* pHandle) { - cmpParam->pSources = taosArrayGet(pSources, startIndex); - cmpParam->numOfSources = (endIndex - startIndex + 1); + pParam->pSources = taosArrayGet(pSources, startIndex); + pParam->numOfSources = (endIndex - startIndex + 1); int32_t code = 0; @@ -244,7 +242,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int if (pHandle->pBuf == NULL) { if (!osTempSpaceAvailable()) { code = TSDB_CODE_NO_AVAIL_DISK; - qError("Sort compare init failed since %s", terrstr(code)); + qError("Sort compare init failed since %s, %s", terrstr(code), pHandle->idStr); return code; } @@ -257,12 +255,12 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int } if (pHandle->type == SORT_SINGLESOURCE_SORT) { - for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SSortSource* pSource = cmpParam->pSources[i]; + for (int32_t i = 0; i < pParam->numOfSources; ++i) { + SSortSource* pSource = pParam->pSources[i]; // set current source is done if (taosArrayGetSize(pSource->pageIdList) == 0) { - setCurrentSourceIsDone(pSource, pHandle); + setCurrentSourceDone(pSource, pHandle); continue; } @@ -277,15 +275,21 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int releaseBufPage(pHandle->pBuf, pPage); } } else { - for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SSortSource* pSource = cmpParam->pSources[i]; + qDebug("start init for the multiway merge sort, %s", pHandle->idStr); + int64_t st = taosGetTimestampUs(); + + for (int32_t i = 0; i < pParam->numOfSources; ++i) { + SSortSource* pSource = pParam->pSources[i]; pSource->src.pBlock = pHandle->fetchfp(pSource->param); // set current source is done if (pSource->src.pBlock == NULL) { - setCurrentSourceIsDone(pSource, pHandle); + setCurrentSourceDone(pSource, pHandle); } } + + int64_t et = taosGetTimestampUs(); + qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr); } return code; From 629c87685f535d6da44230c787fad67a70c48655 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 29 Nov 2022 11:04:47 +0800 Subject: [PATCH 10/95] fix/txnId: wrap around INT64_MAX --- source/libs/tdb/src/db/tdbDb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index bccea07269..c79279c658 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -104,6 +104,9 @@ int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void ( SPager *pPager; int ret; int64_t txnId = ++pDb->txnId; + if (txnId == INT64_MAX) { + pDb->txnId = 0; + } TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); if (!pTxn) { From a1585f16f10076cffc0d2c82737b4e42e5c3fedb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Nov 2022 11:06:20 +0800 Subject: [PATCH 11/95] ehn(query): improve the merge sort performance. --- source/libs/executor/src/sortoperator.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 2f504cee3d..0287af62d2 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -577,8 +577,13 @@ int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) { tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort); for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + SOperatorInfo* pDownstream = pOperator->pDownstream[i]; + if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) { + pDownstream->fpSet._openFn(pDownstream); + } + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); - ps->param = pOperator->pDownstream[i]; + ps->param = pDownstream; ps->onlyRef = true; tsortAddSource(pInfo->pSortHandle, ps); From dac930352b7764c3f0b36b9703a8251ad1d31784 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Nov 2022 12:22:04 +0800 Subject: [PATCH 12/95] enh(query): optimize the perf in compare. --- include/common/ttypes.h | 2 - source/libs/scalar/src/sclvector.c | 62 +++++++++++++++++++----------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 761ffd0f1c..6350057c1f 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -278,11 +278,9 @@ typedef struct { #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) #define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX) -#define IS_VALID_BIGINT(_t) ((_t) >= INT64_MIN && (_t) <= INT64_MAX) #define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) <= UINT8_MAX) #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX) #define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) <= UINT32_MAX) -#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) <= UINT64_MAX) #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c4ff5b2b01..98391d95f4 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1543,9 +1543,44 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, int32_t step, __compar_fn_t fp, int32_t optr) { int32_t num = 0; - bool * pRes = (bool *)pOut->columnData->pData; + bool *pRes = (bool *)pOut->columnData->pData; - if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_JSON || GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_JSON) { + if (IS_MATHABLE_TYPE(GET_PARAM_TYPE(pLeft)) && IS_MATHABLE_TYPE(GET_PARAM_TYPE(pRight))) { + if (!(pLeft->columnData->hasNull || pRight->columnData->hasNull)) { + for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { + int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; + int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; + + char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); + char *pRightData = colDataGetData(pRight->columnData, rightIndex); + + pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); + if (pRes[i]) { + ++num; + } + } + } else { + for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { + int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; + int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; + + if (colDataIsNull_f(pLeft->columnData->nullbitmap, leftIndex) || + colDataIsNull_f(pRight->columnData->nullbitmap, rightIndex)) { + pRes[i] = false; + continue; + } + + char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); + char *pRightData = colDataGetData(pRight->columnData, rightIndex); + + pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); + if (pRes[i]) { + ++num; + } + } + } + } else { + // if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_JSON || GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_JSON) { for (int32_t i = startIndex; i < numOfRows && i >= startIndex; i += step) { int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; @@ -1556,8 +1591,8 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa continue; } - char * pLeftData = colDataGetData(pLeft->columnData, leftIndex); - char * pRightData = colDataGetData(pRight->columnData, rightIndex); + char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); + char *pRightData = colDataGetData(pRight->columnData, rightIndex); int64_t leftOut = 0; int64_t rightOut = 0; bool freeLeft = false; @@ -1592,25 +1627,6 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa taosMemoryFreeClear(pRightData); } } - } else { - for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { - int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; - int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; - - if (colDataIsNull_s(pLeft->columnData, leftIndex) || - colDataIsNull_s(pRight->columnData, rightIndex)) { - pRes[i] = false; - continue; - } - - char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); - char *pRightData = colDataGetData(pRight->columnData, rightIndex); - - pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); - if (pRes[i]) { - ++num; - } - } } return num; From 0b1cd9f824b1fad7054fe0799600594635b25c87 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 29 Nov 2022 16:00:51 +0800 Subject: [PATCH 13/95] tdb/journal: rollback & restore multiple journal files --- source/libs/tdb/src/db/tdbPager.c | 50 +++++++++++++++++++++++++++---- source/libs/tdb/src/db/tdbTable.c | 2 +- source/libs/tdb/src/inc/tdbInt.h | 2 +- source/libs/tdb/src/inc/tdbOs.h | 25 +++++++++------- 4 files changed, 61 insertions(+), 18 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 082fec4902..7f0fdf9efc 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -832,12 +832,12 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) { return 0; } -int tdbPagerRestore(SPager *pPager, SBTree *pBt) { +static int tdbPagerRestore(SPager *pPager, SBTree *pBt, const char *jFileName) { int ret = 0; SPgno journalSize = 0; u8 *pageBuf = NULL; - tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755); + tdb_fd_t jfd = tdbOsOpen(jFileName, TDB_O_RDWR, 0755); if (jfd == NULL) { return 0; } @@ -910,12 +910,50 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { return 0; } -int tdbPagerRollback(SPager *pPager) { - if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { - tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); - terrno = TAOS_SYSTEM_ERROR(errno); +int tdbPagerRestoreJournals(SPager *pPager, SBTree *pBt) { + tdbDirEntryPtr pDirEntry; + tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); + if (pDir == NULL) { + tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); return -1; } + while ((pDirEntry = tdbReadDir(pDir)) != NULL) { + char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); + if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { + if (tdbPagerRestore(pPager, pBt, name) < 0) { + tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), name); + return -1; + } + } + } + + tdbCloseDir(&pDir); + + return 0; +} + +int tdbPagerRollback(SPager *pPager) { + tdbDirEntryPtr pDirEntry; + tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); + if (pDir == NULL) { + tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); + return -1; + } + + while ((pDirEntry = tdbReadDir(pDir)) != NULL) { + char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); + + if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { + if (tdbOsRemove(name) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } + } + + tdbCloseDir(&pDir); + return 0; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 636c4fd997..c5c2d6aebe 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -117,7 +117,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF if (rollback) { tdbPagerRollback(pPager); } else { - ret = tdbPagerRestore(pPager, pTb->pBt); + ret = tdbPagerRestoreJournals(pPager, pTb->pBt); if (ret < 0) { tdbOsFree(pTb); return -1; diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index c46ab68689..055a8a1062 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -197,7 +197,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initP TXN *pTxn); void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); -int tdbPagerRestore(SPager *pPager, SBTree *pBt); +int tdbPagerRestoreJournals(SPager *pPager, SBTree *pBt); int tdbPagerRollback(SPager *pPager); // tdbPCache.c ==================================== diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index 3419034dd1..db4283e267 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -47,16 +47,21 @@ typedef TdFilePtr tdb_fd_t; #define TDB_O_RDWR (TD_FILE_WRITE) | (TD_FILE_READ) #define tdbOsOpen(PATH, OPTION, MODE) taosOpenFile((PATH), (OPTION)) - -#define tdbOsClose(FD) taosCloseFile(&(FD)) -#define tdbOsRead taosReadFile -#define tdbOsPRead taosPReadFile -#define tdbOsWrite taosWriteFile -#define tdbOsPWrite taosPWriteFile -#define tdbOsFSync taosFsyncFile -#define tdbOsLSeek taosLSeekFile -#define tdbOsRemove remove -#define tdbOsFileSize(FD, PSIZE) taosFStatFile(FD, PSIZE, NULL) +#define tdbOsClose(FD) taosCloseFile(&(FD)) +#define tdbOsRead taosReadFile +#define tdbOsPRead taosPReadFile +#define tdbOsWrite taosWriteFile +#define tdbOsPWrite taosPWriteFile +#define tdbOsFSync taosFsyncFile +#define tdbOsLSeek taosLSeekFile +#define tdbDirPtr TdDirPtr +#define tdbDirEntryPtr TdDirEntryPtr +#define tdbReadDir taosReadDir +#define tdbGetDirEntryName taosGetDirEntryName +#define tdbDirEntryBaseName taosDirEntryBaseName +#define tdbCloseDir taosCloseDir +#define tdbOsRemove remove +#define tdbOsFileSize(FD, PSIZE) taosFStatFile(FD, PSIZE, NULL) /* directory */ #define tdbOsMkdir taosMkDir From 445c4f28f0d09f3f38813f5c26f016c10350f265 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 29 Nov 2022 23:34:22 +0800 Subject: [PATCH 14/95] enh(query): optimize the perf. --- source/common/src/tdatablock.c | 5 ++++- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/libs/executor/src/sortoperator.c | 3 +-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a0f795a729..20eda52aac 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1139,6 +1139,9 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF void blockDataCleanup(SSDataBlock* pDataBlock) { SDataBlockInfo* pInfo = &pDataBlock->info; + int32_t existedRows = pInfo->rows; + ASSERT(existedRows <= pDataBlock->info.capacity); + pInfo->rows = 0; pInfo->id.uid = 0; pInfo->id.groupId = 0; @@ -1152,7 +1155,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - colInfoDataCleanup(p, pInfo->capacity); + colInfoDataCleanup(p, existedRows); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4ba311212a..660d189451 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3589,7 +3589,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - colDataAppendInt64(pColData, outputRowIndex, &pBlockData->aTSKEY[rowIndex]); + ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 0287af62d2..b355b85861 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -720,7 +720,6 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { } qDebug("start to merge final sorted rows, %s", GET_TASKID(pTaskInfo)); - SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->matchInfo.pList, pOperator); if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; @@ -787,7 +786,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); SSDataBlock* pInputBlock = createDataBlockFromDescNode(pChildNode->pOutputDataBlockDesc); - initResultSizeInfo(&pOperator->resultInfo, 4096); + initResultSizeInfo(&pOperator->resultInfo, 1024); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); pInfo->groupSort = pMergePhyNode->groupSort; From a8703c7fc22d661d1fbd7e196ddeb475b4ffbfd7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 00:11:21 +0800 Subject: [PATCH 15/95] enh(query): optimize the perf. --- source/libs/executor/src/timewindowoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index a6a477a9e3..dae2dafe9e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1286,7 +1286,6 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { return NULL; } - blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL); @@ -1737,7 +1736,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, 4096); + initResultSizeInfo(&pOperator->resultInfo, 1024); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); From 3f0c93a62308112a1ba2a5093277fb8c342ff509 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 14:53:25 +0800 Subject: [PATCH 16/95] enh(query): optimize query perf. --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 294a4bd3e4..c149e52fed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -40,7 +40,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd } pFD->szPage = szPage; pFD->pgno = 0; - pFD->pBuf = taosMemoryCalloc(1, szPage); + pFD->pBuf = taosMemoryMalloc(szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; taosCloseFile(&pFD->pFD); From ae816331441c1602d500a5f3d208da76e1fe323b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 14:55:27 +0800 Subject: [PATCH 17/95] enh(query): optimize query perf. --- source/common/src/tdatablock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 20eda52aac..af45db053a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1242,7 +1242,7 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, true); + code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, false); if (code) { return code; } From 8dcbafb78066547b8f8f04c4772e2f7a559f1415 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 15:24:03 +0800 Subject: [PATCH 18/95] enh(query): avoiding unnecessary remove operation. --- source/util/src/tpagedbuf.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index e1a43ace47..b7dbdad053 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -507,7 +507,9 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { dBufPrintStatis(pBuf); + bool needRemoveFile = false; if (pBuf->pFile != NULL) { + needRemoveFile = true; uDebug( "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page " "size:%.2f Kb, %s\n", @@ -534,9 +536,14 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { } } - if (taosRemoveFile(pBuf->path) < 0) { - uDebug("WARNING tPage remove file failed. path=%s", pBuf->path); + if (needRemoveFile) { + int32_t ret = taosRemoveFile(pBuf->path); + if (ret != 0) { // print the error and discard this error info + int32_t code = TAOS_SYSTEM_ERROR(errno); + uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(code)); + } } + taosMemoryFreeClear(pBuf->path); size_t n = taosArrayGetSize(pBuf->pIdList); From d66b7f3de07d6bb1149b5be41b5aab6ac0fa2e7e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 15:25:19 +0800 Subject: [PATCH 19/95] refactor: do some internal refactor. --- source/util/src/tpagedbuf.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index b7dbdad053..0f4c51060b 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -539,8 +539,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { if (needRemoveFile) { int32_t ret = taosRemoveFile(pBuf->path); if (ret != 0) { // print the error and discard this error info - int32_t code = TAOS_SYSTEM_ERROR(errno); - uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(code)); + uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(errno)); } } From 4fef284cfe07d71afe4f088b0d7767f79f1ae636 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 21:26:27 +0800 Subject: [PATCH 20/95] refactor: do some internal refactor. --- include/libs/function/function.h | 59 +++++++----------------- include/libs/function/tudf.h | 26 +++++++++++ source/dnode/mgmt/node_util/inc/dmUtil.h | 2 +- 3 files changed, 43 insertions(+), 44 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 240772bfc2..8fe9d98e57 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -137,22 +137,22 @@ typedef struct SqlFunctionCtx { int16_t functionId; // function id char *pOutput; // final result output buffer, point to sdata->data int32_t numOfParams; - SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param - SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ - int32_t offset; - struct SResultRowEntryInfo *resultInfo; - SSubsidiaryResInfo subsidiaries; - SPoint1 start; - SPoint1 end; - SFuncExecFuncs fpSet; - SScalarFuncExecFuncs sfp; - struct SExprInfo *pExpr; - struct SSDataBlock *pSrcBlock; - struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity - SSerializeDataHandle saveHandle; - bool isStream; - - char udfName[TSDB_FUNC_NAME_LEN]; + // input parameter, e.g., top(k, 20), the number of results of top query is kept in param + SFunctParam *param; + // corresponding output buffer for timestamp of each result, e.g., diff/csum + SColumnInfoData *pTsOutput; + int32_t offset; + SResultRowEntryInfo *resultInfo; + SSubsidiaryResInfo subsidiaries; + SPoint1 start; + SPoint1 end; + SFuncExecFuncs fpSet; + SScalarFuncExecFuncs sfp; + struct SExprInfo *pExpr; + struct SSDataBlock *pSrcBlock; + struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity + SSerializeDataHandle saveHandle; + char udfName[TSDB_FUNC_NAME_LEN]; } SqlFunctionCtx; typedef struct tExprNode { @@ -182,7 +182,6 @@ struct SScalarParam { }; void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell); -//int32_t getNumOfResult(SqlFunctionCtx *pCtx, int32_t num, SSDataBlock *pResBlock); bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry); @@ -194,32 +193,6 @@ typedef struct SPoint { int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2, int32_t inputType); -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// udf api -/** - * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf - * @return error code - */ -int32_t udfcOpen(); - -/** - * destroy udfd proxy - * @return error code - */ -int32_t udfcClose(); - -/** - * start udfd that serves udf function invocation under dnode startDnodeId - * @param startDnodeId - * @return - */ -int32_t udfStartUdfd(int32_t startDnodeId); -/** - * stop udfd - * @return - */ -int32_t udfStopUdfd(); - #ifdef __cplusplus } #endif diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 31cc53bb9f..b71d50d43c 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -85,6 +85,32 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, int32_t cleanUpUdfs(); +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// udf api +/** + * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf + * @return error code + */ +int32_t udfcOpen(); + +/** + * destroy udfd proxy + * @return error code + */ +int32_t udfcClose(); + +/** + * start udfd that serves udf function invocation under dnode startDnodeId + * @param startDnodeId + * @return + */ +int32_t udfStartUdfd(int32_t startDnodeId); +/** + * stop udfd + * @return + */ +int32_t udfStopUdfd(); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 8719e988e7..2124b387ec 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -39,7 +39,7 @@ #include "sync.h" #include "wal.h" -#include "libs/function/function.h" +#include "libs/function/tudf.h" #ifdef __cplusplus extern "C" { #endif From 26ab0894a88f97699cbf9cb6a05bd666e0e618b6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 30 Nov 2022 21:56:02 +0800 Subject: [PATCH 21/95] fix(query): fix syntax error. --- source/libs/executor/src/executil.c | 2 -- source/libs/executor/src/timewindowoperator.c | 7 +------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b135566caa..7c37d66eb6 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1536,8 +1536,6 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->start.key = INT64_MIN; pCtx->end.key = INT64_MIN; pCtx->numOfParams = pExpr->base.numOfParams; - pCtx->isStream = false; - pCtx->param = pFunct->pParam; pCtx->saveHandle.currentPage = -1; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index dae2dafe9e..35548a6ade 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1717,7 +1717,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSuppor void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) { for (int32_t i = 0; i < numOfExpr; i++) { - pCtx[i].isStream = true; +// pCtx[i].isStream = true; } } @@ -1783,11 +1783,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh goto _error; } - if (isStream) { - ASSERT(num > 0); - initStreamFunciton(pSup->pCtx, pSup->numOfExprs); - } - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, pInfo); if (pInfo->timeWindowInterpo) { From cefe4be1abef27af09104d7a053ecc3478d93bfb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 00:34:06 +0800 Subject: [PATCH 22/95] refactor: do some internal refactor. --- include/common/tdatablock.h | 1 + source/common/src/tdatablock.c | 19 +++++++++ .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 41 ++++++++++--------- source/util/src/tarray.c | 2 +- source/util/src/thash.c | 4 +- 8 files changed, 48 insertions(+), 25 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index db6ca65cf0..a5d3a0829d 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -234,6 +234,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); +int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void blockDataCleanup(SSDataBlock* pDataBlock); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index af45db053a..f5e73d8f0e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1239,6 +1239,25 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { return TSDB_CODE_SUCCESS; } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, true); + if (code) { + return code; + } + } + + pDataBlock->info.capacity = numOfRows; + return TSDB_CODE_SUCCESS; +} + +int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows) { + int32_t code = 0; + if (numOfRows == 0 || numOfRows <= pDataBlock->info.capacity) { + return TSDB_CODE_SUCCESS; + } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index c149e52fed..294a4bd3e4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -40,7 +40,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd } pFD->szPage = szPage; pFD->pgno = 0; - pFD->pBuf = taosMemoryMalloc(szPage); + pFD->pBuf = taosMemoryCalloc(1, szPage); if (pFD->pBuf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; taosCloseFile(&pFD->pFD); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 300ba52934..3e28798a3a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1749,7 +1749,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n pAggSup->currentPageId = -1; pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); - pAggSup->pResultRowHashTable = tSimpleHashInit(10, hashFn); + pAggSup->pResultRowHashTable = tSimpleHashInit(100, hashFn); if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 84b7678b9f..ca3734d91e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -886,7 +886,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); - blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); + blockDataEnsureCapacityNoClear(pInfo->pResBlock, pOperator->resultInfo.capacity); code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 35548a6ade..551d668763 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1658,24 +1658,6 @@ static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) { // the primary timestamp column bool needed = false; - pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn)); - pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys)); - - { // ts column - SColumn c = {0}; - c.colId = 1; - c.slotId = pInfo->primaryTsIndex; - c.type = TSDB_DATA_TYPE_TIMESTAMP; - c.bytes = sizeof(int64_t); - taosArrayPush(pInfo->pInterpCols, &c); - - SGroupKeys key = {0}; - key.bytes = c.bytes; - key.type = c.type; - key.isNull = true; // to denote no value is assigned yet - key.pData = taosMemoryCalloc(1, c.bytes); - taosArrayPush(pInfo->pPrevValues, &key); - } for (int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = pCtx[i].pExpr; @@ -1696,6 +1678,27 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt } } + if (needed) { + pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys)); + + { // ts column + SColumn c = {0}; + c.colId = 1; + c.slotId = pInfo->primaryTsIndex; + c.type = TSDB_DATA_TYPE_TIMESTAMP; + c.bytes = sizeof(int64_t); + taosArrayPush(pInfo->pInterpCols, &c); + + SGroupKeys key; + key.bytes = c.bytes; + key.type = c.type; + key.isNull = true; // to denote no value is assigned yet + key.pData = taosMemoryCalloc(1, c.bytes); + taosArrayPush(pInfo->pPrevValues, &key); + } + } + return needed; } @@ -1737,7 +1740,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(&pOperator->resultInfo, 1024); - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + blockDataEnsureCapacityNoClear(pInfo->binfo.pRes, pOperator->resultInfo.capacity); int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index f53b1cfd7f..6a16c044f9 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -33,7 +33,7 @@ SArray* taosArrayInit(size_t size, size_t elemSize) { } pArray->size = 0; - pArray->pData = taosMemoryCalloc(size, elemSize); + pArray->pData = taosMemoryMalloc(size * elemSize); if (pArray->pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pArray); diff --git a/source/util/src/thash.c b/source/util/src/thash.c index a0411483ca..b07d485c87 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -244,7 +244,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp capacity = 4; } - SHashObj *pHashObj = (SHashObj *)taosMemoryCalloc(1, sizeof(SHashObj)); + SHashObj *pHashObj = (SHashObj *)taosMemoryMalloc(sizeof(SHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -264,7 +264,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); - pHashObj->hashList = (SHashEntry **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); + pHashObj->hashList = (SHashEntry **)taosMemoryMalloc(pHashObj->capacity * sizeof(void *)); if (pHashObj->hashList == NULL) { taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; From c996837fab936338a81491e9c4ae88412091f7ef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 01:10:37 +0800 Subject: [PATCH 23/95] refactor: do some internal refactor. --- source/util/src/tarray.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 6a16c044f9..f53b1cfd7f 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -33,7 +33,7 @@ SArray* taosArrayInit(size_t size, size_t elemSize) { } pArray->size = 0; - pArray->pData = taosMemoryMalloc(size * elemSize); + pArray->pData = taosMemoryCalloc(size, elemSize); if (pArray->pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pArray); From f8a2ab8338cedbdb68d3f2c1fc4236c1d95fee57 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 17:24:26 +0800 Subject: [PATCH 24/95] refactor: do some internal refactor. --- include/common/tcommon.h | 5 +- include/common/tdatablock.h | 6 +- source/common/src/tdatablock.c | 34 +++-- source/dnode/vnode/inc/vnode.h | 3 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 123 ++++++++---------- source/libs/executor/src/executil.c | 13 +- source/libs/executor/src/executor.c | 4 +- source/libs/executor/src/executorimpl.c | 2 - source/libs/executor/src/scanoperator.c | 39 ++---- source/libs/executor/src/sysscanoperator.c | 85 ++++++------ source/libs/executor/src/timewindowoperator.c | 6 +- source/libs/scalar/src/sclvector.c | 2 +- source/util/src/thash.c | 5 +- 13 files changed, 156 insertions(+), 171 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 77f1879b81..6ec3d5db10 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -249,10 +249,11 @@ typedef struct SColumnInfoData { typedef struct SQueryTableDataCond { uint64_t suid; - int32_t order; // desc|asc order to iterate the data block + int32_t order; // desc|asc order to iterate the data block int32_t numOfCols; SColumnInfo* colList; - int32_t type; // data block load type: + int32_t* pSlotList; // the column output destation slot, and it may be null + int32_t type; // data block load type: STimeWindow twindows; int64_t startVersion; int64_t endVersion; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index a5d3a0829d..4208d85648 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -41,7 +41,7 @@ typedef struct SBlockOrderInfo { BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) -#define colDataSetNotNull_f(bm_, r_) \ +#define colDataClearNull_f(bm_, r_) \ do { \ BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \ } while (0) @@ -151,9 +151,6 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui for (int32_t i = start; i < start + nRows; ++i) { colDataSetNull_f(pColumnInfoData->nullbitmap, i); } - - int32_t bytes = pColumnInfoData->info.bytes; - memset(pColumnInfoData->pData + start * bytes, 0, nRows * bytes); } pColumnInfoData->hasNull = true; @@ -238,6 +235,7 @@ int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRo void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void blockDataCleanup(SSDataBlock* pDataBlock); +void blockDataEmpty(SSDataBlock* pDataBlock); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f5e73d8f0e..c300c3e6f0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1137,17 +1137,15 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF } void blockDataCleanup(SSDataBlock* pDataBlock) { + blockDataEmpty(pDataBlock); SDataBlockInfo* pInfo = &pDataBlock->info; - - int32_t existedRows = pInfo->rows; - ASSERT(existedRows <= pDataBlock->info.capacity); - - pInfo->rows = 0; pInfo->id.uid = 0; pInfo->id.groupId = 0; - pInfo->window.ekey = 0; - pInfo->window.skey = 0; +} +void blockDataEmpty(SSDataBlock* pDataBlock) { + SDataBlockInfo* pInfo = &pDataBlock->info; + ASSERT(pInfo->rows <= pDataBlock->info.capacity); if (pInfo->capacity == 0) { return; } @@ -1155,8 +1153,12 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - colInfoDataCleanup(p, existedRows); + colInfoDataCleanup(p, pInfo->capacity); } + + pInfo->rows = 0; + pInfo->window.ekey = 0; + pInfo->window.skey = 0; } // todo temporarily disable it @@ -1643,6 +1645,8 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, n, total); + + // clear the offset value of the unused entries. memset(&pColInfoData->varmeta.offset[total - n], 0, n); } else { int32_t bytes = pColInfoData->info.bytes; @@ -1657,7 +1661,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { } if (pBlock->info.rows <= n) { - blockDataCleanup(pBlock); + blockDataEmpty(pBlock); } else { size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { @@ -1674,12 +1678,22 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_ if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); + } else { // reset the bitmap value + int32_t stopIndex = BitmapLen(n) * 8; + for(int32_t i = n + 1; i < stopIndex; ++i) { + colDataClearNull_f(pColInfoData->nullbitmap, i); + } + + int32_t remain = BitmapLen(total) - BitmapLen(n); + if (remain > 0) { + memset(pColInfoData->nullbitmap, 0, remain); + } } } int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) { if (n == 0) { - blockDataCleanup(pBlock); + blockDataEmpty(pBlock); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7ef3207b4d..53b8eadb33 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -159,11 +159,10 @@ typedef struct STsdbReader STsdbReader; int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - STsdbReader **ppReader, const char *idstr); + SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr); void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); -bool tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid); void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 660d189451..27d99bd42d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -85,7 +85,8 @@ typedef struct SBlockLoadSuppInfo { SArray* pColAgg; SColumnDataAgg tsColAgg; SColumnDataAgg** plist; - int16_t* colIds; // column ids for loading file block data + int16_t* colIds; // column ids for loading file block data + int16_t* slotIds; // the ordinal index in the destination SSDataBlock int32_t numOfCols; char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. bool smaValid; // the sma on all queried columns are activated @@ -214,25 +215,25 @@ static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFil static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SSDataBlock* pBlock) { - size_t numOfCols = blockDataGetNumOfCols(pBlock); - +static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) { pSupInfo->smaValid = true; pSupInfo->numOfCols = numOfCols; pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t)); pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES); - if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL) { + pSupInfo->slotIds = taosMemoryMalloc(numOfCols * sizeof(int16_t)); + if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL || pSupInfo->slotIds == NULL) { taosMemoryFree(pSupInfo->colIds); taosMemoryFree(pSupInfo->buildBuf); + taosMemoryFree(pSupInfo->slotIds); return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); - pSupInfo->colIds[i] = pCol->info.colId; + pSupInfo->colIds[i] = pCols[i].colId; + pSupInfo->slotIds[i] = pSlotIdList[i]; - if (IS_VAR_DATA_TYPE(pCol->info.type)) { - pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes); + if (IS_VAR_DATA_TYPE(pCols[i].type)) { + pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); } } @@ -566,7 +567,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) } static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, int32_t capacity, - const char* idstr) { + SSDataBlock* pResBlock, const char* idstr) { int32_t code = 0; int8_t level = 0; STsdbReader* pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader)); @@ -585,6 +586,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->suid = pCond->suid; pReader->order = pCond->order; pReader->capacity = capacity; + pReader->pResBlock = pResBlock; pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; @@ -592,6 +594,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket ASSERT(pCond->numOfCols > 0); + // todo refactor. limitOutputBufferSize(pCond, &pReader->capacity); // allocate buffer in order to load data blocks from file @@ -611,13 +614,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - pReader->pResBlock = createResBlock(pCond, pReader->capacity); - if (pReader->pResBlock == NULL) { - code = terrno; - goto _end; - } - - setColumnIdSlotList(&pReader->suppInfo, pReader->pResBlock); + setColumnIdSlotList(&pReader->suppInfo, pCond->colList, pCond->pSlotList, pCond->numOfCols); *ppReader = pReader; return code; @@ -1041,17 +1038,16 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo } static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { - SReaderStatus* pStatus = &pReader->status; - SDataBlockIter* pBlockIter = &pStatus->blockIter; + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pStatus->fileBlockData; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter); SSDataBlock* pResBlock = pReader->pResBlock; - int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock); - - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + int32_t numOfOutputCols = pSupInfo->numOfCols; SColVal cv = {0}; int64_t st = taosGetTimestampUs(); @@ -1087,8 +1083,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t i = 0; int32_t rowIndex = 0; - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); - if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + if (pSupInfo->colIds[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); i += 1; } @@ -1097,12 +1093,13 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t num = pBlockData->nColData; while (i < numOfOutputCols && colIndex < num) { rowIndex = 0; - pColData = taosArrayGet(pResBlock->pDataBlock, i); SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex); - if (pData->cid < pColData->info.colId) { + if (pData->cid < pSupInfo->colIds[i]) { colIndex += 1; - } else if (pData->cid == pColData->info.colId) { + } else if (pData->cid == pSupInfo->colIds[i]) { + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { colDataAppendNNULL(pColData, 0, dumpedRows); } else { @@ -1119,6 +1116,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colIndex += 1; i += 1; } else { // the specified column does not exist in file block, fill with null data + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1126,7 +1124,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // fill the mis-matched columns with null value while (i < numOfOutputCols) { - pColData = taosArrayGet(pResBlock->pDataBlock, i); + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -3535,7 +3533,7 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pScanInfo) { - int32_t numOfRows = pBlock->info.rows; + int32_t outputRowIndex = pBlock->info.rows; int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); int64_t uid = pScanInfo->uid; @@ -3545,23 +3543,26 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* SColVal colVal = {0}; int32_t i = 0, j = 0; - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - if (pColInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - colDataAppend(pColInfoData, numOfRows, (const char*)&pTSRow->ts, false); + if (pSupInfo->colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; } while (i < numOfCols && j < pSchema->numOfCols) { - pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - col_id_t colId = pColInfoData->info.colId; + col_id_t colId = pSupInfo->colIds[i]; if (colId == pSchema->columns[j].colId) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + tTSRowGetVal(pTSRow, pSchema, j, &colVal); - doCopyColVal(pColInfoData, numOfRows, i, &colVal, pSupInfo); + doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); i += 1; j += 1; } else if (colId < pSchema->columns[j].colId) { - colDataAppendNULL(pColInfoData, numOfRows); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + + colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; } else if (colId > pSchema->columns[j].colId) { j += 1; @@ -3570,8 +3571,8 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* // set null value since current column does not exist in the "pSchema" while (i < numOfCols) { - pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - colDataAppendNULL(pColInfoData, numOfRows); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; } @@ -3586,27 +3587,25 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t outputRowIndex = pResBlock->info.rows; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); - if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pReader->suppInfo.colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; } SColVal cv = {0}; int32_t numOfInputCols = pBlockData->nColData; - int32_t numOfOutputCols = pResBlock->pDataBlock->size; + int32_t numOfOutputCols = pSupInfo->numOfCols; while (i < numOfOutputCols && j < numOfInputCols) { - SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, i); - SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); - - if (pData->cid < pCol->info.colId) { + SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); + if (pData->cid < pSupInfo->colIds[i]) { j += 1; continue; } - if (pData->cid == pCol->info.colId) { + SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + if (pData->cid == pSupInfo->colIds[i]) { tColDataGetValue(pData, rowIndex, &cv); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); j += 1; @@ -3619,7 +3618,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S } while (i < numOfOutputCols) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); colDataAppendNULL(pCol, outputRowIndex); i += 1; } @@ -3718,14 +3717,14 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, - STsdbReader** ppReader, const char* idstr) { + SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) { STimeWindow window = pCond->twindows; if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { pCond->twindows.skey += 1; pCond->twindows.ekey -= 1; } - int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, 4096, idstr); + int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, pResBlock->info.capacity, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3751,7 +3750,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } // here we only need one more row, so the capacity is set to be ONE. - code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, idstr); + code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3765,7 +3764,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } pCond->order = order; - code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, idstr); + code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3837,10 +3836,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; -_err: + _err: tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); return code; -} + } void tsdbReaderClose(STsdbReader* pReader) { if (pReader == NULL) { @@ -3874,7 +3873,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pSupInfo->colIds); taosArrayDestroy(pSupInfo->pColAgg); - for (int32_t i = 0; i < blockDataGetNumOfCols(pReader->pResBlock); ++i) { + for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) { if (pSupInfo->buildBuf[i] != NULL) { taosMemoryFreeClear(pSupInfo->buildBuf[i]); } @@ -3891,8 +3890,6 @@ void tsdbReaderClose(STsdbReader* pReader) { clearBlockScanInfoBuf(&pReader->blockInfoBuf); } - blockDataDestroy(pReader->pResBlock); - if (pReader->pFileReader != NULL) { tsdbDataFReaderClose(&pReader->pFileReader); } @@ -4007,16 +4004,6 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { return false; } -bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) { - STableBlockScanInfo* pBlockScanInfo = - *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid)); - if (pBlockScanInfo == NULL) { // no data block for the table of given uid - return false; - } - - return true; -} - static void setBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow) { ASSERT(pReader != NULL); *rows = pReader->pResBlock->info.rows; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 7c37d66eb6..3f62191ca1 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1241,6 +1241,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod } } + // set the output flag for each column in SColMatchInfo, according to the *numOfOutputCols = 0; int32_t num = LIST_LENGTH(pOutputNodeList->pSlots); for (int32_t i = 0; i < num; ++i) { @@ -1599,20 +1600,22 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) { int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) { pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); + pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); - if (pCond->colList == NULL) { + pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)*pCond->numOfCols); + if (pCond->colList == NULL || pCond->pSlotList == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFreeClear(pCond->colList); + taosMemoryFreeClear(pCond->pSlotList); return terrno; } - // pCond->twindow = pTableScanNode->scanRange; // TODO: get it from stable scan node pCond->twindows = pTableScanNode->scanRange; pCond->suid = pTableScanNode->scan.suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = -1; - // pCond->type = pTableScanNode->scanFlag; int32_t j = 0; for (int32_t i = 0; i < pCond->numOfCols; ++i) { @@ -1625,6 +1628,8 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi pCond->colList[j].type = pColNode->node.resType.type; pCond->colList[j].bytes = pColNode->node.resType.bytes; pCond->colList[j].colId = pColNode->colId; + + pCond->pSlotList[j] = pNode->slotId; j += 1; } @@ -1962,7 +1967,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, - struct SExecTaskInfo* pTaskInfo) { + SExecTaskInfo* pTaskInfo) { int64_t st = taosGetTimestampUs(); const char* idStr = GET_TASKID(pTaskInfo); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 34bd9cf8ca..d675ac05d5 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1064,7 +1064,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int32_t num = tableListGetSize(pTaskInfo->pTableInfoList); if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num, - &pTableScanInfo->base.dataReader, NULL) < 0 || + pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 || pTableScanInfo->base.dataReader == NULL) { ASSERT(0); } @@ -1116,7 +1116,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); ASSERT(size == 1); - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->dataReader, NULL); + tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->pRes, &pInfo->dataReader, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3e28798a3a..aa2682c490 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1190,8 +1190,6 @@ void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) { // T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); // } // -// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); -// // if (pQueryAttr->limit.offset > blockInfo.rows) { // pQueryAttr->limit.offset -= blockInfo.rows; // pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ca3734d91e..1729c9f1e8 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -285,7 +285,7 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo if (pLimit->offset > 0 && pLimitInfo->remainOffset > 0) { if (pLimitInfo->remainOffset >= pBlock->info.rows) { pLimitInfo->remainOffset -= pBlock->info.rows; - pBlock->info.rows = 0; + blockDataEmpty(pBlock); qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset, id); } else { blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); @@ -390,7 +390,6 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca return terrno; } - relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); // restore the previous value @@ -638,16 +637,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - blockDataCleanup(pBlock); - SDataBlockInfo* pBInfo = &pBlock->info; - - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(pTableScanInfo->base.dataReader, &rows, &pBInfo->id.uid, &pBInfo->window); - - blockDataEnsureCapacity(pBlock, rows); // todo remove it latter - pBInfo->rows = rows; - - ASSERT(pBInfo->id.uid != 0); + ASSERT(pBlock->info.id.uid != 0); pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); uint32_t status = 0; @@ -778,7 +768,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { ASSERT(pInfo->base.dataReader == NULL); int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, - (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo)); + pInfo->pResBlock, (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -879,14 +869,14 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.pdInfo.interval = extractIntervalInfo(pTableScanNode); pInfo->base.readHandle = *readHandle; + pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired; + pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); - pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired; - initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); - blockDataEnsureCapacityNoClear(pInfo->pResBlock, pOperator->resultInfo.capacity); + blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { @@ -993,10 +983,8 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo; SSDataBlock* pBlock = pTableScanInfo->pResBlock; - blockDataCleanup(pBlock); - STsdbReader* pReader = NULL; - int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader, + int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, (STsdbReader**)&pReader, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -2301,7 +2289,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->initTableReader) { pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->base.dataReader = NULL; - code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, &pTSInfo->base.dataReader, NULL); + code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, pTSInfo->pResBlock, &pTSInfo->base.dataReader, NULL); if (code != 0) { terrno = code; destroyTableScanOperatorInfo(pTableScanOp); @@ -2531,11 +2519,10 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { blockDataCleanup(pBlock); int64_t st = taosGetTimestampUs(); - - void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex); + void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex); SReadHandle* pHandle = &pInfo->base.readHandle; - int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->base.dataReader, GET_TASKID(pTaskInfo)); + int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &pInfo->base.dataReader, GET_TASKID(pTaskInfo)); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } @@ -2553,12 +2540,6 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } blockDataCleanup(pBlock); - - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.id.uid, &pBlock->info.window); - blockDataEnsureCapacity(pBlock, rows); - pBlock->info.rows = rows; - if (pQueryCond->order == TSDB_ORDER_ASC) { pQueryCond->twindows.skey = pBlock->info.window.ekey + 1; } else { diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 3d35326749..7ac6cee2af 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1899,53 +1899,52 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo) { - SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - goto _error; - } + SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } - { - SQueryTableDataCond cond = {0}; + pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc); + blockDataEnsureCapacity(pInfo->pResBlock, 1); - int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; - size_t num = tableListGetSize(pTableListInfo); - void* pList = tableListGetInfo(pTableListInfo, 0); - - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str); - cleanupQueryTableDataCond(&cond); - if (code != 0) { - goto _error; - } - } - - pInfo->readHandle = *readHandle; - pInfo->uid = pBlockScanNode->suid; - - pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc); - blockDataEnsureCapacity(pInfo->pResBlock, 1); - - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + { + SQueryTableDataCond cond = {0}; + int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond); if (code != TSDB_CODE_SUCCESS) { - goto _error; + goto _error; } - setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, - OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); - return pOperator; + STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; + size_t num = tableListGetSize(pTableListInfo); + void* pList = tableListGetInfo(pTableListInfo, 0); - _error: - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; + code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str); + cleanupQueryTableDataCond(&cond); + if (code != 0) { + goto _error; + } + } + + pInfo->readHandle = *readHandle; + pInfo->uid = pBlockScanNode->suid; + + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, + OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); + return pOperator; + +_error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; } \ No newline at end of file diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 551d668763..5f077c98d7 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1739,8 +1739,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, 1024); - blockDataEnsureCapacityNoClear(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + initResultSizeInfo(&pOperator->resultInfo, 512); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); @@ -4319,7 +4319,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, 4096); + initResultSizeInfo(&pOperator->resultInfo, 512); int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 98391d95f4..6074c72424 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1782,7 +1782,7 @@ void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, if (colDataIsNull_s(pOut->columnData, i)) { int8_t v = 0; colDataAppendInt8(pOut->columnData, i, &v); - colDataSetNotNull_f(pOut->columnData->nullbitmap, i); + colDataClearNull_f(pOut->columnData->nullbitmap, i); } } pOut->columnData->hasNull = false; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index b07d485c87..e9548613aa 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -279,7 +279,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp return NULL; } - void *p = taosMemoryCalloc(pHashObj->capacity, sizeof(SHashEntry)); + void *p = taosMemoryMalloc(pHashObj->capacity * sizeof(SHashEntry)); if (p == NULL) { taosArrayDestroy(pHashObj->pMemBlock); taosMemoryFree(pHashObj->hashList); @@ -290,6 +290,9 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp for (int32_t i = 0; i < pHashObj->capacity; ++i) { pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry)); + pHashObj->hashList[i]->num = 0; + pHashObj->hashList[i]->latch = 0; + pHashObj->hashList[i]->next = NULL; } taosArrayPush(pHashObj->pMemBlock, &p); From 9f55ca7f80fa3c8ff83fbc471c74353264fa82e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 17:45:33 +0800 Subject: [PATCH 25/95] test:update the test cases. --- tests/script/tsim/parser/groupby-basic.sim | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/script/tsim/parser/groupby-basic.sim b/tests/script/tsim/parser/groupby-basic.sim index 4c70399e4b..2e436945e0 100644 --- a/tests/script/tsim/parser/groupby-basic.sim +++ b/tests/script/tsim/parser/groupby-basic.sim @@ -78,8 +78,8 @@ $ts2 = $tb2 . .ts print ===============================groupby_operation print -print ==== select count(*), c1 from group_tb0 group by c1 -sql select count(*), c1 from group_tb0 group by c1 +print ==== select count(*), c1 from group_tb0 group by c1 order by c1 +sql select count(*), c1 from group_tb0 group by c1 order by c1 print rows: $rows print $data00 $data01 $data02 $data03 print $data10 $data11 $data12 $data13 @@ -98,18 +98,18 @@ endi if $data90 != 10 then return -1 endi -if $data01 != 7 then +if $data01 != 0 then return -1 endi -if $data11 != 6 then +if $data11 != 1 then return -1 endi -if $data91 != 3 then +if $data91 != 9 then return -1 endi -print ==== select first(ts),c1 from group_tb0 group by c1; -sql select first(ts),c1 from group_tb0 group by c1; +print ==== select first(ts),c1 from group_tb0 group by c1 order by c1; +sql select first(ts),c1 from group_tb0 group by c1 order by c1; print rows: $rows print $data00 $data01 $data02 $data03 print $data10 $data11 $data12 $data13 @@ -120,16 +120,16 @@ if $row != 10 then return -1 endi -if $data00 != @22-01-01 00:00:00.007@ then +if $data00 != @22-01-01 00:00:00.000@ then return -1 endi -if $data01 != 7 then +if $data01 != 0 then return -1 endi -if $data90 != @22-01-01 00:00:00.003@ then +if $data90 != @22-01-01 00:00:00.009@ then return -1 endi -if $data91 != 3 then +if $data91 != 9 then return -1 endi From 8c62d5f9fa027c28382ec84641fdd37101cc38d2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 18:34:26 +0800 Subject: [PATCH 26/95] fix(query): fix bug in tsdbread. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 9 +++-- source/libs/executor/src/scanoperator.c | 13 ++++---- source/libs/executor/src/timewindowoperator.c | 33 +++++++++++-------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 27d99bd42d..7c39e378cb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1621,7 +1621,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* int64_t st = taosGetTimestampUs(); int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader); - blockDataUpdateTsWindow(pBlock, 0); + blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotIds[0]); pBlock->info.id.uid = pBlockScanInfo->uid; setComposedBlockFlag(pReader, true); @@ -2493,7 +2493,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { _end: pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; - blockDataUpdateTsWindow(pResBlock, 0); + blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotIds[0]); setComposedBlockFlag(pReader, true); double el = (taosGetTimestampUs() - st) / 1000.0; @@ -3534,7 +3534,6 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t outputRowIndex = pBlock->info.rows; - int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); int64_t uid = pScanInfo->uid; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; @@ -3549,7 +3548,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* i += 1; } - while (i < numOfCols && j < pSchema->numOfCols) { + while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) { col_id_t colId = pSupInfo->colIds[i]; if (colId == pSchema->columns[j].colId) { @@ -3570,7 +3569,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* } // set null value since current column does not exist in the "pSchema" - while (i < numOfCols) { + while (i < pSupInfo->numOfCols) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1729c9f1e8..9fc3575491 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2513,10 +2513,8 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t readIdx = source->readerIdx; SSDataBlock* pBlock = source->inputBlock; - STableMergeScanInfo* pTableScanInfo = pOperator->info; - SQueryTableDataCond* pQueryCond = taosArrayGet(pTableScanInfo->queryConds, readIdx); - blockDataCleanup(pBlock); + SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx); int64_t st = taosGetTimestampUs(); void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex); @@ -2534,12 +2532,11 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } // process this data block based on the probabilities - bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample); + bool processThisBlock = processBlockWithProbability(&pInfo->sample); if (!processThisBlock) { continue; } - blockDataCleanup(pBlock); if (pQueryCond->order == TSDB_ORDER_ASC) { pQueryCond->twindows.skey = pBlock->info.window.ekey + 1; } else { @@ -2547,7 +2544,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } uint32_t status = 0; - loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); + loadDataBlock(pOperator, &pInfo->base, pBlock, &status); // code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -2561,7 +2558,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); pOperator->resultInfo.totalRows += pBlock->info.rows; - pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; + pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; tsdbReaderClose(pInfo->base.dataReader); pInfo->base.dataReader = NULL; @@ -2641,6 +2638,8 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { param.readerIdx = i; param.pOperator = pOperator; param.inputBlock = createOneDataBlock(pInfo->pResBlock, false); + blockDataEnsureCapacity(param.inputBlock, pOperator->resultInfo.capacity); + taosArrayPush(pInfo->sortSourceParams, ¶m); SQueryTableDataCond cond; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5f077c98d7..52f06ffab4 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1659,22 +1659,11 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt // the primary timestamp column bool needed = false; - for (int32_t i = 0; i < numOfCols; ++i) { + for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = pCtx[i].pExpr; - if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) { - SFunctParam* pParam = &pExpr->base.pParam[0]; - - SColumn c = *pParam->pCol; - taosArrayPush(pInfo->pInterpCols, &c); needed = true; - - SGroupKeys key = {0}; - key.bytes = c.bytes; - key.type = c.type; - key.isNull = false; - key.pData = taosMemoryCalloc(1, c.bytes); - taosArrayPush(pInfo->pPrevValues, &key); + break; } } @@ -1699,6 +1688,24 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt } } + for (int32_t i = 0; i < numOfCols; ++i) { + SExprInfo* pExpr = pCtx[i].pExpr; + + if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) { + SFunctParam* pParam = &pExpr->base.pParam[0]; + + SColumn c = *pParam->pCol; + taosArrayPush(pInfo->pInterpCols, &c); + + SGroupKeys key = {0}; + key.bytes = c.bytes; + key.type = c.type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, c.bytes); + taosArrayPush(pInfo->pPrevValues, &key); + } + } + return needed; } From 740daefdc54a49a0451566a28db77b6cee9098f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 18:39:15 +0800 Subject: [PATCH 27/95] fix(query): fix a typo. --- source/libs/executor/src/scanoperator.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9fc3575491..87aacfdd34 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -996,15 +996,13 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU if (hasBlock) { SDataBlockInfo* pBInfo = &pBlock->info; - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window); +// int32_t rows = 0; +// tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window); SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL); - blockDataEnsureCapacity(pBlock, rows); - pBlock->info.rows = rows; - relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true); - doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, rows); +// relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true); + doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->id.uid); } From 7facf64e5a01b3e7ac55f7c581d964e638f88c9a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 1 Dec 2022 18:58:30 +0800 Subject: [PATCH 28/95] fix: optimize ctg code --- source/libs/catalog/inc/catalogInt.h | 2 + source/libs/catalog/src/catalog.c | 2 +- source/libs/catalog/src/ctgCache.c | 59 ++++++++++-- source/libs/catalog/src/ctgUtil.c | 138 +++++++++++++++++++++++---- 4 files changed, 173 insertions(+), 28 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index c95d0fe462..d73dcb3b30 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -764,6 +764,8 @@ void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); void ctgFreeVgInfo(SDBVgInfo* vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); +int32_t ctgGetVgInfoFromHashValue2(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); + int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); void ctgResetTbMetaTask(SCtgTask* pTask); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3a398d1551..f17abaa31f 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -536,7 +536,7 @@ int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* return TSDB_CODE_SUCCESS; } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue2(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); _return: diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 19b7ee32ae..db3a53aeb0 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -277,6 +277,49 @@ _return: return TSDB_CODE_SUCCESS; } + +int32_t ctgAcquireStbMetaFromCache2(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgTbCache **pTb) { + SCtgTbCache *pCache = NULL; + char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + if (NULL == stName) { + ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + pCache = taosHashAcquire(dbCache->tbCache, stName, strlen(stName)); + if (NULL == pCache) { + ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", suid, stName, dbFName); + taosHashRelease(dbCache->stbCache, stName); + goto _return; + } + + taosHashRelease(dbCache->stbCache, stName); + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + *pTb = pCache; + + ctgDebug("stb 0x%" PRIx64 " meta got in cache, dbFName:%s", suid, dbFName); + + CTG_CACHE_STAT_INC(numOfMetaHit, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(numOfMetaMiss, 1); + + *pTb = NULL; + + return TSDB_CODE_SUCCESS; +} + int32_t ctgAcquireTbIndexFromCache(SCatalog *pCtg, char *dbFName, char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; SCtgTbCache *pCache = NULL; @@ -384,17 +427,17 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** memcpy(*pTableMeta, tbMeta, metaSize); - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + + if (tbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosHashRelease(dbCache->tbCache, tbCache); + } + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); - ctgAcquireStbMetaFromCache(pCtg, dbFName, ctx->tbInfo.suid, &dbCache, &tbCache); - if (NULL == tbCache) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - taosMemoryFreeClear(*pTableMeta); - ctgDebug("stb 0x%" PRIx64 " meta not in cache", ctx->tbInfo.suid); - return TSDB_CODE_SUCCESS; - } + ctgAcquireStbMetaFromCache2(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); STableMeta *stbMeta = tbCache->pMeta; if (stbMeta->suid != ctx->tbInfo.suid) { diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index f795be3ac2..ef7b80a676 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -837,6 +837,57 @@ _return: CTG_RET(code); } +int ctgVgInfoComp2(const void* lp, const void* rp) { + SVgroupInfo* pLeft = (SVgroupInfo*)lp; + SVgroupInfo* pRight = (SVgroupInfo*)rp; + if (pLeft->hashBegin < pRight->hashBegin) { + return -1; + } else if (pLeft->hashBegin > pRight->hashBegin) { + return 1; + } + + return 0; +} + +int32_t ctgHashValueComp(void const* lp, void const* rp) { + uint32_t* key = (uint32_t*)lp; + SVgroupInfo* pVg = *(SVgroupInfo**)rp; + + if (*key < pVg->hashBegin) { + return -1; + } else if (*key > pVg->hashEnd) { + return 1; + } + + return 0; +} + +int ctgVgInfoComp(const void* lp, const void* rp) { + SVgroupInfo* pLeft = *(SVgroupInfo**)lp; + SVgroupInfo* pRight = *(SVgroupInfo**)rp; + if (pLeft->hashBegin < pRight->hashBegin) { + return -1; + } else if (pLeft->hashBegin > pRight->hashBegin) { + return 1; + } + + return 0; +} + + +int32_t ctgHashValueComp2(void const* lp, void const* rp) { + uint32_t* key = (uint32_t*)lp; + SVgroupInfo* pVg = (SVgroupInfo*)rp; + + if (*key < pVg->hashBegin) { + return -1; + } else if (*key > pVg->hashEnd) { + return 1; + } + + return 0; +} + int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { int32_t code = 0; @@ -869,7 +920,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName } if (NULL == vgInfo) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, + ctgError("0no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -883,29 +934,78 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName CTG_RET(code); } -int32_t ctgHashValueComp(void const* lp, void const* rp) { - uint32_t* key = (uint32_t*)lp; - SVgroupInfo* pVg = *(SVgroupInfo**)rp; - if (*key < pVg->hashBegin) { - return -1; - } else if (*key > pVg->hashEnd) { - return 1; +int32_t ctgGetVgInfoFromHashValue2(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { + int32_t code = 0; + + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, db); + + if (vgNum <= 0) { + ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", db, vgNum); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); } - return 0; -} + SVgroupInfo* vgInfo = NULL; + char tbFullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pTableName, tbFullName); -int ctgVgInfoComp(const void* lp, const void* rp) { - SVgroupInfo* pLeft = *(SVgroupInfo**)lp; - SVgroupInfo* pRight = *(SVgroupInfo**)rp; - if (pLeft->hashBegin < pRight->hashBegin) { - return -1; - } else if (pLeft->hashBegin > pRight->hashBegin) { - return 1; + uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, + dbInfo->hashPrefix, dbInfo->hashSuffix); + + static SArray* pVgList = NULL; + static bool created = false; + + if (!created) { + ctgDebug("create vg array, %d", taosHashGetSize(dbInfo->vgHash)); + pVgList = taosArrayInit(100, sizeof(SVgroupInfo)); + + void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + taosArrayPush(pVgList, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); + } + + taosArraySort(pVgList, ctgVgInfoComp2); + created = true; } - return 0; + vgInfo = taosArraySearch(pVgList, &hashValue, ctgHashValueComp2, TD_EQ); + + if (NULL == vgInfo) { + void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + vgInfo = pIter; + if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { + taosHashCancelIterate(dbInfo->vgHash, pIter); + break; + } + + pIter = taosHashIterate(dbInfo->vgHash, pIter); + vgInfo = NULL; + } + + if (vgInfo) { + taosArrayDestroy(pVgList); + created = false; + ctgDebug("need to re-create vg array, %d", taosHashGetSize(dbInfo->vgHash)); + } + } + + if (NULL == vgInfo) { + ctgError("1no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, + taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + *pVgroup = *vgInfo; + + ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, + vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, + vgInfo->epSet.eps[vgInfo->epSet.inUse].port); + + CTG_RET(code); } int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, @@ -982,7 +1082,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* SVgroupInfo** p = taosArraySearch(pVgList, &hashValue, ctgHashValueComp, TD_EQ); if (NULL == p) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, + ctgError("2no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, taosHashGetSize(dbInfo->vgHash)); taosArrayDestroy(pVgList); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); From ee5d0a384f8df9f342e984a3be9de6c504aa2038 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 1 Dec 2022 19:05:33 +0800 Subject: [PATCH 29/95] fix: refactor table scan count --- source/libs/executor/inc/executorimpl.h | 12 +- source/libs/executor/src/scanoperator.c | 228 ++++++++++++++---------- 2 files changed, 141 insertions(+), 99 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 3896a8a419..9d797f8520 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -489,8 +489,8 @@ typedef struct STableCountScanSupp { bool groupByDbName; bool groupByStbName; - char dbName[TSDB_DB_NAME_LEN]; - char stbName[TSDB_TABLE_NAME_LEN]; + char dbNameFilter[TSDB_DB_NAME_LEN]; + char stbNameFilter[TSDB_TABLE_NAME_LEN]; } STableCountScanSupp; @@ -498,16 +498,10 @@ typedef struct STableCountScanOperatorInfo { SReadHandle readHandle; SSDataBlock* pRes; - SName tableName; - - SNodeList* groupTags; - SNodeList* scanCols; - SNodeList* pseudoCols; - STableCountScanSupp supp; int32_t currGrpIdx; - SArray* stbUidList; // when group by db_name and stable_name + SArray* stbUidList; // when group by db_name and/or stable_name } STableCountScanOperatorInfo; typedef struct SOptrBasicInfo { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3dd027ad04..07ef30f8c5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -486,10 +486,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, - tstrerror(terrno), idStr); + qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", + pBlock->info.id.uid, tstrerror(terrno), idStr); } else { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr); + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), + idStr); } metaReaderClear(&mr); return terrno; @@ -1435,8 +1436,8 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock NULL); if (closedWin && pInfo->partitionSup.needCalc) { gpId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId); - appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid, &gpId, - NULL); + appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid, + &gpId, NULL); } } } @@ -2461,7 +2462,8 @@ static void destroyTagScanOperatorInfo(void* param) { taosMemoryFreeClear(param); } -SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -2917,6 +2919,21 @@ _error: // TableCountScanOperator static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator); static void destoryTableCountScanOperator(void* param); +static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid); +static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName); +static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName); +static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName); +static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes); +static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum, + size_t perfdbTableNum); +static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes, + size_t infodbTableNum, size_t perfdbTableNum); static const char* GROUP_TAG_DB_NAME = "db_name"; static const char* GROUP_TAG_STABLE_NAME = "stable_name"; @@ -2978,8 +2995,8 @@ int32_t tblCountScanGetInputs(SNodeList* groupTags, SName* tableName, STableCoun } } } else { - strncpy(supp->dbName, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN); - strncpy(supp->stbName, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN); + strncpy(supp->dbNameFilter, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN); + strncpy(supp->stbNameFilter, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN); } return TSDB_CODE_SUCCESS; } @@ -3061,7 +3078,7 @@ void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* if (pSupp->stbNameSlotId != -1) { SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId); if (strlen(stbName) != 0) { - char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; strncpy(varDataVal(varStbName), stbName, strlen(stbName)); varDataSetLen(varStbName, strlen(stbName)); colDataAppend(colInfoData, 0, varStbName, false); @@ -3087,33 +3104,43 @@ static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountSc getPerfDbMeta(NULL, &perfdbTableNum); if (pSupp->groupByDbName) { - if (pInfo->currGrpIdx == 0) { - uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); - pRes->info.id.groupId = groupId; - fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); - } else if (pInfo->currGrpIdx == 1) { - uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB)); - pRes->info.id.groupId = groupId; - fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); - } else { - setOperatorCompleted(pOperator); - return NULL; - } - pInfo->currGrpIdx++; + buildSysDbGroupedTableCount(pOperator, pInfo, pSupp, pRes, infodbTableNum, perfdbTableNum); return (pRes->info.rows > 0) ? pRes : NULL; } else { - if (strcmp(pSupp->dbName, TSDB_INFORMATION_SCHEMA_DB) == 0) { - fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); - } else if (strcmp(pSupp->dbName, TSDB_PERFORMANCE_SCHEMA_DB) == 0) { - fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); - } else if (strlen(pSupp->dbName) == 0) { - fillTableCountScanDataBlock(pSupp, "", "", infodbTableNum + perfdbTableNum, pRes); - } - setOperatorCompleted(pOperator); + buildSysDbFilterTableCount(pOperator, pSupp, pRes, infodbTableNum, perfdbTableNum); return (pRes->info.rows > 0) ? pRes : NULL; } } +static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes, + size_t infodbTableNum, size_t perfdbTableNum) { + if (strcmp(pSupp->dbNameFilter, TSDB_INFORMATION_SCHEMA_DB) == 0) { + fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); + } else if (strcmp(pSupp->dbNameFilter, TSDB_PERFORMANCE_SCHEMA_DB) == 0) { + fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); + } else if (strlen(pSupp->dbNameFilter) == 0) { + fillTableCountScanDataBlock(pSupp, "", "", infodbTableNum + perfdbTableNum, pRes); + } + setOperatorCompleted(pOperator); +} + +static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum, + size_t perfdbTableNum) { + if (pInfo->currGrpIdx == 0) { + uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); + pRes->info.id.groupId = groupId; + fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); + } else if (pInfo->currGrpIdx == 1) { + uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB)); + pRes->info.id.groupId = groupId; + fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); + } else { + setOperatorCompleted(pOperator); + } + pInfo->currGrpIdx++; +} + static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableCountScanOperatorInfo* pInfo = pOperator->info; @@ -3128,89 +3155,110 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { return buildSysDbTableCount(pOperator, pInfo); } + return buildVnodeDbTableCount(pOperator, pInfo, pSupp, pRes); +} + +static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes) { const char* db = NULL; int32_t vgId = 0; char dbName[TSDB_DB_NAME_LEN] = {0}; - { - // get dbname - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - SName sn = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&sn, dbName); - } + // get dbname + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + SName sn = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&sn, dbName); + if (pSupp->groupByDbName) { - if (pSupp->groupByStbName) { - if (pInfo->stbUidList == NULL) { - pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); - if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) { - qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); - } + buildVnodeGroupedTableCount(pOperator, pInfo, pSupp, pRes, vgId, dbName); + } else { + buildVnodeFilteredTbCount(pOperator, pInfo, pSupp, pRes, dbName); + } + return pRes->info.rows > 0 ? pRes : NULL; +} + +static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) { + if (pSupp->groupByStbName) { + if (pInfo->stbUidList == NULL) { + pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); + if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) { + qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); } - if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) { - tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx); + } + if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) { + tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx); + buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid); - char stbName[TSDB_TABLE_NAME_LEN] = {0}; - metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); + pInfo->currGrpIdx++; + } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) { + buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName); - char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName); - uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); - pRes->info.id.groupId = groupId; - - SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); - int64_t ctbNum = stats.ctbNum; - - fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); - - pInfo->currGrpIdx++; - } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) { - char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, ""); - uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); - pRes->info.id.groupId = groupId; - int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); - fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); - - pInfo->currGrpIdx++; - } else { - setOperatorCompleted(pOperator); - return NULL; - } + pInfo->currGrpIdx++; } else { - uint64_t groupId = calcGroupId(dbName, strlen(dbName)); - pRes->info.id.groupId = groupId; - int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); - fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); setOperatorCompleted(pOperator); } } else { - if (strlen(pSupp->dbName) != 0) { - if (strlen(pSupp->stbName) != 0) { - tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbName); - SMetaStbStats stats = {0}; - metaGetStbStats(pInfo->readHandle.meta, uid, &stats); - int64_t ctbNum = stats.ctbNum; - fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbName, ctbNum, pRes); - } else { - int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); - fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); - } + uint64_t groupId = calcGroupId(dbName, strlen(dbName)); + pRes->info.id.groupId = groupId; + int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); + setOperatorCompleted(pOperator); + } +} + +static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { + if (strlen(pSupp->dbNameFilter) != 0) { + if (strlen(pSupp->stbNameFilter) != 0) { + tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter); + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, uid, &stats); + int64_t ctbNum = stats.ctbNum; + fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes); } else { int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } - setOperatorCompleted(pOperator); + } else { + int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); } - return pRes->info.rows > 0 ? pRes : NULL; + setOperatorCompleted(pOperator); +} + +static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName) { + char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, ""); + uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); + pRes->info.id.groupId = groupId; + int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); +} + +static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) { + char stbName[TSDB_TABLE_NAME_LEN] = {0}; + metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); + + char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName); + uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); + pRes->info.id.groupId = groupId; + + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); + int64_t ctbNum = stats.ctbNum; + + fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); } static void destoryTableCountScanOperator(void* param) { STableCountScanOperatorInfo* pTableCountScanInfo = param; blockDataDestroy(pTableCountScanInfo->pRes); - nodesDestroyList(pTableCountScanInfo->groupTags); taosArrayDestroy(pTableCountScanInfo->stbUidList); taosMemoryFreeClear(param); } From efcc3b06bb3af6791d670922c290ec5de297d3e9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 20:01:09 +0800 Subject: [PATCH 30/95] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 1 + source/libs/executor/src/executil.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7c39e378cb..f86540e3e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3870,6 +3870,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFreeClear(pSupInfo->plist); taosMemoryFree(pSupInfo->colIds); + taosMemoryFree(pSupInfo->slotIds); taosArrayDestroy(pSupInfo->pColAgg); for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 3f62191ca1..6c219f8bd4 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1637,7 +1637,10 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi return TSDB_CODE_SUCCESS; } -void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFreeClear(pCond->colList); } +void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { + taosMemoryFreeClear(pCond->colList); + taosMemoryFreeClear(pCond->pSlotList); +} int32_t convertFillType(int32_t mode) { int32_t type = TSDB_FILL_NONE; From 043cc1104e9b4be09f2d8a82a1341bd3fb484266 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 1 Dec 2022 20:26:56 +0800 Subject: [PATCH 31/95] more code --- source/dnode/vnode/src/vnd/vnodeCommit.c | 74 +++++++++++++++------- source/dnode/vnode/src/vnd/vnodeOpen.c | 6 +- source/dnode/vnode/src/vnd/vnodeSnapshot.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +- 4 files changed, 55 insertions(+), 33 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 4bdaf8d353..ce9404348c 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -15,12 +15,11 @@ #include "vnd.h" -#define VND_INFO_FNAME "vnode.json" +#define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo); -static int vnodeCommitImpl(void *arg); static void vnodeWaitCommit(SVnode *pVnode); int vnodeBegin(SVnode *pVnode) { @@ -107,7 +106,8 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // free info binary taosMemoryFree(data); - vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d", pInfo->config.vgId, fname, pInfo->config.syncCfg.replicaNum); + vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d", pInfo->config.vgId, fname, + pInfo->config.syncCfg.replicaNum); return 0; @@ -185,21 +185,61 @@ _err: return -1; } +typedef struct { + SVnodeInfo info; + SVnode *pVnode; +} SCommitInfo; +static void vnodePrepareCommit(SVnode *pVnode) { + tsem_wait(&pVnode->canCommit); + + vnodeBufPoolUnRef(pVnode->inUse); + pVnode->inUse = NULL; +} +static int32_t vnodeCommitTask(void *arg) { + int32_t code = 0; + + SVnode *pVnode = (SVnode *)pVnode; + + code = vnodeCommit(pVnode); + if (code) goto _exit; + + tsem_post(&pVnode->canCommit); + +_exit: + return code; +} int vnodeAsyncCommit(SVnode *pVnode) { - vnodeWaitCommit(pVnode); + int32_t code = 0; - // vnodeBufPoolSwitch(pVnode); - // tsdbPrepareCommit(pVnode->pTsdb); + // prepare to commit + vnodePrepareCommit(pVnode); - vnodeScheduleTask(vnodeCommitImpl, pVnode); + // schedule the task + SVnodeInfo *pInfo = (SVnodeInfo *)taosMemoryCalloc(1, sizeof(*pInfo)); + if (NULL == pInfo) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pInfo->config = pVnode->config; + pInfo->state.committed = pVnode->state.applied; + pInfo->state.commitTerm = pVnode->state.applyTerm; + pInfo->state.commitID = pVnode->state.commitID; + vnodeScheduleTask(vnodeCommitTask, pVnode); - return 0; +_exit: + if (code) { + vError("vgId:%d %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), + pVnode->state.commitID); + } else { + vDebug("vgId:%d %s done", TD_VID(pVnode), __func__); + } + return code; } int vnodeSyncCommit(SVnode *pVnode) { vnodeAsyncCommit(pVnode); - vnodeWaitCommit(pVnode); - tsem_post(&(pVnode->canCommit)); + tsem_wait(&pVnode->canCommit); + tsem_post(&pVnode->canCommit); return 0; } @@ -318,20 +358,6 @@ void vnodeRollback(SVnode *pVnode) { (void)taosRemoveFile(tFName); } -static int vnodeCommitImpl(void *arg) { - SVnode *pVnode = (SVnode *)arg; - - // metaCommit(pVnode->pMeta); - tqCommit(pVnode->pTq); - // tsdbCommit(pVnode->pTsdb, ); - - // vnodeBufPoolRecycle(pVnode); - tsem_post(&(pVnode->canCommit)); - return 0; -} - -static FORCE_INLINE void vnodeWaitCommit(SVnode *pVnode) { tsem_wait(&pVnode->canCommit); } - static int vnodeEncodeState(const void *pObj, SJson *pJson) { const SVState *pState = (SVState *)pObj; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 0ff4a46d44..e09fafb756 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -242,14 +242,14 @@ _err: return NULL; } -void vnodePreClose(SVnode *pVnode) { +void vnodePreClose(SVnode *pVnode) { vnodeQueryPreClose(pVnode); - vnodeSyncPreClose(pVnode); + vnodeSyncPreClose(pVnode); } void vnodeClose(SVnode *pVnode) { if (pVnode) { - vnodeCommit(pVnode); + vnodeSyncCommit(pVnode); vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); walClose(pVnode->pWal); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index e8cdf9513f..a34744a1da 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -259,7 +259,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr pWriter->ever = ever; // commit it - code = vnodeCommit(pVnode); + code = vnodeSyncCommit(pVnode); if (code) { taosMemoryFree(pWriter); goto _err; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d63c45fc0e..d8079ca437 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -311,11 +311,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp if (vnodeShouldCommit(pVnode)) { _do_commit: vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); - // commit current change - if (vnodeCommit(pVnode) < 0) { - vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + vnodeAsyncCommit(pVnode); // start a new one if (vnodeBegin(pVnode) < 0) { From 1750b5cc92332fd68d551bdfc83fcec7a9ca03f1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Dec 2022 20:34:06 +0800 Subject: [PATCH 32/95] remove invalid table --- source/libs/executor/src/executil.c | 18 +++++++++--------- source/util/src/tarray.c | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 8a85885d98..d5bd2c6010 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -438,13 +438,12 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* goto end; } } + removeInvalidTable(uidList, tags); int32_t rows = taosArrayGetSize(uidList); if (rows == 0) { goto end; } - // int64_t stt1 = taosGetTimestampUs(); - // qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); code = blockDataEnsureCapacity(pResBlock, rows); if (code != TSDB_CODE_SUCCESS) { @@ -452,7 +451,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* goto end; } - // int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { int64_t* uid = taosArrayGet(uidList, i); for (int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++) { @@ -467,7 +465,9 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* #endif } else { void* tag = taosHashGet(tags, uid, sizeof(int64_t)); - ASSERT(tag); + if (tag == NULL) { + continue; + } STagVal tagVal = {0}; tagVal.cid = pColInfo->info.colId; const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); @@ -923,14 +923,14 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* return -1; } - SArray* pTbList = getTableNameList(pList); - int32_t numOfTables = taosArrayGetSize(pTbList); - SHashObj *uHash = NULL; + SArray* pTbList = getTableNameList(pList); + int32_t numOfTables = taosArrayGetSize(pTbList); + SHashObj* uHash = NULL; size_t listlen = taosArrayGetSize(list); // len > 0 means there already have uids if (listlen > 0) { uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); for (int i = 0; i < listlen; i++) { - int64_t *uid = taosArrayGet(list, i); + int64_t* uid = taosArrayGet(list, i); taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); } } @@ -2011,4 +2011,4 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag) { char* pBuf = NULL; qDebug("%s", dumpBlockData(pBlock, flag, &pBuf)); taosMemoryFree(pBuf); -} \ No newline at end of file +} diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index ae25786375..4bd8294423 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -319,9 +319,9 @@ SArray* taosArrayDup(const SArray* pSrc, __array_item_dup_fn_t fn) { } else { ASSERT(pSrc->elemSize == sizeof(void*)); - for(int32_t i = 0; i < pSrc->size; ++i) { + for (int32_t i = 0; i < pSrc->size; ++i) { void* p = fn(taosArrayGetP(pSrc, i)); - memcpy(((char*)dst->pData )+ i * dst->elemSize, &p, dst->elemSize); + memcpy(((char*)dst->pData) + i * dst->elemSize, &p, dst->elemSize); } } From 24614d03bd2ef21b454d8b9c558fb29977418960 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 21:35:01 +0800 Subject: [PATCH 33/95] fix(query): set correct sma retrieve procedure. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 42 ++++++++++++++----------- source/libs/executor/src/scanoperator.c | 2 ++ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f86540e3e2..24e9181ae6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -84,7 +84,7 @@ typedef struct SIOCostSummary { typedef struct SBlockLoadSuppInfo { SArray* pColAgg; SColumnDataAgg tsColAgg; - SColumnDataAgg** plist; + SColumnDataAgg** plist; // todo remove this int16_t* colIds; // column ids for loading file block data int16_t* slotIds; // the ordinal index in the destination SSDataBlock int32_t numOfCols; @@ -4041,12 +4041,11 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS } SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); - - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); - // int64_t stime = taosGetTimestampUs(); - SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + ASSERT(pReader->pResBlock->info.id.uid == pFBlock->uid); + + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); if (tDataBlkHasSma(pBlock)) { code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg); if (code != TSDB_CODE_SUCCESS) { @@ -4071,7 +4070,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS pSup->plist[0] = pTsAgg; // update the number of NULL data rows - size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock); + size_t numOfCols = pSup->numOfCols; int32_t i = 0, j = 0; size_t size = taosArrayGetSize(pSup->pColAgg); @@ -4095,43 +4094,48 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS } #else + SSDataBlock* pResBlock = pReader->pResBlock; + if (pResBlock->pBlockAgg == NULL) { + size_t num = taosArrayGetSize(pResBlock->pDataBlock); + pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg)); + } + // fill the all null data column - SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg)); +// SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg)); while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); if (pAgg->colId == pSup->colIds[j]) { - taosArrayPush(pNewAggList, pAgg); + pResBlock->pBlockAgg[pSup->slotIds[j]] = pAgg; i += 1; j += 1; } else if (pAgg->colId < pSup->colIds[j]) { i += 1; } else if (pSup->colIds[j] < pAgg->colId) { if (pSup->colIds[j] == PRIMARYKEY_TIMESTAMP_COL_ID) { - taosArrayPush(pNewAggList, &pSup->tsColAgg); + pResBlock->pBlockAgg[pSup->slotIds[j]] = &pSup->tsColAgg; } else { // all date in this block are null SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow}; - taosArrayPush(pNewAggList, &nullColAgg); + taosArrayPush(pSup->pColAgg, &nullColAgg); + + pResBlock->pBlockAgg[pSup->slotIds[j]] = taosArrayGetLast(pSup->pColAgg); } j += 1; } } - taosArrayClear(pSup->pColAgg); - taosArrayAddAll(pSup->pColAgg, pNewAggList); +// taosArrayClear(pSup->pColAgg); - size_t num = taosArrayGetSize(pSup->pColAgg); - for(int32_t k = 0; k < num; ++k) { - pSup->plist[k] = taosArrayGet(pSup->pColAgg, k); - } - - taosArrayDestroy(pNewAggList); +// size_t num = taosArrayGetSize(pSup->pColAgg); +// for(int32_t k = 0; k < num; ++k) { +// pSup->plist[k] = taosArrayGet(pSup->pColAgg, k); +// } #endif pReader->cost.smaDataLoad += 1; - *pBlockStatis = pSup->plist; +// *pBlockStatis = pSup->plist; tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr); return code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 87aacfdd34..b0104c6025 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -236,6 +236,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, return false; } +#if 0 // if (allColumnsHaveAgg == true) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); @@ -256,6 +257,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i]; } +#endif return true; } From 1faf616f1d089ce592c8dcd56326d7140c75aea0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Dec 2022 22:17:53 +0800 Subject: [PATCH 34/95] fix(query): fix null ptr access in evaluation block distribution. --- source/libs/executor/src/sysscanoperator.c | 37 ++++++++++++---------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 7ac6cee2af..96f21babe6 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1874,27 +1874,30 @@ static void destroyBlockDistScanOperatorInfo(void* param) { } static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) { - memset(pCond, 0, sizeof(SQueryTableDataCond)); + memset(pCond, 0, sizeof(SQueryTableDataCond)); - pCond->order = TSDB_ORDER_ASC; - pCond->numOfCols = 1; - pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); - if (pCond->colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return terrno; - } + pCond->order = TSDB_ORDER_ASC; + pCond->numOfCols = 1; + pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); + pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)); + if (pCond->colList == NULL || pCond->pSlotList == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return terrno; + } - pCond->colList->colId = 1; - pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP; - pCond->colList->bytes = sizeof(TSKEY); + pCond->colList->colId = 1; + pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP; + pCond->colList->bytes = sizeof(TSKEY); - pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; - pCond->suid = uid; - pCond->type = TIMEWINDOW_RANGE_CONTAINED; - pCond->startVersion = -1; - pCond->endVersion = -1; + pCond->pSlotList[0] = 0; - return TSDB_CODE_SUCCESS; + pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + pCond->suid = uid; + pCond->type = TIMEWINDOW_RANGE_CONTAINED; + pCond->startVersion = -1; + pCond->endVersion = -1; + + return TSDB_CODE_SUCCESS; } SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, From 83782fb27bf679b44353511e0eb70393e4193902 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Dec 2022 10:07:48 +0800 Subject: [PATCH 35/95] fix(query): fix invalid tag fill and clear nullbitmp --- include/common/tdatablock.h | 6 ++--- source/common/src/tdatablock.c | 8 +++---- source/libs/executor/src/tfill.c | 28 +++++++++++++++++----- tests/script/tsim/parser/lastrow_query.sim | 4 ++-- tests/script/tsim/valgrind/checkUdf.sim | 3 ++- 5 files changed, 33 insertions(+), 16 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 4208d85648..e1d3b01611 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -41,9 +41,9 @@ typedef struct SBlockOrderInfo { BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) -#define colDataClearNull_f(bm_, r_) \ - do { \ - BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \ +#define colDataClearNull_f(bm_, r_) \ + do { \ + BMCharPos(bm_, r_) &= ((char)(~(1u << (7u - BitPos(r_))))); \ } while (0) #define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c300c3e6f0..e9be89a68b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1679,15 +1679,15 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_ pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); } else { // reset the bitmap value - int32_t stopIndex = BitmapLen(n) * 8; - for(int32_t i = n + 1; i < stopIndex; ++i) { + /*int32_t stopIndex = BitmapLen(n) * 8; + for(int32_t i = n; i < stopIndex; ++i) { colDataClearNull_f(pColInfoData->nullbitmap, i); } int32_t remain = BitmapLen(total) - BitmapLen(n); if (remain > 0) { - memset(pColInfoData->nullbitmap, 0, remain); - } + memset(pColInfoData->nullbitmap+BitmapLen(n), 0, remain); + }*/ } } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 42b1d058b5..4c8967744e 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -60,8 +60,18 @@ static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowInd if (pCol->notFillCol) { bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstColInfo, rowIndex); if (!filled) { - SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; - SGroupKeys* pKey = taosArrayGet(p, i); + SRowVal* p = NULL; + if (FILL_IS_ASC_FILL(pFillInfo)) { + if (pFillInfo->prev.key != 0) { + p = &pFillInfo->prev; // prev has been set value + } else { // otherwise, use the value in the next row + p = &pFillInfo->next; + } + } else { + p = &pFillInfo->next; + } + + SGroupKeys* pKey = taosArrayGet(p->pRowVal, i); doSetVal(pDstColInfo, rowIndex, pKey); } } else { @@ -261,7 +271,10 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) { static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull); -static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray* pRow) { +static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal) { + SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, pFillInfo->srcTsSlotId); + pRowVal->key = ((int64_t*)pTsCol->pData)[rowIndex]; + for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { int32_t type = pFillInfo->pFillCol[i].pExpr->pExpr->nodeType; if (type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) { @@ -272,7 +285,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray bool isNull = colDataIsNull_s(pSrcCol, rowIndex); char* p = colDataGetData(pSrcCol, rowIndex); - saveColData(pRow, i, p, isNull); + saveColData(pRowVal->pRowVal, i, p, isNull); } else { ASSERT(0); } @@ -296,7 +309,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t // set the next value for interpolation if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) { - copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next.pRowVal); + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, &pFillInfo->next); } if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && @@ -318,7 +331,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) { int32_t nextRowIndex = pFillInfo->index + 1; - copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, pFillInfo->next.pRowVal); + copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next); } // copy rows to dst buffer @@ -334,6 +347,9 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t if (!colDataIsNull_s(pSrc, pFillInfo->index)) { colDataAppend(pDst, index, src, false); saveColData(pFillInfo->prev.pRowVal, i, src, false); + if (pFillInfo->srcTsSlotId == dstSlotId) { + pFillInfo->prev.key = *(int64_t*)src; + } } else { // the value is null if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { colDataAppend(pDst, index, (const char*)&pFillInfo->currentKey, false); diff --git a/tests/script/tsim/parser/lastrow_query.sim b/tests/script/tsim/parser/lastrow_query.sim index 5f557fd7bd..9ca9d407b8 100644 --- a/tests/script/tsim/parser/lastrow_query.sim +++ b/tests/script/tsim/parser/lastrow_query.sim @@ -70,10 +70,10 @@ sql select _wstart, t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00 if $row != 2 then return -1 endi -if $data01 != NULL then +if $data01 != 8 then return -1 endi -if $data02 != NULL then +if $data02 != 8 then return -1 endi if $data03 != NULL then diff --git a/tests/script/tsim/valgrind/checkUdf.sim b/tests/script/tsim/valgrind/checkUdf.sim index 594d649714..caf316bd86 100644 --- a/tests/script/tsim/valgrind/checkUdf.sim +++ b/tests/script/tsim/valgrind/checkUdf.sim @@ -121,12 +121,13 @@ if $data01 != 152.420471066 then return -1 endi -sql select udf2(f2) from udf.t2 group by 1-udf1(f1); +sql select udf2(f2) from udf.t2 group by 1-udf1(f1) order by 1-udf1(f1) print $rows , $data00 , $data10 if $rows != 2 then return -1 endi if $data00 != 2.000000000 then + print expect 2.000000000 , actual: $data00 return -1 endi if $data10 != 12.083045974 then From 129c692f3e3eb40f263bda41d08e09f719c67814 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Dec 2022 15:44:24 +0800 Subject: [PATCH 36/95] more async commit --- source/dnode/vnode/src/inc/vnd.h | 3 +- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tsdb/tsdbCommit.c | 23 ++++---- source/dnode/vnode/src/vnd/vnodeCommit.c | 67 ++++++++++++------------ 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index e2b7327e8f..b62b8b533e 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -77,7 +77,7 @@ void vnodeBufPoolReset(SVBufPool* pPool); // vnodeQuery.c int32_t vnodeQueryOpen(SVnode* pVnode); -void vnodeQueryPreClose(SVnode *pVnode); +void vnodeQueryPreClose(SVnode* pVnode); void vnodeQueryClose(SVnode* pVnode); int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg, bool direct); int vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg, bool direct); @@ -86,7 +86,6 @@ int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg); // vnodeCommit.c int32_t vnodeBegin(SVnode* pVnode); int32_t vnodeShouldCommit(SVnode* pVnode); -int32_t vnodeCommit(SVnode* pVnode); void vnodeRollback(SVnode* pVnode); int32_t vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg); int32_t vnodeCommitInfo(const char* dir, const SVnodeInfo* pInfo); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index f229b3b127..f8a9522d7f 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -154,6 +154,7 @@ int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback); int tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); +int32_t tsdbPrepareCommit(STsdb* pTsdb); int32_t tsdbCommit(STsdb* pTsdb); int32_t tsdbFinishCommit(STsdb* pTsdb); int32_t tsdbRollbackCommit(STsdb* pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 391e10e223..f3acfca39c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -150,20 +150,26 @@ _exit: return code; } +int32_t tsdbPrepareCommit(STsdb *pTsdb) { + taosThreadRwlockWrlock(&pTsdb->rwLock); + ASSERT(pTsdb->imem == NULL); + pTsdb->imem = pTsdb->mem; + pTsdb->mem = NULL; + taosThreadRwlockUnlock(&pTsdb->rwLock); + + return 0; +} + int32_t tsdbCommit(STsdb *pTsdb) { if (!pTsdb) return 0; int32_t code = 0; int32_t lino = 0; SCommitter commith; - SMemTable *pMemTable = pTsdb->mem; + SMemTable *pMemTable = pTsdb->imem; // check if (pMemTable->nRow == 0 && pMemTable->nDel == 0) { - taosThreadRwlockWrlock(&pTsdb->rwLock); - pTsdb->mem = NULL; - taosThreadRwlockUnlock(&pTsdb->rwLock); - tsdbUnrefMemTable(pMemTable); goto _exit; } @@ -811,12 +817,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { int32_t lino = 0; memset(pCommitter, 0, sizeof(*pCommitter)); - ASSERT(pTsdb->mem && pTsdb->imem == NULL && "last tsdb commit incomplete"); - - taosThreadRwlockWrlock(&pTsdb->rwLock); - pTsdb->imem = pTsdb->mem; - pTsdb->mem = NULL; - taosThreadRwlockUnlock(&pTsdb->rwLock); + ASSERT(pTsdb->imem && "last tsdb commit incomplete"); pCommitter->pTsdb = pTsdb; pCommitter->commitID = pTsdb->pVnode->state.commitID; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index ce9404348c..79e4f46241 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -15,12 +15,17 @@ #include "vnd.h" +typedef struct { + SVnodeInfo info; + SVnode *pVnode; +} SCommitInfo; + #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" -static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); -static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo); -static void vnodeWaitCommit(SVnode *pVnode); +static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); +static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo); +static int vnodeCommitImpl(SCommitInfo *pInfo); int vnodeBegin(SVnode *pVnode) { // alloc buffer pool @@ -185,27 +190,28 @@ _err: return -1; } -typedef struct { - SVnodeInfo info; - SVnode *pVnode; -} SCommitInfo; static void vnodePrepareCommit(SVnode *pVnode) { tsem_wait(&pVnode->canCommit); vnodeBufPoolUnRef(pVnode->inUse); pVnode->inUse = NULL; + + tsdbPrepareCommit(pVnode->pTsdb); } static int32_t vnodeCommitTask(void *arg) { int32_t code = 0; - SVnode *pVnode = (SVnode *)pVnode; + SCommitInfo *pInfo = (SCommitInfo *)arg; - code = vnodeCommit(pVnode); + // commit + code = vnodeCommitImpl(pInfo); if (code) goto _exit; - tsem_post(&pVnode->canCommit); + // end commit + tsem_post(&pInfo->pVnode->canCommit); _exit: + taosMemoryFree(pInfo); return code; } int vnodeAsyncCommit(SVnode *pVnode) { @@ -215,16 +221,18 @@ int vnodeAsyncCommit(SVnode *pVnode) { vnodePrepareCommit(pVnode); // schedule the task - SVnodeInfo *pInfo = (SVnodeInfo *)taosMemoryCalloc(1, sizeof(*pInfo)); + pVnode->state.commitTerm = pVnode->state.applyTerm; + + SCommitInfo *pInfo = (SCommitInfo *)taosMemoryCalloc(1, sizeof(*pInfo)); if (NULL == pInfo) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - pInfo->config = pVnode->config; - pInfo->state.committed = pVnode->state.applied; - pInfo->state.commitTerm = pVnode->state.applyTerm; - pInfo->state.commitID = pVnode->state.commitID; - vnodeScheduleTask(vnodeCommitTask, pVnode); + pInfo->info.config = pVnode->config; + pInfo->info.state.committed = pVnode->state.applied; + pInfo->info.state.commitTerm = pVnode->state.applyTerm; + pInfo->info.state.commitID = pVnode->state.commitID; + vnodeScheduleTask(vnodeCommitTask, pInfo); _exit: if (code) { @@ -243,11 +251,12 @@ int vnodeSyncCommit(SVnode *pVnode) { return 0; } -int vnodeCommit(SVnode *pVnode) { - int32_t code = 0; - int32_t lino = 0; - SVnodeInfo info = {0}; - char dir[TSDB_FILENAME_LEN]; +static int vnodeCommitImpl(SCommitInfo *pInfo) { + int32_t code = 0; + int32_t lino = 0; + + char dir[TSDB_FILENAME_LEN] = {0}; + SVnode *pVnode = pInfo->pVnode; vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode), pVnode->state.commitID, pVnode->state.applied, pVnode->state.applyTerm); @@ -258,19 +267,13 @@ int vnodeCommit(SVnode *pVnode) { return -1; } - pVnode->state.commitTerm = pVnode->state.applyTerm; - // save info - info.config = pVnode->config; - info.state.committed = pVnode->state.applied; - info.state.commitTerm = pVnode->state.applyTerm; - info.state.commitID = pVnode->state.commitID; if (pVnode->pTfs) { snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); } else { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } - if (vnodeSaveInfo(dir, &info) < 0) { + if (vnodeSaveInfo(dir, &pInfo->info) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } @@ -281,9 +284,6 @@ int vnodeCommit(SVnode *pVnode) { code = smaPreCommit(pVnode->pSma); TSDB_CHECK_CODE(code, lino, _exit); - vnodeBufPoolUnRef(pVnode->inUse); - pVnode->inUse = NULL; - // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { code = TSDB_CODE_FAILED; @@ -304,7 +304,7 @@ int vnodeCommit(SVnode *pVnode) { } // commit info - if (vnodeCommitInfo(dir, &info) < 0) { + if (vnodeCommitInfo(dir, &pInfo->info) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } @@ -322,14 +322,13 @@ int vnodeCommit(SVnode *pVnode) { TSDB_CHECK_CODE(code, lino, _exit); } - pVnode->state.committed = info.state.committed; + pVnode->state.committed = pInfo->info.state.committed; if (smaPostCommit(pVnode->pSma) < 0) { vError("vgId:%d, failed to post-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } - // apply the commit (TODO) // walEndSnapshot(pVnode->pWal); syncEndSnapshot(pVnode->sync); From 1e8b9fdda0195861a51fff76ab166cbb82f1c7b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 2 Dec 2022 15:46:36 +0800 Subject: [PATCH 37/95] more core --- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d8079ca437..2d26358638 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -291,7 +291,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp); break; case TDMT_VND_COMMIT: - goto _do_commit; + vnodeSyncCommit(pVnode); + vnodeBegin(pVnode); + goto _exit; default: vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType); return -1; @@ -309,7 +311,6 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp // commit if need if (vnodeShouldCommit(pVnode)) { - _do_commit: vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); vnodeAsyncCommit(pVnode); @@ -320,6 +321,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } } +_exit: return 0; _err: From 42a512c39fa8a456496d1b154dc5be22b1bbb1bf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Dec 2022 15:52:32 +0800 Subject: [PATCH 38/95] refactor: do some internal refactor. --- source/dnode/vnode/inc/vnode.h | 22 +-- source/dnode/vnode/src/tsdb/tsdbRead.c | 176 +++++++++++------------- source/libs/executor/src/executor.c | 18 ++- source/libs/executor/src/scanoperator.c | 37 ++--- tests/system-test/2-query/unique.py | 6 +- 5 files changed, 116 insertions(+), 143 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 53b8eadb33..64c15c6c0e 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -161,17 +161,17 @@ int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t n int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr); -void tsdbReaderClose(STsdbReader *pReader); -bool tsdbNextDataBlock(STsdbReader *pReader); -void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); -SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); -int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); -int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); -int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); -void *tsdbGetIdx(SMeta *pMeta); -void *tsdbGetIvtIdx(SMeta *pMeta); -uint64_t getReaderMaxVersion(STsdbReader *pReader); +void tsdbReaderClose(STsdbReader *pReader); +bool tsdbNextDataBlock(STsdbReader *pReader); +void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockSMA, bool *allHave); +SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); +int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); +int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); +void *tsdbGetIdx(SMeta *pMeta); +void *tsdbGetIvtIdx(SMeta *pMeta); +uint64_t getReaderMaxVersion(STsdbReader *pReader); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, uint64_t suid, void **pReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 24e9181ae6..859ae026a2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -81,12 +81,15 @@ typedef struct SIOCostSummary { double createScanInfoList; } SIOCostSummary; +typedef struct SColInfo { + int16_t colId; + int16_t slotId; +} SColInfo; + typedef struct SBlockLoadSuppInfo { SArray* pColAgg; SColumnDataAgg tsColAgg; - SColumnDataAgg** plist; // todo remove this - int16_t* colIds; // column ids for loading file block data - int16_t* slotIds; // the ordinal index in the destination SSDataBlock + SColInfo* colInfo; // column ids for loading file block data int32_t numOfCols; char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. bool smaValid; // the sma on all queried columns are activated @@ -159,6 +162,7 @@ struct STsdbReader { STsdb* pTsdb; uint64_t suid; int16_t order; + bool freeBlock; STimeWindow window; // the primary query time window that applies to all queries SSDataBlock* pResBlock; int32_t capacity; @@ -218,22 +222,21 @@ static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWi static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) { pSupInfo->smaValid = true; pSupInfo->numOfCols = numOfCols; - pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t)); - pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES); - pSupInfo->slotIds = taosMemoryMalloc(numOfCols * sizeof(int16_t)); - if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL || pSupInfo->slotIds == NULL) { - taosMemoryFree(pSupInfo->colIds); - taosMemoryFree(pSupInfo->buildBuf); - taosMemoryFree(pSupInfo->slotIds); + pSupInfo->colInfo = taosMemoryMalloc(numOfCols * (sizeof(SColInfo) + POINTER_BYTES)); + if (pSupInfo->colInfo == NULL) { + taosMemoryFree(pSupInfo->colInfo); return TSDB_CODE_OUT_OF_MEMORY; } + pSupInfo->buildBuf = (char**)((char*)pSupInfo->colInfo + (sizeof(SColInfo) * numOfCols)); for (int32_t i = 0; i < numOfCols; ++i) { - pSupInfo->colIds[i] = pCols[i].colId; - pSupInfo->slotIds[i] = pSlotIdList[i]; + pSupInfo->colInfo[i].colId = pCols[i].colId; + pSupInfo->colInfo[i].slotId = pSlotIdList[i]; if (IS_VAR_DATA_TYPE(pCols[i].type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); + } else { + pSupInfo->buildBuf[i] = NULL; } } @@ -245,7 +248,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) { STColumn* pTCol = &pSchema->columns[i]; - if (pTCol->colId == pSupInfo->colIds[j]) { + if (pTCol->colId == pSupInfo->colInfo[j].colId) { if (!IS_BSMA_ON(pTCol)) { pSupInfo->smaValid = false; return; @@ -253,7 +256,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) i += 1; j += 1; - } else if (pTCol->colId < pSupInfo->colIds[j]) { + } else if (pTCol->colId < pSupInfo->colInfo[j].colId) { // do nothing i += 1; } else { @@ -455,7 +458,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb if (pLReader->pInfo == NULL) { // here we ignore the first column, which is always be the primary timestamp column pLReader->pInfo = - tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1); + tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1); if (pLReader->pInfo == NULL) { tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr); return terrno; @@ -594,14 +597,22 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket ASSERT(pCond->numOfCols > 0); + if (pReader->pResBlock == NULL) { + pReader->freeBlock = true; + pReader->pResBlock = createResBlock(pCond, pReader->capacity); + if (pReader->pResBlock == NULL) { + code = terrno; + goto _end; + } + } + // todo refactor. limitOutputBufferSize(pCond, &pReader->capacity); // allocate buffer in order to load data blocks from file SBlockLoadSuppInfo* pSup = &pReader->suppInfo; pSup->pColAgg = taosArrayInit(pCond->numOfCols, sizeof(SColumnDataAgg)); - pSup->plist = taosMemoryCalloc(pCond->numOfCols, POINTER_BYTES); - if (pSup->pColAgg == NULL || pSup->plist == NULL) { + if (pSup->pColAgg == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } @@ -1083,8 +1094,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t i = 0; int32_t rowIndex = 0; - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); - if (pSupInfo->colIds[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + if (pSupInfo->colInfo[i].colId == PRIMARYKEY_TIMESTAMP_COL_ID) { copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); i += 1; } @@ -1095,10 +1106,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn rowIndex = 0; SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex); - if (pData->cid < pSupInfo->colIds[i]) { + if (pData->cid < pSupInfo->colInfo[i].colId) { colIndex += 1; - } else if (pData->cid == pSupInfo->colIds[i]) { - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + } else if (pData->cid == pSupInfo->colInfo[i].colId) { + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { colDataAppendNNULL(pColData, 0, dumpedRows); @@ -1116,7 +1127,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colIndex += 1; i += 1; } else { // the specified column does not exist in file block, fill with null data - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1124,7 +1135,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // fill the mis-matched columns with null value while (i < numOfOutputCols) { - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1162,7 +1173,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI tBlockDataReset(pBlockData); TABLEID tid = {.suid = pReader->suid, .uid = uid}; int32_t code = - tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1); + tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1621,7 +1632,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* int64_t st = taosGetTimestampUs(); int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader); - blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotIds[0]); + blockDataUpdateTsWindow(pBlock, pReader->suppInfo.colInfo[0].slotId); pBlock->info.id.uid = pBlockScanInfo->uid; setComposedBlockFlag(pReader, true); @@ -2493,7 +2504,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { _end: pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; - blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotIds[0]); + blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.colInfo[0].slotId); setComposedBlockFlag(pReader, true); double el = (taosGetTimestampUs() - st) / 1000.0; @@ -3542,24 +3553,24 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* SColVal colVal = {0}; int32_t i = 0, j = 0; - if (pSupInfo->colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + if (pSupInfo->colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; } while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) { - col_id_t colId = pSupInfo->colIds[i]; + col_id_t colId = pSupInfo->colInfo[i].colId; if (colId == pSchema->columns[j].colId) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); tTSRowGetVal(pTSRow, pSchema, j, &colVal); doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); i += 1; j += 1; } else if (colId < pSchema->columns[j].colId) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; @@ -3570,7 +3581,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* // set null value since current column does not exist in the "pSchema" while (i < pSupInfo->numOfCols) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; } @@ -3586,8 +3597,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t outputRowIndex = pResBlock->info.rows; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - if (pReader->suppInfo.colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + if (pReader->suppInfo.colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; } @@ -3598,13 +3609,13 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S while (i < numOfOutputCols && j < numOfInputCols) { SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); - if (pData->cid < pSupInfo->colIds[i]) { + if (pData->cid < pSupInfo->colInfo[i].colId) { j += 1; continue; } - SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotIds[i]); - if (pData->cid == pSupInfo->colIds[i]) { + SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + if (pData->cid == pSupInfo->colInfo[i].colId) { tColDataGetValue(pData, rowIndex, &cv); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); j += 1; @@ -3617,7 +3628,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S } while (i < numOfOutputCols) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]); + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); colDataAppendNULL(pCol, outputRowIndex); i += 1; } @@ -3723,7 +3734,14 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL pCond->twindows.ekey -= 1; } - int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, pResBlock->info.capacity, pResBlock, idstr); + int32_t capacity = 0; + if (pResBlock == NULL) { + capacity = 4096; + } else { + capacity = pResBlock->info.capacity; + } + + int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3868,10 +3886,6 @@ void tsdbReaderClose(STsdbReader* pReader) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - taosMemoryFreeClear(pSupInfo->plist); - taosMemoryFree(pSupInfo->colIds); - taosMemoryFree(pSupInfo->slotIds); - taosArrayDestroy(pSupInfo->pColAgg); for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) { if (pSupInfo->buildBuf[i] != NULL) { @@ -3879,9 +3893,12 @@ void tsdbReaderClose(STsdbReader* pReader) { } } - taosMemoryFree(pSupInfo->buildBuf); - tBlockDataDestroy(&pReader->status.fileBlockData, true); + if (pReader->freeBlock) { + pReader->pResBlock = blockDataDestroy(pReader->pResBlock); + } + taosMemoryFree(pSupInfo->colInfo); + tBlockDataDestroy(&pReader->status.fileBlockData, true); cleanupDataBlockIterator(&pReader->status.blockIter); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); @@ -4025,18 +4042,18 @@ void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64 } } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg ***pBlockSMA, bool* allHave) { int32_t code = 0; *allHave = false; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { - *pBlockStatis = NULL; + *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } // there is no statistics data for composed block if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) { - *pBlockStatis = NULL; + *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } @@ -4054,7 +4071,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS return code; } } else { - *pBlockStatis = NULL; + *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } @@ -4067,32 +4084,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID; pTsAgg->min = pReader->pResBlock->info.window.skey; pTsAgg->max = pReader->pResBlock->info.window.ekey; - pSup->plist[0] = pTsAgg; // update the number of NULL data rows size_t numOfCols = pSup->numOfCols; int32_t i = 0, j = 0; size_t size = taosArrayGetSize(pSup->pColAgg); -#if 0 - while (j < numOfCols && i < size) { - SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); - if (pAgg->colId == pSup->colIds[j]) { - if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) { - pSup->plist[j] = pAgg; - } else { - *allHave = false; - break; - } - i += 1; - j += 1; - } else if (pAgg->colId < pSup->colIds[j]) { - i += 1; - } else if (pSup->colIds[j] < pAgg->colId) { - j += 1; - } - } -#else SSDataBlock* pResBlock = pReader->pResBlock; if (pResBlock->pBlockAgg == NULL) { @@ -4100,52 +4097,40 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg)); } - // fill the all null data column -// SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg)); - while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); - if (pAgg->colId == pSup->colIds[j]) { - pResBlock->pBlockAgg[pSup->slotIds[j]] = pAgg; + if (pAgg->colId == pSup->colInfo[j].colId) { + pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = pAgg; i += 1; j += 1; - } else if (pAgg->colId < pSup->colIds[j]) { + } else if (pAgg->colId < pSup->colInfo[j].colId) { i += 1; - } else if (pSup->colIds[j] < pAgg->colId) { - if (pSup->colIds[j] == PRIMARYKEY_TIMESTAMP_COL_ID) { - pResBlock->pBlockAgg[pSup->slotIds[j]] = &pSup->tsColAgg; + } else if (pSup->colInfo[j].colId < pAgg->colId) { + if (pSup->colInfo[j].colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = &pSup->tsColAgg; } else { // all date in this block are null - SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow}; + SColumnDataAgg nullColAgg = {.colId = pSup->colInfo[j].colId, .numOfNull = pBlock->nRow}; taosArrayPush(pSup->pColAgg, &nullColAgg); - pResBlock->pBlockAgg[pSup->slotIds[j]] = taosArrayGetLast(pSup->pColAgg); + pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = taosArrayGetLast(pSup->pColAgg); } j += 1; } } -// taosArrayClear(pSup->pColAgg); - -// size_t num = taosArrayGetSize(pSup->pColAgg); -// for(int32_t k = 0; k < num; ++k) { -// pSup->plist[k] = taosArrayGet(pSup->pColAgg, k); -// } - -#endif - + *pBlockSMA = pResBlock->pBlockAgg; pReader->cost.smaDataLoad += 1; -// *pBlockStatis = pSup->plist; tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr); return code; } -static SArray* doRetrieveDataBlock(STsdbReader* pReader) { +static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; if (pStatus->composedDataBlock) { - return pReader->pResBlock->pDataBlock; + return pReader->pResBlock; } SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); @@ -4166,10 +4151,10 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { } copyBlockDataToSDataBlock(pReader, pBlockScanInfo); - return pReader->pResBlock->pDataBlock; + return pReader->pResBlock; } -SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { +SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { return doRetrieveDataBlock(pReader->innerReader[0]); @@ -4196,7 +4181,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { // allocate buffer in order to load data blocks from file memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg)); - memset(pReader->suppInfo.plist, 0, POINTER_BYTES); pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; tsdbDataFReaderClose(&pReader->pFileReader); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index d675ac05d5..df11add919 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -957,21 +957,27 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s pCond->order = TSDB_ORDER_ASC; pCond->numOfCols = pMtInfo->schema->nCols; pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); - if (pCond->colList == NULL) { + pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t) * pCond->numOfCols); + if (pCond->colList == NULL || pCond->pSlotList == NULL) { + taosMemoryFreeClear(pCond->colList); + taosMemoryFreeClear(pCond->pSlotList); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return terrno; } - pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + pCond->twindows = TSWINDOW_INITIALIZER; pCond->suid = pMtInfo->suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = sContext->snapVersion; for (int32_t i = 0; i < pCond->numOfCols; ++i) { - pCond->colList[i].type = pMtInfo->schema->pSchema[i].type; - pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes; - pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId; + SColumnInfo* pColInfo = &pCond->colList[i]; + pColInfo->type = pMtInfo->schema->pSchema[i].type; + pColInfo->bytes = pMtInfo->schema->pSchema[i].bytes; + pColInfo->colId = pMtInfo->schema->pSchema[i].colId; + + pCond->pSlotList[i] = i; } return TSDB_CODE_SUCCESS; @@ -1116,7 +1122,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); ASSERT(size == 1); - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->pRes, &pInfo->dataReader, NULL); + tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b0104c6025..f09b7dae58 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -224,10 +224,8 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA } static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { - bool allColumnsHaveAgg = true; - SColumnDataAgg** pColAgg = NULL; - - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); + bool allColumnsHaveAgg = true; + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pBlock->pBlockAgg, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -387,11 +385,12 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); - if (pCols == NULL) { + SSDataBlock* p = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); + if (p == NULL) { return terrno; } + ASSERT(p == pBlock); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); // restore the previous value @@ -994,19 +993,10 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU return NULL; } - bool hasBlock = tsdbNextDataBlock(pReader); - if (hasBlock) { - SDataBlockInfo* pBInfo = &pBlock->info; - -// int32_t rows = 0; -// tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window); - - SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL); - -// relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true); + if (tsdbNextDataBlock(pReader)) { + /*SSDataBlock* p = */tsdbRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); - - pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->id.uid); + pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); } tsdbReaderClose(pReader); @@ -2030,20 +2020,13 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { qDebug("tmqsnap doRawScan called"); if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { - SSDataBlock* pBlock = &pInfo->pRes; - if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) { if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(pInfo->dataReader, &rows, &pBlock->info.id.uid, &pBlock->info.window); - pBlock->info.rows = rows; - - SArray* pCols = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); - pBlock->pDataBlock = pCols; - if (pCols == NULL) { + SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); + if (pBlock == NULL) { longjmp(pTaskInfo->env, terrno); } diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index 2b0336d2d7..6af9b130ef 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -429,10 +429,10 @@ class TDTestCase: tdSql.checkRows(2) # nest query - tdSql.query(f"select unique(c1) from (select _rowts , t1 ,c1 , tbname from {dbname}.stb1 ) ") + tdSql.query(f"select unique(c1) v from (select _rowts , t1 ,c1 , tbname from {dbname}.stb1 ) order by v") tdSql.checkRows(11) - tdSql.checkData(0,0,6) - tdSql.checkData(10,0,3) + tdSql.checkData(1,0,0) + tdSql.checkData(10,0,9) tdSql.query(f"select unique(t1) from (select _rowts , t1 , tbname from {dbname}.stb1 )") tdSql.checkRows(2) tdSql.checkData(0,0,4) From 76fd76cd1ad996255debbe496cf97c29fb173af0 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 16:22:06 +0800 Subject: [PATCH 39/95] enh: optimize db vgroup cache and child table meta retrieving --- include/libs/qcom/query.h | 1 + source/libs/catalog/inc/catalogInt.h | 3 +- source/libs/catalog/src/catalog.c | 2 +- source/libs/catalog/src/ctgCache.c | 51 +++++++-- source/libs/catalog/src/ctgUtil.c | 154 ++++----------------------- 5 files changed, 66 insertions(+), 145 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 6319281212..b8d09e2262 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -129,6 +129,7 @@ typedef struct SDBVgInfo { int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT int64_t stateTs; SHashObj* vgHash; // key:vgId, value:SVgroupInfo + SArray* vgArray; } SDBVgInfo; typedef struct SUseDbOutput { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index d73dcb3b30..085b2c1112 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -764,8 +764,6 @@ void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); void ctgFreeVgInfo(SDBVgInfo* vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); -int32_t ctgGetVgInfoFromHashValue2(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); - int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); void ctgResetTbMetaTask(SCtgTask* pTask); @@ -791,6 +789,7 @@ int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); int32_t ctgdGetOneHandle(SCatalog **pHandle); +int ctgVgInfoComp(const void* lp, const void* rp); extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f17abaa31f..3a398d1551 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -536,7 +536,7 @@ int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* return TSDB_CODE_SUCCESS; } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue2(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); _return: diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index db3a53aeb0..c319a8350f 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -226,6 +226,7 @@ _return: return TSDB_CODE_SUCCESS; } +/* int32_t ctgAcquireStbMetaFromCache(SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; SCtgTbCache *pCache = NULL; @@ -276,9 +277,9 @@ _return: return TSDB_CODE_SUCCESS; } +*/ - -int32_t ctgAcquireStbMetaFromCache2(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgTbCache **pTb) { +int32_t ctgAcquireStbMetaFromCache(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgTbCache **pTb) { SCtgTbCache *pCache = NULL; char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); if (NULL == stName) { @@ -437,7 +438,7 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); - ctgAcquireStbMetaFromCache2(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); + ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); STableMeta *stbMeta = tbCache->pMeta; if (stbMeta->suid != ctx->tbInfo.suid) { @@ -503,10 +504,15 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, // PROCESS FOR CHILD TABLE - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + if (tbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosHashRelease(dbCache->tbCache, tbCache); + } + ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); - ctgAcquireStbMetaFromCache(pCtg, dbFName, *suid, &dbCache, &tbCache); + ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid); @@ -846,6 +852,18 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId dbFName = p + 1; } + if (dbInfo->vgHash && NULL == dbInfo->vgArray) { + dbInfo->vgArray = taosArrayInit(100, sizeof(SVgroupInfo)); + + void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + taosArrayPush(dbInfo->vgArray, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); + } + + taosArraySort(dbInfo->vgArray, ctgVgInfoComp); + } + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->pCtg = pCtg; msg->dbId = dbId; @@ -1592,6 +1610,20 @@ void ctgFreeAllInstance(void) { taosHashClear(gCtgMgmt.pCluster); } +int32_t ctgVgInfoIdComp(void const* lp, void const* rp) { + int32_t* key = (int32_t*)lp; + SVgroupInfo* pVg = (SVgroupInfo*)rp; + + if (*key < pVg->vgId) { + return -1; + } else if (*key > pVg->vgId) { + return 1; + } + + return 0; +} + + int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; @@ -1973,7 +2005,13 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { SVgroupInfo *pInfo = taosHashGet(vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); if (NULL == pInfo) { - ctgDebug("no vgroup %d in db %s, ignore epset update", msg->vgId, msg->dbFName); + ctgDebug("no vgroup %d in db %s vgHash, ignore epset update", msg->vgId, msg->dbFName); + goto _return; + } + + SVgroupInfo *pInfo2 = taosArraySearch(vgInfo->vgArray, &msg->vgId, ctgVgInfoIdComp, TD_EQ); + if (NULL == pInfo2) { + ctgDebug("no vgroup %d in db %s vgArray, ignore epset update", msg->vgId, msg->dbFName); goto _return; } @@ -1984,6 +2022,7 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { msg->epSet.numOfEps, pNewEp->fqdn, pNewEp->port, msg->dbFName); pInfo->epSet = msg->epSet; + pInfo2->epSet = msg->epSet; _return: diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index ef7b80a676..6fba94cc8a 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -236,10 +236,8 @@ void ctgFreeVgInfo(SDBVgInfo* vgInfo) { return; } - if (vgInfo->vgHash) { - taosHashCleanup(vgInfo->vgHash); - vgInfo->vgHash = NULL; - } + taosHashCleanup(vgInfo->vgHash); + taosArrayDestroy(vgInfo->vgArray); taosMemoryFreeClear(vgInfo); } @@ -837,7 +835,7 @@ _return: CTG_RET(code); } -int ctgVgInfoComp2(const void* lp, const void* rp) { +int ctgVgInfoComp(const void* lp, const void* rp) { SVgroupInfo* pLeft = (SVgroupInfo*)lp; SVgroupInfo* pRight = (SVgroupInfo*)rp; if (pLeft->hashBegin < pRight->hashBegin) { @@ -850,32 +848,6 @@ int ctgVgInfoComp2(const void* lp, const void* rp) { } int32_t ctgHashValueComp(void const* lp, void const* rp) { - uint32_t* key = (uint32_t*)lp; - SVgroupInfo* pVg = *(SVgroupInfo**)rp; - - if (*key < pVg->hashBegin) { - return -1; - } else if (*key > pVg->hashEnd) { - return 1; - } - - return 0; -} - -int ctgVgInfoComp(const void* lp, const void* rp) { - SVgroupInfo* pLeft = *(SVgroupInfo**)lp; - SVgroupInfo* pRight = *(SVgroupInfo**)rp; - if (pLeft->hashBegin < pRight->hashBegin) { - return -1; - } else if (pLeft->hashBegin > pRight->hashBegin) { - return 1; - } - - return 0; -} - - -int32_t ctgHashValueComp2(void const* lp, void const* rp) { uint32_t* key = (uint32_t*)lp; SVgroupInfo* pVg = (SVgroupInfo*)rp; @@ -891,7 +863,7 @@ int32_t ctgHashValueComp2(void const* lp, void const* rp) { int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -907,6 +879,9 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); + vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); + +/* void* pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; @@ -918,84 +893,11 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName pIter = taosHashIterate(dbInfo->vgHash, pIter); vgInfo = NULL; } +*/ if (NULL == vgInfo) { ctgError("0no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, - taosHashGetSize(dbInfo->vgHash)); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - *pVgroup = *vgInfo; - - ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, - vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, - vgInfo->epSet.eps[vgInfo->epSet.inUse].port); - - CTG_RET(code); -} - - -int32_t ctgGetVgInfoFromHashValue2(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { - int32_t code = 0; - - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); - char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); - - if (vgNum <= 0) { - ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", db, vgNum); - CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); - } - - SVgroupInfo* vgInfo = NULL; - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); - - uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, - dbInfo->hashPrefix, dbInfo->hashSuffix); - - static SArray* pVgList = NULL; - static bool created = false; - - if (!created) { - ctgDebug("create vg array, %d", taosHashGetSize(dbInfo->vgHash)); - pVgList = taosArrayInit(100, sizeof(SVgroupInfo)); - - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - taosArrayPush(pVgList, pIter); - pIter = taosHashIterate(dbInfo->vgHash, pIter); - } - - taosArraySort(pVgList, ctgVgInfoComp2); - created = true; - } - - vgInfo = taosArraySearch(pVgList, &hashValue, ctgHashValueComp2, TD_EQ); - - if (NULL == vgInfo) { - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - vgInfo = pIter; - if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { - taosHashCancelIterate(dbInfo->vgHash, pIter); - break; - } - - pIter = taosHashIterate(dbInfo->vgHash, pIter); - vgInfo = NULL; - } - - if (vgInfo) { - taosArrayDestroy(pVgList); - created = false; - ctgDebug("need to re-create vg array, %d", taosHashGetSize(dbInfo->vgHash)); - } - } - - if (NULL == vgInfo) { - ctgError("1no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, - taosHashGetSize(dbInfo->vgHash)); + taosArrayGetSize(dbInfo->vgArray)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1013,7 +915,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* int32_t code = 0; SCtgTask* pTask = tReq->pTask; SMetaRes res = {0}; - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); if (vgNum <= 0) { ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -1023,20 +925,13 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* int32_t tbNum = taosArrayGetSize(pNames); if (1 == vgNum) { - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - if (NULL == pIter) { - ctgError("empty vgHash, db:%s, vgroup number:%d", dbFName, vgNum); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - for (int32_t i = 0; i < tbNum; ++i) { vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == vgInfo) { - taosHashCancelIterate(dbInfo->vgHash, pIter); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - *vgInfo = *(SVgroupInfo*)pIter; + *vgInfo = *(SVgroupInfo*)taosArrayGet(dbInfo->vgArray, 0); ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); @@ -1051,19 +946,9 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* } } - taosHashCancelIterate(dbInfo->vgHash, pIter); return TSDB_CODE_SUCCESS; } - SArray* pVgList = taosArrayInit(vgNum, POINTER_BYTES); - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - taosArrayPush(pVgList, &pIter); - pIter = taosHashIterate(dbInfo->vgHash, pIter); - } - - taosArraySort(pVgList, ctgVgInfoComp); - char tbFullName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFullName, "%s.", dbFName); int32_t offset = strlen(tbFullName); @@ -1079,20 +964,15 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); - SVgroupInfo** p = taosArraySearch(pVgList, &hashValue, ctgHashValueComp, TD_EQ); - - if (NULL == p) { + vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); + if (NULL == vgInfo) { ctgError("2no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, - taosHashGetSize(dbInfo->vgHash)); - taosArrayDestroy(pVgList); + taosArrayGetSize(dbInfo->vgArray)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - vgInfo = *p; - SVgroupInfo* pNewVg = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == pNewVg) { - taosArrayDestroy(pVgList); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -1112,8 +992,6 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* } } - taosArrayDestroy(pVgList); - CTG_RET(code); } @@ -1190,6 +1068,10 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { pIter = taosHashIterate(src->vgHash, pIter); } + if (src->vgArray) { + (*dst)->vgArray = taosArrayDup(src->vgArray, NULL); + } + return TSDB_CODE_SUCCESS; } From 7b18f0a8422b678e8f412ae9bebcc3a51f28e50d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 16:36:56 +0800 Subject: [PATCH 40/95] fix: fix compile issue --- source/libs/catalog/src/ctgUtil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 6fba94cc8a..120e261c32 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -896,8 +896,8 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName */ if (NULL == vgInfo) { - ctgError("0no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, - taosArrayGetSize(dbInfo->vgArray)); + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, + (int32_t)taosArrayGetSize(dbInfo->vgArray)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } From 3c1e52eb91783c2717e54af453b399cbf2c50c5c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 16:40:09 +0800 Subject: [PATCH 41/95] fix: fix size_t compile issue --- source/libs/catalog/src/ctgUtil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 120e261c32..17ee8bb68e 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -966,8 +966,8 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); if (NULL == vgInfo) { - ctgError("2no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, - taosArrayGetSize(dbInfo->vgArray)); + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, + (int32_t)taosArrayGetSize(dbInfo->vgArray)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } From 39482b24db8660c685f6caf5a848401f647f90f7 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 16:42:38 +0800 Subject: [PATCH 42/95] feat(shell): supported affected rows int64_t --- include/client/taos.h | 1 + source/client/inc/clientInt.h | 4 ++-- source/client/src/clientMain.c | 12 ++++++++++++ source/libs/executor/src/dataDispatcher.c | 2 +- tools/shell/src/shellEngine.c | 16 ++++++++++++++-- 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 2940f1dfd0..69774b750f 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -185,6 +185,7 @@ DLL_EXPORT void taos_kill_query(TAOS *taos); DLL_EXPORT int taos_field_count(TAOS_RES *res); DLL_EXPORT int taos_num_fields(TAOS_RES *res); DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index aae20c587d..6c2a7d75a0 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -171,9 +171,9 @@ typedef struct SReqResultInfo { char** convertBuf; TAOS_ROW row; SResultColumn* pCol; - uint32_t numOfRows; + uint64_t numOfRows; uint64_t totalRows; - uint32_t current; + uint64_t current; bool localResultFetched; bool completed; int32_t precision; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 976d1dd1b0..fecb94d02c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -438,11 +438,23 @@ const char *taos_data_type(int type) { const char *taos_get_client_info() { return version; } +// return int32_t int taos_affected_rows(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { return 0; } + SRequestObj *pRequest = (SRequestObj *)res; + SReqResultInfo *pResInfo = &pRequest->body.resInfo; + return (int)pResInfo->numOfRows; +} + +// return int64_t +int64_t taos_affected_rows(TAOS_RES *res) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { + return 0; + } + SRequestObj *pRequest = (SRequestObj *)res; SReqResultInfo *pResInfo = &pRequest->body.resInfo; return pResInfo->numOfRows; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index d4248fc420..78f6155cf7 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -189,7 +189,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { } SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); - pOutput->numOfRows = pEntry->numOfRows; + pOutput->fr = pEntry->numOfRows; pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 118a6caf7a..362a9b3c80 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -214,6 +214,18 @@ void shellRunSingleCommandImp(char *command) { return; } + // pre string + char * pre = "Query OK"; + if (shellRegexMatch(command, "^\\s*delete\\s*from\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Deleted OK"; + } else if(shellRegexMatch(command, "^\\s*insert\\s*into\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Inserted OK"; + } else if(shellRegexMatch(command, "^\\s*create\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Created OK"; + } else if(shellRegexMatch(command, "^\\s*drop\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Droped OK"; + } + TAOS_FIELD *pFields = taos_fetch_fields(pSql); if (pFields != NULL) { // select and show kinds of commands int32_t error_no = 0; @@ -229,10 +241,10 @@ void shellRunSingleCommandImp(char *command) { } taos_free_result(pSql); } else { - int32_t num_rows_affacted = taos_affected_rows(pSql); + int64_t num_rows_affacted = taos_affected_rows64(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d row(s) affected (%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); + printf("%s, %" PRId64 " row(s) affected (%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); // call auto tab callbackAutoTab(command, NULL, false); From 878f2a2c19c8d9f6859546651c1e66d18a245781 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 17:01:40 +0800 Subject: [PATCH 43/95] fix(shell): reset dataDispatcher mistake modify --- source/libs/executor/src/dataDispatcher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 78f6155cf7..d4248fc420 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -189,7 +189,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { } SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); - pOutput->fr = pEntry->numOfRows; + pOutput->numOfRows = pEntry->numOfRows; pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; From 2aa65c4b9c6d7ed7d1d09cf41640cfc9b59f3d67 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 17:09:04 +0800 Subject: [PATCH 44/95] feat(shell): affected rows int64 --- source/client/src/clientImpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 846accf51b..77699e89c2 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -317,7 +317,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); } @@ -1527,7 +1527,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) return NULL; } - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId); STscObj* pTscObj = pRequest->pTscObj; From 0f0e575481febaae34d084b42378342942dff712 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 17:14:49 +0800 Subject: [PATCH 45/95] feat(shell): affected rows int64 --- source/client/src/clientMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index fecb94d02c..8020134c52 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -450,7 +450,7 @@ int taos_affected_rows(TAOS_RES *res) { } // return int64_t -int64_t taos_affected_rows(TAOS_RES *res) { +int64_t taos_affected_rows64(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { return 0; } @@ -971,7 +971,7 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); From 83a4f76ecea33eeaaa04db2d03dbb08455aeb917 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 17:15:51 +0800 Subject: [PATCH 46/95] feat(shell): affected rows int64 --- tools/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 362a9b3c80..14757247b7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -244,7 +244,7 @@ void shellRunSingleCommandImp(char *command) { int64_t num_rows_affacted = taos_affected_rows64(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("%s, %" PRId64 " row(s) affected (%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); + printf("%s, %" PRId64 " row(s) affected (%.6fs)\r\n", pre, num_rows_affacted, (et - st) / 1E6); // call auto tab callbackAutoTab(command, NULL, false); From 867f7abeea3099dcf3ea61fb64a8dabb9c53f049 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 17:44:39 +0800 Subject: [PATCH 47/95] test: add asan case --- tests/parallel_test/cases.task | 44 +++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e514b069a8..13bfe4c981 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -623,12 +623,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -,,,system-test,python3 ./test.py -f 2-query/stablity.py -,,,system-test,python3 ./test.py -f 2-query/stablity_1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py @@ -788,12 +788,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 2 @@ -881,12 +881,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 3 @@ -974,10 +974,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 4 From 27ed740eb56317a5bcb5dc4328fecca8179a51b4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Dec 2022 17:52:49 +0800 Subject: [PATCH 48/95] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 89 +++++++++++++------------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 859ae026a2..ac003783ae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -81,15 +81,11 @@ typedef struct SIOCostSummary { double createScanInfoList; } SIOCostSummary; -typedef struct SColInfo { - int16_t colId; - int16_t slotId; -} SColInfo; - typedef struct SBlockLoadSuppInfo { SArray* pColAgg; SColumnDataAgg tsColAgg; - SColInfo* colInfo; // column ids for loading file block data + int16_t* colId; + int16_t* slotId; int32_t numOfCols; char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. bool smaValid; // the sma on all queried columns are activated @@ -222,16 +218,17 @@ static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWi static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) { pSupInfo->smaValid = true; pSupInfo->numOfCols = numOfCols; - pSupInfo->colInfo = taosMemoryMalloc(numOfCols * (sizeof(SColInfo) + POINTER_BYTES)); - if (pSupInfo->colInfo == NULL) { - taosMemoryFree(pSupInfo->colInfo); + pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t)*2 + POINTER_BYTES)); + if (pSupInfo->colId == NULL) { + taosMemoryFree(pSupInfo->colId); return TSDB_CODE_OUT_OF_MEMORY; } - pSupInfo->buildBuf = (char**)((char*)pSupInfo->colInfo + (sizeof(SColInfo) * numOfCols)); + pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols)); + pSupInfo->buildBuf = (char**) ((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); for (int32_t i = 0; i < numOfCols; ++i) { - pSupInfo->colInfo[i].colId = pCols[i].colId; - pSupInfo->colInfo[i].slotId = pSlotIdList[i]; + pSupInfo->colId[i] = pCols[i].colId; + pSupInfo->slotId[i] = pSlotIdList[i]; if (IS_VAR_DATA_TYPE(pCols[i].type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); @@ -248,7 +245,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) { STColumn* pTCol = &pSchema->columns[i]; - if (pTCol->colId == pSupInfo->colInfo[j].colId) { + if (pTCol->colId == pSupInfo->colId[j]) { if (!IS_BSMA_ON(pTCol)) { pSupInfo->smaValid = false; return; @@ -256,7 +253,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) i += 1; j += 1; - } else if (pTCol->colId < pSupInfo->colInfo[j].colId) { + } else if (pTCol->colId < pSupInfo->colId[j]) { // do nothing i += 1; } else { @@ -458,7 +455,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb if (pLReader->pInfo == NULL) { // here we ignore the first column, which is always be the primary timestamp column pLReader->pInfo = - tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1); + tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1); if (pLReader->pInfo == NULL) { tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr); return terrno; @@ -1094,8 +1091,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t i = 0; int32_t rowIndex = 0; - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); - if (pSupInfo->colInfo[i].colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); i += 1; } @@ -1106,10 +1103,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn rowIndex = 0; SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex); - if (pData->cid < pSupInfo->colInfo[i].colId) { + if (pData->cid < pSupInfo->colId[i]) { colIndex += 1; - } else if (pData->cid == pSupInfo->colInfo[i].colId) { - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + } else if (pData->cid == pSupInfo->colId[i]) { + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { colDataAppendNNULL(pColData, 0, dumpedRows); @@ -1127,7 +1124,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colIndex += 1; i += 1; } else { // the specified column does not exist in file block, fill with null data - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1135,7 +1132,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // fill the mis-matched columns with null value while (i < numOfOutputCols) { - pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1173,7 +1170,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI tBlockDataReset(pBlockData); TABLEID tid = {.suid = pReader->suid, .uid = uid}; int32_t code = - tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1); + tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1632,7 +1629,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* int64_t st = taosGetTimestampUs(); int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader); - blockDataUpdateTsWindow(pBlock, pReader->suppInfo.colInfo[0].slotId); + blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotId[0]); pBlock->info.id.uid = pBlockScanInfo->uid; setComposedBlockFlag(pReader, true); @@ -2504,7 +2501,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { _end: pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; - blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.colInfo[0].slotId); + blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); setComposedBlockFlag(pReader, true); double el = (taosGetTimestampUs() - st) / 1000.0; @@ -3553,24 +3550,24 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* SColVal colVal = {0}; int32_t i = 0, j = 0; - if (pSupInfo->colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + if (pSupInfo->colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; } while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) { - col_id_t colId = pSupInfo->colInfo[i].colId; + col_id_t colId = pSupInfo->colId[i]; if (colId == pSchema->columns[j].colId) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); tTSRowGetVal(pTSRow, pSchema, j, &colVal); doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); i += 1; j += 1; } else if (colId < pSchema->columns[j].colId) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; @@ -3581,7 +3578,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* // set null value since current column does not exist in the "pSchema" while (i < pSupInfo->numOfCols) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; } @@ -3597,8 +3594,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t outputRowIndex = pResBlock->info.rows; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - if (pReader->suppInfo.colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) { - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + if (pReader->suppInfo.colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; } @@ -3609,13 +3606,13 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S while (i < numOfOutputCols && j < numOfInputCols) { SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); - if (pData->cid < pSupInfo->colInfo[i].colId) { + if (pData->cid < pSupInfo->colId[i]) { j += 1; continue; } - SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); - if (pData->cid == pSupInfo->colInfo[i].colId) { + SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pData->cid == pSupInfo->colId[i]) { tColDataGetValue(pData, rowIndex, &cv); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); j += 1; @@ -3628,7 +3625,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S } while (i < numOfOutputCols) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId); + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNULL(pCol, outputRowIndex); i += 1; } @@ -3897,7 +3894,7 @@ void tsdbReaderClose(STsdbReader* pReader) { pReader->pResBlock = blockDataDestroy(pReader->pResBlock); } - taosMemoryFree(pSupInfo->colInfo); + taosMemoryFree(pSupInfo->colId); tBlockDataDestroy(&pReader->status.fileBlockData, true); cleanupDataBlockIterator(&pReader->status.blockIter); @@ -4099,21 +4096,21 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg ***pBlockS while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); - if (pAgg->colId == pSup->colInfo[j].colId) { - pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = pAgg; + if (pAgg->colId == pSup->colId[j]) { + pResBlock->pBlockAgg[pSup->slotId[j]] = pAgg; i += 1; j += 1; - } else if (pAgg->colId < pSup->colInfo[j].colId) { + } else if (pAgg->colId < pSup->colId[j]) { i += 1; - } else if (pSup->colInfo[j].colId < pAgg->colId) { - if (pSup->colInfo[j].colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = &pSup->tsColAgg; + } else if (pSup->colId[j] < pAgg->colId) { + if (pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID) { + pResBlock->pBlockAgg[pSup->slotId[j]] = &pSup->tsColAgg; } else { // all date in this block are null - SColumnDataAgg nullColAgg = {.colId = pSup->colInfo[j].colId, .numOfNull = pBlock->nRow}; + SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = pBlock->nRow}; taosArrayPush(pSup->pColAgg, &nullColAgg); - pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = taosArrayGetLast(pSup->pColAgg); + pResBlock->pBlockAgg[pSup->slotId[j]] = taosArrayGetLast(pSup->pColAgg); } j += 1; } From 55fb892e70bd6631196f1078aea31b27f25f7abc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 2 Dec 2022 15:50:53 +0800 Subject: [PATCH 49/95] fix(tsdb/cache): use lru erase to invalidate cache entries --- source/dnode/vnode/src/tsdb/tsdbCache.c | 125 +++++++----------------- 1 file changed, 35 insertions(+), 90 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index d71eb33951..7309ea3407 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -228,23 +228,23 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST invalidate = true; break; - } - } else { - SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; - if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); - taosMemoryFree(pLastCol->colVal.value.pData); + } else { // new inserting key is greater than cached, update cached entry + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); - lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); } - memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); - } - taosArraySet(pLast, iCol, &lastCol); + taosArraySet(pLast, iCol, &lastCol); + } } } } @@ -253,65 +253,10 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST taosMemoryFreeClear(pTSchema); taosLRUCacheRelease(pCache, h, invalidate); - /* - cacheRow = (STSRow *)taosLRUCacheValue(pCache, h); - if (row->ts >= cacheRow->ts) { - if (row->ts == cacheRow->ts) { - STSRow *mergedRow = NULL; - SRowMerger merger = {0}; - STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - - tRowMergerInit(&merger, &tsdbRowFromTSRow(0, cacheRow), pTSchema); - - tRowMerge(&merger, &tsdbRowFromTSRow(1, row)); - - tRowMergerGetRow(&merger, &mergedRow); - tRowMergerClear(&merger); - - taosMemoryFreeClear(pTSchema); - - row = mergedRow; - dup = false; - } - - if (TD_ROW_LEN(row) <= TD_ROW_LEN(cacheRow)) { - tdRowCpy(cacheRow, row); - if (!dup) { - taosMemoryFree(row); - } - - taosLRUCacheRelease(pCache, h, false); - } else { - taosLRUCacheRelease(pCache, h, true); - // tsdbCacheDeleteLastrow(pCache, uid, TSKEY_MAX); - if (dup) { - cacheRow = tdRowDup(row); - } else { - cacheRow = row; - } - _taos_lru_deleter_t deleter = deleteTableCacheLastrow; - LRUStatus status = taosLRUCacheInsert(pCache, key, keyLen, cacheRow, TD_ROW_LEN(cacheRow), deleter, NULL, - TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - // tsdbCacheInsertLastrow(pCache, uid, row, dup); - } - }*/ - } /*else { - if (dup) { - cacheRow = tdRowDup(row); - } else { - cacheRow = row; + if (invalidate) { + taosLRUCacheErase(pCache, key, keyLen); } - - _taos_lru_deleter_t deleter = deleteTableCacheLastrow; - LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, cacheRow, TD_ROW_LEN(cacheRow), deleter, NULL, TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - }*/ + } return code; } @@ -349,28 +294,28 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb SColVal colVal = {0}; tTSRowGetVal(row, pTSchema, iCol, &colVal); - if (!COL_VAL_IS_VALUE(&colVal)) { + if (COL_VAL_IS_VALUE(&colVal)) { if (keyTs == tTsVal1->ts && COL_VAL_IS_VALUE(tColVal)) { invalidate = true; break; - } - } else { - SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; - if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); - taosMemoryFree(pLastCol->colVal.value.pData); + } else { + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); - lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); } - memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); - } - taosArraySet(pLast, iCol, &lastCol); + taosArraySet(pLast, iCol, &lastCol); + } } } } @@ -379,9 +324,9 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb taosMemoryFreeClear(pTSchema); taosLRUCacheRelease(pCache, h, invalidate); - - // clear last cache anyway, lazy load when get last lookup - // taosLRUCacheRelease(pCache, h, true); + if (invalidate) { + taosLRUCacheErase(pCache, key, keyLen); + } } return code; From b08ee99c4c746f1c2c991fc54819bf4d017e1f7e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 18:09:38 +0800 Subject: [PATCH 50/95] fix: vgroup array not made in some cases --- include/libs/qcom/query.h | 1 + source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/catalog.c | 14 ++++----- source/libs/catalog/src/ctgCache.c | 28 ++++++++--------- source/libs/catalog/src/ctgUtil.c | 45 ++++++++++++++++++---------- source/libs/qcom/src/queryUtil.c | 16 ++++++++-- 6 files changed, 62 insertions(+), 44 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index b8d09e2262..268649524f 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -239,6 +239,7 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t char* parseTagDatatoJson(void* p); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); +void freeVgInfo(SDBVgInfo* vgInfo); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, void* (*mallocFp)(int64_t)); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 085b2c1112..ba99906589 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -762,7 +762,6 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) int32_t ctgGenerateVgList(SCatalog* pCtg, SHashObj* vgHash, SArray** pList); void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); -void ctgFreeVgInfo(SDBVgInfo* vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); @@ -790,6 +789,7 @@ int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); int32_t ctgdGetOneHandle(SCatalog **pHandle); int ctgVgInfoComp(const void* lp, const void* rp); +int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3a398d1551..2b1bdeadf8 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -505,8 +505,7 @@ _return: taosMemoryFreeClear(tbMeta); if (vgInfo) { - taosHashCleanup(vgInfo->vgHash); - taosMemoryFreeClear(vgInfo); + freeVgInfo(vgInfo); } if (vgList) { @@ -546,8 +545,7 @@ _return: } if (vgInfo) { - taosHashCleanup(vgInfo->vgHash); - taosMemoryFreeClear(vgInfo); + freeVgInfo(vgInfo); } CTG_RET(code); @@ -778,8 +776,7 @@ _return: } if (vgInfo) { - taosHashCleanup(vgInfo->vgHash); - taosMemoryFreeClear(vgInfo); + freeVgInfo(vgInfo); } CTG_API_LEAVE(code); @@ -836,8 +833,7 @@ _return: ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } else if (dbInfo) { - taosHashCleanup(dbInfo->vgHash); - taosMemoryFreeClear(dbInfo); + freeVgInfo(dbInfo); } CTG_API_LEAVE(code); @@ -849,7 +845,7 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId int32_t code = 0; if (NULL == pCtg || NULL == dbFName || NULL == dbInfo) { - ctgFreeVgInfo(dbInfo); + freeVgInfo(dbInfo); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index c319a8350f..4626d4527d 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -843,7 +843,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); taosMemoryFree(op); - ctgFreeVgInfo(dbInfo); + freeVgInfo(dbInfo); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -852,16 +852,12 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId dbFName = p + 1; } - if (dbInfo->vgHash && NULL == dbInfo->vgArray) { - dbInfo->vgArray = taosArrayInit(100, sizeof(SVgroupInfo)); - - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - taosArrayPush(dbInfo->vgArray, pIter); - pIter = taosHashIterate(dbInfo->vgHash, pIter); - } - - taosArraySort(dbInfo->vgArray, ctgVgInfoComp); + code = ctgMakeVgArray(dbInfo); + if (code) { + taosMemoryFree(op); + taosMemoryFree(msg); + freeVgInfo(dbInfo); + CTG_ERR_RET(code); } tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); @@ -877,7 +873,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId _return: - ctgFreeVgInfo(dbInfo); + freeVgInfo(dbInfo); CTG_RET(code); } @@ -1675,7 +1671,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { goto _return; } - ctgFreeVgInfo(vgInfo); + freeVgInfo(vgInfo); } vgCache->vgInfo = dbInfo; @@ -1696,7 +1692,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { _return: - ctgFreeVgInfo(msg->dbInfo); + freeVgInfo(msg->dbInfo); taosMemoryFreeClear(msg); CTG_RET(code); @@ -1749,7 +1745,7 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); - ctgFreeVgInfo(dbCache->vgCache.vgInfo); + freeVgInfo(dbCache->vgCache.vgInfo); dbCache->vgCache.vgInfo = NULL; ctgDebug("db vgInfo removed, dbFName:%s", msg->dbFName); @@ -2139,7 +2135,7 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) { switch (op->opId) { case CTG_OP_UPDATE_VGROUP: { SCtgUpdateVgMsg *msg = op->data; - ctgFreeVgInfo(msg->dbInfo); + freeVgInfo(msg->dbInfo); taosMemoryFreeClear(op->data); break; } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 17ee8bb68e..67db0c79d6 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -231,18 +231,7 @@ void ctgFreeTbCache(SCtgDBCache* dbCache) { CTG_CACHE_STAT_DEC(numOfTbl, tblNum); } -void ctgFreeVgInfo(SDBVgInfo* vgInfo) { - if (NULL == vgInfo) { - return; - } - - taosHashCleanup(vgInfo->vgHash); - taosArrayDestroy(vgInfo->vgArray); - - taosMemoryFreeClear(vgInfo); -} - -void ctgFreeVgInfoCache(SCtgDBCache* dbCache) { ctgFreeVgInfo(dbCache->vgCache.vgInfo); } +void ctgFreeVgInfoCache(SCtgDBCache* dbCache) { freeVgInfo(dbCache->vgCache.vgInfo); } void ctgFreeDbCache(SCtgDBCache* dbCache) { if (NULL == dbCache) { @@ -364,8 +353,7 @@ void ctgFreeSUseDbOutput(SUseDbOutput* pOutput) { } if (pOutput->dbVgroup) { - taosHashCleanup(pOutput->dbVgroup->vgHash); - taosMemoryFreeClear(pOutput->dbVgroup); + freeVgInfo(pOutput->dbVgroup); } taosMemoryFree(pOutput); @@ -571,8 +559,7 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void** pRes) { case CTG_TASK_GET_DB_VGROUP: { if (*pRes) { SDBVgInfo* pInfo = (SDBVgInfo*)*pRes; - taosHashCleanup(pInfo->vgHash); - taosMemoryFreeClear(*pRes); + freeVgInfo(pInfo); } break; } @@ -1035,7 +1022,33 @@ int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) { } } +int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { + if (NULL == dbInfo) { + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgHash && NULL == dbInfo->vgArray) { + dbInfo->vgArray = taosArrayInit(100, sizeof(SVgroupInfo)); + if (NULL == dbInfo->vgArray) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + taosArrayPush(dbInfo->vgArray, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); + } + + taosArraySort(dbInfo->vgArray, ctgVgInfoComp); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { + CTG_ERR_RET(ctgMakeVgArray(src)); + *dst = taosMemoryMalloc(sizeof(SDBVgInfo)); if (NULL == *dst) { qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 6eadf80e3d..96e02c3643 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -447,6 +447,19 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } + +void freeVgInfo(SDBVgInfo* vgInfo) { + if (NULL == vgInfo) { + return; + } + + taosHashCleanup(vgInfo->vgHash); + taosArrayDestroy(vgInfo->vgArray); + + taosMemoryFreeClear(vgInfo); +} + + int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { if (NULL == pSrc) { *pDst = NULL; @@ -475,8 +488,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { if (0 != taosHashPut((*pDst)->vgHash, vgId, sizeof(*vgId), vgInfo, sizeof(*vgInfo))) { qError("taosHashPut failed, vgId:%d", vgInfo->vgId); taosHashCancelIterate(pSrc->vgHash, pIter); - taosHashCleanup((*pDst)->vgHash); - taosMemoryFreeClear(*pDst); + freeVgInfo(*pDst); return TSDB_CODE_OUT_OF_MEMORY; } From 693a602c62bc888a2d436404df49fc2172952cc9 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 18:27:18 +0800 Subject: [PATCH 51/95] feat(api): modify prompt text --- tools/shell/src/shellEngine.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 14757247b7..5be46fe54a 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -217,13 +217,13 @@ void shellRunSingleCommandImp(char *command) { // pre string char * pre = "Query OK"; if (shellRegexMatch(command, "^\\s*delete\\s*from\\s*.*", REG_EXTENDED | REG_ICASE)) { - pre = "Deleted OK"; + pre = "Delete OK"; } else if(shellRegexMatch(command, "^\\s*insert\\s*into\\s*.*", REG_EXTENDED | REG_ICASE)) { - pre = "Inserted OK"; + pre = "Insert OK"; } else if(shellRegexMatch(command, "^\\s*create\\s*.*", REG_EXTENDED | REG_ICASE)) { - pre = "Created OK"; + pre = "Create OK"; } else if(shellRegexMatch(command, "^\\s*drop\\s*.*", REG_EXTENDED | REG_ICASE)) { - pre = "Droped OK"; + pre = "Drop OK"; } TAOS_FIELD *pFields = taos_fetch_fields(pSql); From 61cd7539475df28316708933cc46417a381699ab Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 18:59:03 +0800 Subject: [PATCH 52/95] enh: remove sync batch propose in vnode --- source/dnode/vnode/src/vnd/vnodeSync.c | 74 ++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index d01fdabac8..ee95e5b7d1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "vnd.h" -#define BATCH_DISABLE 1 +#define BATCH_ENABLE 0 static inline bool vnodeIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || @@ -87,7 +87,7 @@ static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { } static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) { - if (code == TSDB_CODE_SYN_NOT_LEADER) { + if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) { vnodeRedirectRpcMsg(pVnode, pMsg); } else { const STraceId *trace = &pMsg->info.traceId; @@ -99,15 +99,12 @@ static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) } } +#if BATCH_ENABLE + static void inline vnodeProposeBatchMsg(SVnode *pVnode, SRpcMsg **pMsgArr, bool *pIsWeakArr, int32_t *arrSize) { if (*arrSize <= 0) return; -#if BATCH_DISABLE - int32_t code = syncPropose(pVnode->sync, pMsgArr[0], pIsWeakArr[0]); -#else int32_t code = syncProposeBatch(pVnode->sync, pMsgArr, pIsWeakArr, *arrSize); -#endif - if (code > 0) { for (int32_t i = 0; i < *arrSize; ++i) { vnodeHandleWriteMsg(pVnode, pMsgArr[i]); @@ -177,7 +174,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) continue; } - if (isBlock || BATCH_DISABLE) { + if (isBlock) { vnodeProposeBatchMsg(pVnode, pMsgArr, pIsWeakArr, &arrayPos); } @@ -185,7 +182,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) pIsWeakArr[arrayPos] = isWeak; arrayPos++; - if (isBlock || msg == numOfMsgs - 1 || BATCH_DISABLE) { + if (isBlock || msg == numOfMsgs - 1) { vnodeProposeBatchMsg(pVnode, pMsgArr, pIsWeakArr, &arrayPos); } } @@ -194,6 +191,65 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) taosMemoryFree(pIsWeakArr); } +#else + +static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) { + int32_t code = syncPropose(pVnode->sync, pMsg, isWeak); + if (code > 0) { + vnodeHandleWriteMsg(pVnode, pMsg); + } else if (code == 0) { + vnodeWaitBlockMsg(pVnode, pMsg); + } else { + if (terrno != 0) code = terrno; + vnodeHandleProposeError(pVnode, pMsg, code); + } + + return code; +} + +void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { + SVnode *pVnode = pInfo->ahandle; + int32_t vgId = pVnode->config.vgId; + int32_t code = 0; + SRpcMsg *pMsg = NULL; + vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs); + + for (int32_t msg = 0; msg < numOfMsgs; msg++) { + if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; + bool isWeak = vnodeIsMsgWeak(pMsg->msgType); + + const STraceId *trace = &pMsg->info.traceId; + vGTrace("vgId:%d, msg:%p get from vnode-write queue, weak:%d block:%d msg:%d:%d, handle:%p", vgId, pMsg, isWeak, + vnodeIsMsgBlock(pMsg->msgType), msg, numOfMsgs, pMsg->info.handle); + + if (!pVnode->restored) { + vGError("vgId:%d, msg:%p failed to process since restore not finished", vgId, pMsg); + vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + continue; + } + + code = vnodePreProcessWriteMsg(pVnode, pMsg); + if (code != 0) { + vGError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, terrstr()); + if (terrno != 0) code = terrno; + vnodeHandleProposeError(pVnode, pMsg, code); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + continue; + } + + code = vnodeProposeMsg(pVnode, pMsg, isWeak); + + vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + } +} + +#endif + void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnode *pVnode = pInfo->ahandle; int32_t vgId = pVnode->config.vgId; From cafcec2571a409b6f8cd2adb0d74a2f86a3946be Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 2 Dec 2022 20:09:44 +0800 Subject: [PATCH 53/95] fix: make vgroup array when needed --- source/libs/catalog/src/ctgUtil.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 67db0c79d6..802ecde63e 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -849,6 +849,7 @@ int32_t ctgHashValueComp(void const* lp, void const* rp) { int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { int32_t code = 0; + CTG_ERR_RET(ctgMakeVgArray(dbInfo)); int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); char db[TSDB_DB_FNAME_LEN] = {0}; @@ -902,6 +903,9 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* int32_t code = 0; SCtgTask* pTask = tReq->pTask; SMetaRes res = {0}; + + CTG_ERR_RET(ctgMakeVgArray(dbInfo)); + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); if (vgNum <= 0) { ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); From 9fb4fa0a7f8443848b0171158d994fe164e10fa8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 20:48:39 +0800 Subject: [PATCH 54/95] feat(api): add int64 affected rows modify tmsg.h struct --- include/common/tmsg.h | 4 ++-- source/client/inc/clientInt.h | 2 +- source/client/src/clientImpl.c | 2 +- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/exchangeoperator.c | 10 +++++----- source/libs/scheduler/inc/schInt.h | 2 +- source/libs/scheduler/src/schJob.c | 4 ++-- source/libs/scheduler/src/schRemote.c | 8 ++++---- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0a082a37e4..9ad2e4375b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1403,8 +1403,8 @@ typedef struct { int8_t streamBlockType; int32_t compLen; int32_t numOfBlocks; - int32_t numOfRows; - int32_t numOfCols; + int64_t numOfRows; // from int32_t change to int64_t + int64_t numOfCols; int64_t skey; int64_t ekey; int64_t version; // for stream diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 6c2a7d75a0..ea76f726ea 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -171,7 +171,7 @@ typedef struct SReqResultInfo { char** convertBuf; TAOS_ROW row; SResultColumn* pCol; - uint64_t numOfRows; + uint64_t numOfRows; // from int32_t change to int64_t uint64_t totalRows; uint64_t current; bool localResultFetched; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 77699e89c2..6444150263 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1941,7 +1941,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR pResultInfo->pRspMsg = (const char*)pRsp; pResultInfo->pData = (void*)pRsp->data; - pResultInfo->numOfRows = htonl(pRsp->numOfRows); + pResultInfo->numOfRows = htonll(pRsp->numOfRows); pResultInfo->current = 0; pResultInfo->completed = (pRsp->completed == 1); pResultInfo->payloadLen = htonl(pRsp->compLen); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 7100be58e3..60f8171bac 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -684,7 +684,7 @@ void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pC int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput); int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart); -void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int32_t dataLen, int64_t startTs, SOperatorInfo* pOperator); STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 963a273290..03e9e411bb 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -115,14 +115,14 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 + " execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb, try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, i + 1, totalSources); } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", + " execId:%d blocks:%d, numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); } @@ -367,14 +367,14 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pSourceDataInfo->pRsp = pMsg->pData; SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->numOfRows = htonll(pRsp->numOfRows); pRsp->compLen = htonl(pRsp->compLen); pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%" PRId64 ", %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, pRsp->numOfRows, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); @@ -472,7 +472,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas return TSDB_CODE_SUCCESS; } -void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int32_t dataLen, int64_t startTs, SOperatorInfo* pOperator) { pInfo->totalRows += numOfRows; pInfo->totalSize += dataLen; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index dfc48e7d9f..77aafa9a27 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -297,7 +297,7 @@ typedef struct SSchJob { SExecResult execRes; void *fetchRes; // TODO free it or not bool fetched; - int32_t resNumOfRows; + int64_t resNumOfRows; // from int32_t to int64_t SSchResInfo userRes; char *sql; SQueryProfileSummary summary; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 0eb29a3667..37a3cc9286 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -412,7 +412,7 @@ int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) { SCH_JOB_DLOG("empty res and set query complete, code:%x", code); } - SCH_JOB_DLOG("fetch done, totalRows:%d", pJob->resNumOfRows); + SCH_JOB_DLOG("fetch done, totalRows:%" PRId64, pJob->resNumOfRows); _return: @@ -528,7 +528,7 @@ void schProcessOnDataFetched(SSchJob *pJob) { schPostJobRes(pJob, SCH_OP_FETCH); int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); - atomic_store_32(&pJob->resNumOfRows, htonl(pRsp->numOfRows)); + atomic_store_64(&pJob->resNumOfRows, htonll(pRsp->numOfRows)); atomic_store_ptr(&pJob->fetchRes, pRsp); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1a6d7df349..101fdfe052 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -108,7 +108,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs } atomic_store_ptr(&pJob->fetchRes, rsp); - atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); + atomic_add_fetch_64(&pJob->resNumOfRows, htonll(rsp->numOfRows)); if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); @@ -279,7 +279,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa } } - atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); SCH_TASK_DLOG("submit succeed, affectedRows:%d, blocks:%d", rsp->affectedRows, rsp->nBlocks); SCH_LOCK(SCH_WRITE, &pJob->resLock); @@ -317,7 +317,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa tDecodeSVDeleteRsp(&coder, &rsp); tDecoderClear(&coder); - atomic_add_fetch_32(&pJob->resNumOfRows, rsp.affectedRows); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); SCH_TASK_DLOG("delete succeed, affectedRows:%" PRId64, rsp.affectedRows); } @@ -344,7 +344,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SCH_ERR_JRET(schSaveJobExecRes(pJob, &rsp)); - atomic_add_fetch_32(&pJob->resNumOfRows, rsp.affectedRows); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); taosMemoryFreeClear(msg); From 855b0c8cbe5b4ff45effd78a7b348367715c95be Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 20:57:46 +0800 Subject: [PATCH 55/95] feat(api): add int64 affected rows modify tmsg.h struct --- source/client/src/clientImpl.c | 2 +- source/libs/executor/src/exchangeoperator.c | 6 +++--- source/libs/scheduler/src/schJob.c | 2 +- source/libs/scheduler/src/schRemote.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6444150263..f823b527ea 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1941,7 +1941,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR pResultInfo->pRspMsg = (const char*)pRsp; pResultInfo->pData = (void*)pRsp->data; - pResultInfo->numOfRows = htonll(pRsp->numOfRows); + pResultInfo->numOfRows = htobe64(pRsp->numOfRows); pResultInfo->current = 0; pResultInfo->completed = (pRsp->completed == 1); pResultInfo->payloadLen = htonl(pRsp->compLen); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 03e9e411bb..dfbfe62e13 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -367,7 +367,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pSourceDataInfo->pRsp = pMsg->pData; SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; - pRsp->numOfRows = htonll(pRsp->numOfRows); + pRsp->numOfRows = htobe64(pRsp->numOfRows); pRsp->compLen = htonl(pRsp->compLen); pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); @@ -655,7 +655,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, @@ -664,7 +664,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; pExchangeInfo->current += 1; } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 37a3cc9286..ea49b0c4e0 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -528,7 +528,7 @@ void schProcessOnDataFetched(SSchJob *pJob) { schPostJobRes(pJob, SCH_OP_FETCH); int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); - atomic_store_64(&pJob->resNumOfRows, htonll(pRsp->numOfRows)); + atomic_store_64(&pJob->resNumOfRows, htobe64(pRsp->numOfRows)); atomic_store_ptr(&pJob->fetchRes, pRsp); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 101fdfe052..76ac0a2e9f 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -108,7 +108,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs } atomic_store_ptr(&pJob->fetchRes, rsp); - atomic_add_fetch_64(&pJob->resNumOfRows, htonll(rsp->numOfRows)); + atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows)); if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); From 524de026a553a83500d288b27f3a397561d0bd51 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 22:24:35 +0800 Subject: [PATCH 56/95] enh: remove TSDB_CODE_RPC_REDIRECT --- include/common/tmsgcb.h | 3 --- include/libs/qcom/query.h | 18 ++++++++--------- include/util/taoserror.h | 4 ++-- source/client/src/clientMain.c | 7 ++----- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 20 +------------------ source/dnode/mnode/impl/src/mndTrans.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeSync.c | 12 ++++++++++- source/libs/function/src/udfd.c | 7 ++++--- source/libs/qworker/src/qwDbg.c | 6 +++--- source/libs/transport/src/tmsgcb.c | 2 -- source/libs/transport/src/transCli.c | 2 +- source/util/src/terror.c | 4 +--- 13 files changed, 38 insertions(+), 55 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index b5b997dac0..32d00bb422 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -43,7 +43,6 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); typedef void (*SendRspFp)(SRpcMsg* pMsg); -typedef void (*SendRedirectRspFp)(SRpcMsg* pMsg, const SEpSet* pNewEpSet); typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type); typedef void (*ReportStartup)(const char* name, const char* desc); @@ -55,7 +54,6 @@ typedef struct { GetQueueSizeFp qsizeFp; SendReqFp sendReqFp; SendRspFp sendRspFp; - SendRedirectRspFp sendRedirectRspFp; RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; ReportStartup reportStartupFp; @@ -66,7 +64,6 @@ int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg); int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg); void tmsgSendRsp(SRpcMsg* pMsg); -void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet); void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg); void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type); void tmsgReportStartup(const char* name, const char* desc); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index e2e9da6624..cd61c29559 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -265,21 +265,21 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later #define NEED_REDIRECT_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ - (_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ + (_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ - (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_RPC_BROKEN_LINK || \ - (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) + (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING || \ + (_code) == TSDB_CODE_APP_IS_STOPPING) #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ (_type) == TDMT_MND_DROP_STB) -#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_MNODE_NOT_FOUND || \ - SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \ - SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_SYN_RESTORING || \ - (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) +#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ + ((_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ + (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING || \ + (_code) == TSDB_CODE_APP_IS_STOPPING) #define REQUEST_TOTAL_EXEC_TIMES 2 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 114cc45a59..d9e9f5a4fd 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -42,8 +42,8 @@ int32_t* taosGetErrno(); // rpc // #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x // #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x -#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) -#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) +// #define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) //2.x +// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) // #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x // #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x // #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 976d1dd1b0..8ee774ceee 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -146,7 +146,6 @@ void taos_close(TAOS *taos) { int taos_errno(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return terrno; } @@ -154,12 +153,11 @@ int taos_errno(TAOS_RES *res) { return 0; } - return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code; + return ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return (const char *)tstrerror(terrno); } @@ -171,8 +169,7 @@ const char *taos_errstr(TAOS_RES *res) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { - return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY) - : (const char *)tstrerror(pRequest->code); + return (const char *)tstrerror(pRequest->code); } } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 212de0bfb4..857fbcbce5 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -61,7 +61,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { pMsg->info.rsp = NULL; } - if (code == TSDB_CODE_RPC_REDIRECT) { + if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) { mndPostProcessQueryMsg(pMsg); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 6e7d869eed..a3430d0bde 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -33,23 +33,6 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { } } -static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) { - pMsg->info.hasEpSet = 1; - SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info, .msgType = pMsg->msgType}; - int32_t contLen = tSerializeSEpSet(NULL, 0, pNewEpSet); - - rsp.pCont = rpcMallocCont(contLen); - if (rsp.pCont == NULL) { - pMsg->code = TSDB_CODE_OUT_OF_MEMORY; - } else { - tSerializeSEpSet(rsp.pCont, contLen, pNewEpSet); - rsp.contLen = contLen; - } - dmSendRsp(&rsp); - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; -} - int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; @@ -243,7 +226,7 @@ static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { rpcRegisterBrokenLin static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcReleaseHandle(pHandle, type); } static bool rpcRfp(int32_t code, tmsg_t msgType) { - if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_MNODE_NOT_FOUND || + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || @@ -334,7 +317,6 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) { .clientRpc = pDnode->trans.clientRpc, .sendReqFp = dmSendReq, .sendRspFp = dmSendRsp, - .sendRedirectRspFp = dmSendRedirectRsp, .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg, .releaseHandleFp = dmReleaseHandle, .reportStartupFp = dmReportStartup, diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 38bfe7f6cb..4b11884050 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -945,7 +945,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } if (i != 0 && code == 0) { - code = TSDB_CODE_RPC_REDIRECT; + code = TSDB_CODE_MNODE_NOT_FOUND; } mInfo("trans:%d, client:%d send rsp, code:0x%x stage:%s app:%p", pTrans->id, i, code, mndTransStr(pTrans->stage), pInfo->ahandle); @@ -1042,8 +1042,8 @@ static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pA pAction->rawWritten = 0; pAction->msgSent = 0; pAction->msgReceived = 0; - if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || - pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { + if (pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || + pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; mInfo("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), pAction->id, pAction->epSet.inUse); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index ee95e5b7d1..ed612ff6a6 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -67,7 +67,17 @@ void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) { pMsg->info.hasEpSet = 1; SRpcMsg rsp = {.code = TSDB_CODE_SYN_NOT_LEADER, .info = pMsg->info, .msgType = pMsg->msgType + 1}; - tmsgSendRedirectRsp(&rsp, &newEpSet); + int32_t contLen = tSerializeSEpSet(NULL, 0, &newEpSet); + + rsp.pCont = rpcMallocCont(contLen); + if (rsp.pCont == NULL) { + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + tSerializeSEpSet(rsp.pCont, contLen, &newEpSet); + rsp.contLen = contLen; + } + + tmsgSendRsp(&rsp); } static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index ec54f8ff2d..9d249e0831 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -598,9 +598,10 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { return 0; } static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { - if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND|| code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { - if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_SYN_NOT_LEADER || + code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_APP_IS_STOPPING) { + if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; } return true; diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 4c4a41df82..a122508f98 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -214,20 +214,20 @@ void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) { epSet.eps[2].port = 7300; ctx->phase = QW_PHASE_POST_QUERY; - qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, &epSet); + qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, &epSet); *rsped = true; return; } if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { QW_SET_PHASE(ctx, QW_PHASE_POST_QUERY); - qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); + qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); *rsped = true; return; } if ((TDMT_SCH_FETCH == qwMsg->msgType) && (0 == taosRand() % 9)) { - qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); + qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); *rsped = true; return; } diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c index 559715084c..95bc532994 100644 --- a/source/libs/transport/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -54,8 +54,6 @@ void tmsgSendRsp(SRpcMsg* pMsg) { #endif } -void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { (*defaultMsgCb.sendRedirectRspFp)(pMsg, pNewEpSet); } - void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { (*defaultMsgCb.registerBrokenLinkArgFp)(pMsg); } void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { (*defaultMsgCb.releaseHandleFp)(pHandle, type); } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a9afbd7ba8..d588e2db1c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1519,7 +1519,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || - code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_VND_STOPPED) { + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED) { tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4082e8985a..2110c0b3aa 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -46,8 +46,6 @@ STaosError errors[] = { #endif // rpc -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, "Authentication failure") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, "Database not ready, need retry") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "Port already in use") @@ -320,7 +318,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_ALREADY_DEPLOYED, "Snode already deploye TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_DEPLOYED, "Snode not deployed") // vnode -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Invalid Vgroup ID") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode moved to another dnode or was deleted") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_EXIST, "Vnode not exist") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST, "Vnode already exist") From f3e980de149c5053bffa7d861596d4c4d506631b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 22:45:11 +0800 Subject: [PATCH 57/95] enh: remove TSDB_CODE_RPC_REDIRECT --- include/libs/qcom/query.h | 12 +++++------- include/util/taoserror.h | 2 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 6 +++--- source/dnode/vnode/src/inc/vnd.h | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeSync.c | 8 +++++--- source/libs/transport/src/transCli.c | 4 +++- 7 files changed, 21 insertions(+), 19 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index cd61c29559..97ce5ae032 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -261,25 +261,23 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) #define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || (_code) == TSDB_CODE_VND_STOPPED) -#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) +#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) #define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later #define NEED_REDIRECT_ERROR(_code) \ ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ - (_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + (_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ - (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING || \ - (_code) == TSDB_CODE_APP_IS_STOPPING) + (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ (_type) == TDMT_MND_DROP_STB) #define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ - ((_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + ((_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ - (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING || \ - (_code) == TSDB_CODE_APP_IS_STOPPING) + (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) #define REQUEST_TOTAL_EXEC_TIMES 2 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index d9e9f5a4fd..2baf4d14e2 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -43,7 +43,7 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x // #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x // #define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) //2.x -// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) +// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) //2.x // #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x // #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x // #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index a3430d0bde..12aba130d5 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -226,9 +226,9 @@ static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { rpcRegisterBrokenLin static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcReleaseHandle(pHandle, type); } static bool rpcRfp(int32_t code, tmsg_t msgType) { - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_MNODE_NOT_FOUND || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_RPC_BROKEN_LINK || - code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_MNODE_NOT_FOUND || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_VND_STOPPED || + code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index e2b7327e8f..15e64ffaed 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -100,7 +100,7 @@ int32_t vnodeSyncOpen(SVnode* pVnode, char* path); int32_t vnodeSyncStart(SVnode* pVnode); void vnodeSyncPreClose(SVnode* pVnode); void vnodeSyncClose(SVnode* pVnode); -void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg); +void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg, int32_t code); bool vnodeIsLeader(SVnode* pVnode); bool vnodeIsRoleLeader(SVnode* pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d63c45fc0e..fc6b126cc5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -344,7 +344,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); // if ((pMsg->msgType == TDMT_SCH_QUERY) && !vnodeIsLeader(pVnode)) { if ((pMsg->msgType == TDMT_SCH_QUERY) && !syncIsReadyForRead(pVnode->sync)) { - vnodeRedirectRpcMsg(pVnode, pMsg); + vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; } @@ -367,12 +367,12 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { pMsg->msgType == TDMT_VND_BATCH_META) && !syncIsReadyForRead(pVnode->sync)) { // !vnodeIsLeader(pVnode)) { - vnodeRedirectRpcMsg(pVnode, pMsg); + vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; } if (pMsg->msgType == TDMT_VND_TMQ_CONSUME && !pVnode->restored) { - vnodeRedirectRpcMsg(pVnode, pMsg); + vnodeRedirectRpcMsg(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index ed612ff6a6..a155e8da86 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -53,7 +53,7 @@ static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { } } -void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) { +void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) { SEpSet newEpSet = {0}; syncGetRetryEpSet(pVnode->sync, &newEpSet); @@ -66,7 +66,9 @@ void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) { } pMsg->info.hasEpSet = 1; - SRpcMsg rsp = {.code = TSDB_CODE_SYN_NOT_LEADER, .info = pMsg->info, .msgType = pMsg->msgType + 1}; + if (code == 0) code = TSDB_CODE_SYN_NOT_LEADER; + + SRpcMsg rsp = {.code = code, .info = pMsg->info, .msgType = pMsg->msgType + 1}; int32_t contLen = tSerializeSEpSet(NULL, 0, &newEpSet); rsp.pCont = rpcMallocCont(contLen); @@ -98,7 +100,7 @@ static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) { if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) { - vnodeRedirectRpcMsg(pVnode, pMsg); + vnodeRedirectRpcMsg(pVnode, pMsg, code); } else { const STraceId *trace = &pMsg->info.traceId; vGError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", pVnode->config.vgId, pMsg, tstrerror(code), code); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d588e2db1c..7339d487d1 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1519,7 +1519,9 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || - code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED) { + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED || + code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_APP_IS_STOPPING) { tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); From 5e77b3eca3025815967c44111e959d4c5efb64c4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Dec 2022 23:10:03 +0800 Subject: [PATCH 58/95] refactor: add function for extract the required buffer during query. --- source/libs/executor/inc/executorimpl.h | 21 +++--- source/libs/executor/src/cachescanoperator.c | 2 +- source/libs/executor/src/exchangeoperator.c | 2 +- source/libs/executor/src/executor.c | 17 +++-- source/libs/executor/src/executorimpl.c | 73 ++++++------------- source/libs/executor/src/filloperator.c | 4 +- source/libs/executor/src/groupoperator.c | 6 +- source/libs/executor/src/joinoperator.c | 2 +- source/libs/executor/src/projectoperator.c | 6 +- source/libs/executor/src/scanoperator.c | 18 ++--- source/libs/executor/src/sortoperator.c | 8 +- source/libs/executor/src/sysscanoperator.c | 4 +- source/libs/executor/src/timesliceoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 38 ++++------ .../libs/function/src/detail/tavgfunction.c | 23 ++---- 15 files changed, 93 insertions(+), 133 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ec2fadef28..9d32bf830e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -46,7 +46,6 @@ extern "C" { typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); -#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) #define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0) #define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN) #define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0) @@ -109,6 +108,7 @@ typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr); typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr); typedef void (*__optr_close_fn_t)(void* param); typedef int32_t (*__optr_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); +typedef int32_t (*__optr_reqBuf_fn_t)(struct SOperatorInfo* pOptr); typedef struct STaskIdInfo { uint64_t queryId; // this is also a request id @@ -170,8 +170,9 @@ struct SExecTaskInfo { STaskCostInfo cost; int64_t owner; // if it is in execution int32_t code; + int32_t qbufQuota; // total available buffer (in KB) during execution query - int64_t version; // used for stream to record wal version + int64_t version; // used for stream to record wal version, why not move to sschemainfo SStreamTaskInfo streamInfo; SSchemaInfo schemaInfo; STableListInfo* pTableInfoList; // this is a table list @@ -198,6 +199,7 @@ typedef struct SOperatorFpSet { __optr_fn_t getNextFn; __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP __optr_close_fn_t closeFn; + __optr_reqBuf_fn_t reqBufFn; // total used buffer for blocking operator __optr_encode_fn_t encodeResultRow; __optr_decode_fn_t decodeResultRow; __optr_explain_fn_t getExplainFn; @@ -486,12 +488,10 @@ typedef struct STableCountScanSupp { int16_t dbNameSlotId; int16_t stbNameSlotId; int16_t tbCountSlotId; - - bool groupByDbName; - bool groupByStbName; + bool groupByDbName; + bool groupByStbName; char dbNameFilter[TSDB_DB_NAME_LEN]; char stbNameFilter[TSDB_TABLE_NAME_LEN]; - } STableCountScanSupp; typedef struct STableCountScanOperatorInfo { @@ -671,13 +671,14 @@ typedef struct SStreamFillOperatorInfo { #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, - __optr_close_fn_t closeFn, __optr_explain_fn_t explain); -int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); + __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain); +int32_t optrDummyOpenFn(SOperatorInfo* pOperator); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); void setOperatorCompleted(SOperatorInfo* pOperator); void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, void* pInfo, SExecTaskInfo* pTaskInfo); void destroyOperatorInfo(SOperatorInfo* pOperator); +int32_t optrDefaultBufFn(SOperatorInfo* pOperator); void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); void cleanupBasicInfo(SOptrBasicInfo* pInfo); @@ -740,6 +741,8 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* handle, STableCountScanPhysiNode* pNode, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); @@ -812,8 +815,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); -int32_t getMaximumIdleDurationSec(); - STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t order); int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index c432f3c01c..672bb09b14 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -117,7 +117,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, NULL); + createOperatorFpSet(optrDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, optrDefaultBufFn, NULL); pOperator->cost.openCost = 0; return pOperator; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 546f5cc420..76e27f12fc 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -307,7 +307,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); + createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL); return pOperator; _error: diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index aedab23f74..60080d204b 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -503,7 +503,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo } if (pTaskInfo->cost.start == 0) { - pTaskInfo->cost.start = taosGetTimestampMs(); + pTaskInfo->cost.start = taosGetTimestampUs(); } if (isTaskKilled(pTaskInfo)) { @@ -597,7 +597,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { } if (pTaskInfo->cost.start == 0) { - pTaskInfo->cost.start = taosGetTimestampMs(); + pTaskInfo->cost.start = taosGetTimestampUs(); } if (isTaskKilled(pTaskInfo)) { @@ -706,15 +706,20 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) { static void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) { STaskCostInfo* pSummary = &pTaskInfo->cost; + int64_t idleTime = pSummary->start - pSummary->created; SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; if (pSummary->pRecoder != NULL) { qDebug( - "%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, " + "%s :cost summary: idle in queue:%.2f ms, elapsed time:%.2f ms, extract tableList:%.2f ms, " + "createGroupIdMap:%.2f ms, total blocks:%d, " "load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, - GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime, - pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, - pRecorder->totalCheckedRows); + GET_TASKID(pTaskInfo), idleTime / 1000.0, pSummary->elapsedTime / 1000.0, pSummary->extractListTime, + pSummary->groupIdMapTime, pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, + pRecorder->totalRows, pRecorder->totalCheckedRows); + } else { + qDebug("%s :cost summary: idle in queue:%.2f ms, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), idleTime / 1000.0, + pSummary->elapsedTime / 1000.0); } } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fb705dacfa..a489cd1ac8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -85,8 +85,6 @@ typedef struct SAggOperatorInfo { SExprSupp scalarExprSup; } SAggOperatorInfo; -int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } - static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock); static void releaseQueryBuf(size_t numOfTables); @@ -106,7 +104,7 @@ void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; ASSERT(pOperator->pTaskInfo != NULL); - pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start * 1000) / 1000.0; + pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start) / 1000.0; setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } @@ -120,19 +118,21 @@ void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, b pOperator->pTaskInfo = pTaskInfo; } -int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { +int32_t optrDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); pOperator->cost.openCost = 0; return TSDB_CODE_SUCCESS; } SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, - __optr_close_fn_t closeFn, __optr_explain_fn_t explain) { + __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, + __optr_explain_fn_t explain) { SOperatorFpSet fpSet = { ._openFn = openFn, .getNextFn = nextFn, .cleanupFn = cleanup, .closeFn = closeFn, + .reqBufFn = reqBufFn, .getExplainFn = explain, }; @@ -939,10 +939,10 @@ static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, u } static void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, - const int32_t* rowCellOffset) { + const int32_t* rowEntryOffset) { bool returnNotNull = false; for (int32_t j = 0; j < numOfExprs; ++j) { - struct SResultRowEntryInfo* pResInfo = getResultEntryInfo(pRow, j, rowCellOffset); + SResultRowEntryInfo* pResInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (!isRowEntryInitialized(pResInfo)) { continue; } @@ -1145,43 +1145,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG } } -// void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) { -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// -// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0) { -// return; -// } -// -// pQueryAttr->pos = 0; -// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); -// -// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; -// -// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pTsdbReadHandle)) { -// if (isTaskKilled(pRuntimeEnv->qinfo)) { -// T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); -// } -// -// if (pQueryAttr->limit.offset > blockInfo.rows) { -// pQueryAttr->limit.offset -= blockInfo.rows; -// pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; -// pTableQueryInfo->lastKey += step; -// -// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_TASKID(pRuntimeEnv), blockInfo.rows, -// pQuery->limit.offset); -// } else { // find the appropriated start position in current block -// updateOffsetVal(pRuntimeEnv, &blockInfo); -// break; -// } -// } -// -// if (terrno != TSDB_CODE_SUCCESS) { -// T_LONG_JMP(pRuntimeEnv->env, terrno); -// } -// } - // static TSKEY doSkipIntervalProcess(STaskRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, // STableQueryInfo* pTableQueryInfo) { // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1406,11 +1369,11 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; bool hasCountFunc = false; + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - if ((strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "count") == 0) || - (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "hyperloglog") == 0) || - (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_partial") == 0) || - (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_merge") == 0)) { + const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; + if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) || + (strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) { hasCountFunc = true; break; } @@ -1463,7 +1426,6 @@ static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock **ppB *ppBlock = NULL; } - // this is a blocking operator static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { if (OPTR_IS_OPENED(pOperator)) { @@ -1614,6 +1576,15 @@ void destroyOperatorInfo(SOperatorInfo* pOperator) { taosMemoryFreeClear(pOperator); } +// each operator should be set their own function to return total cost buffer +int32_t optrDefaultBufFn(SOperatorInfo* pOperator) { + if (pOperator->blocking) { + ASSERT(0); + } else { + return 0; + } +} + int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz) { *defaultPgsz = 4096; while (*defaultPgsz < rowSize * 4) { @@ -1794,7 +1765,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, optrDefaultBufFn, NULL); if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = downstream->info; @@ -1871,8 +1842,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); -SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* handle, STableCountScanPhysiNode* pNode, - SExecTaskInfo* pTaskInfo); int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, SExecTaskInfo* pTaskInfo) { SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 5ed26f627a..8d5af64777 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -381,7 +381,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -1478,7 +1478,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi pInfo->srcRowIndex = 0; setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 4601175561..5d34064b7e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -470,7 +470,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* setOperatorInfo(pOperator, "GroupbyAggOperator", 0, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -850,7 +850,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -1142,7 +1142,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index e7cce39dfd..d460af971c 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -136,7 +136,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { goto _error; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index da77facb21..65bb40b195 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -118,8 +118,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pOperator->exprSupp.pCtx, numOfCols); setOperatorInfo(pOperator, "ProjectOperator", QUERY_NODE_PHYSICAL_PLAN_PROJECT, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, - destroyProjectOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doProjectOperation, NULL, + destroyProjectOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -415,7 +415,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); setOperatorInfo(pOperator, "IndefinitOperator", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1b05e8b573..0d245f0815 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -899,8 +899,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, - getTableScannerExecInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, + optrDefaultBufFn, getTableScannerExecInfo); // for non-blocking operator, the open cost is always 0 pOperator->cost.openCost = 0; @@ -925,7 +925,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScanImpl, NULL, NULL, optrDefaultBufFn, NULL); return pOperator; } @@ -2139,7 +2139,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, optrDefaultBufFn, NULL); return pOperator; _end: @@ -2328,7 +2328,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn, NULL); return pOperator; @@ -2465,7 +2465,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL); return pOperator; @@ -2866,8 +2866,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, - getTableMergeScanExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, + optrDefaultBufFn, getTableMergeScanExplainExecInfo); pOperator->cost.openCost = 0; return pOperator; @@ -3015,7 +3015,7 @@ SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableC setOperatorInfo(pOperator, "TableCountScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doTableCountScan, NULL, destoryTableCountScanOperator, NULL); + createOperatorFpSet(optrDummyOpenFn, doTableCountScan, NULL, destoryTableCountScanOperator, optrDefaultBufFn, NULL); return pOperator; _error: diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index b355b85861..005b794f0b 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -75,7 +75,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* // TODO dynamic set the available sort buffer pOperator->fpSet = - createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, getExplainExecInfo); + createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, optrDefaultBufFn, getExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -521,8 +521,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); setOperatorInfo(pOperator, "GroupSortOperator", QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, - getGroupSortExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, + optrDefaultBufFn, getGroupSortExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -798,7 +798,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL, - destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo); + destroyMultiwayMergeOperatorInfo, optrDefaultBufFn, getMultiwayMergeExplainExecInfo); code = appendDownstream(pOperator, downStreams, numStreams); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 96f21babe6..32905ca897 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1437,7 +1437,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, optrDefaultBufFn, NULL); return pOperator; _error: @@ -1943,7 +1943,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, optrDefaultBufFn, NULL); return pOperator; _error: diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 90f5dde7c3..2374d80bbf 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -544,7 +544,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, optrDefaultBufFn, NULL); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b0466b6216..2d0893b97b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1049,14 +1049,14 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SIntervalAggOperatorInfo* pInfo = pOperator->info; SExprSupp* pSup = &pOperator->exprSupp; int32_t scanFlag = MAIN_SCAN; - - int64_t st = taosGetTimestampUs(); - SOperatorInfo* downstream = pOperator->pDownstream[0]; + int64_t st = taosGetTimestampUs(); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -1268,9 +1268,6 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { } SSDataBlock* pBlock = pInfo->binfo.pRes; - - ASSERT(pInfo->execModel == OPTR_EXEC_MODEL_BATCH); - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { return NULL; @@ -1765,7 +1762,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->interval = interval; - pInfo->execModel = pTaskInfo->execModel; pInfo->twAggSup = as; pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock; @@ -1797,7 +1793,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, NULL); + createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2015,7 +2011,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, NULL); + createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2088,7 +2084,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, optrDefaultBufFn, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2749,7 +2745,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); + createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); } @@ -3561,7 +3557,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL); if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); @@ -3705,8 +3701,8 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); blockDataEnsureCapacity(pInfo->pUpdateRes, 128); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, - destroyStreamSessionAggOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL, + destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL); } setOperatorInfo(pOperator, name, pPhyNode->type, false, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -4066,7 +4062,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL); initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4314,7 +4310,6 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, iaInfo->win = pTaskInfo->window; iaInfo->inputOrder = TSDB_ORDER_ASC; iaInfo->interval = interval; - iaInfo->execModel = pTaskInfo->execModel; iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId; iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock; @@ -4344,7 +4339,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, false, OP_NOT_OPENED, miaInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4620,7 +4615,6 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge pIntervalInfo->win = pTaskInfo->window; pIntervalInfo->inputOrder = TSDB_ORDER_ASC; pIntervalInfo->interval = interval; - pIntervalInfo->execModel = pTaskInfo->execModel; pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock; pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; @@ -4650,7 +4644,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false, OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4866,8 +4860,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL, + destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index f06bafafe3..1c74d22a82 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -270,7 +270,7 @@ static void i64VectorSumAVX2(const int64_t* plist, int32_t numOfRows, SAvgRes* p #if __AVX2__ // find the start position that are aligned to 32bytes address in memory - int32_t width = (bitWidth>>3u) / sizeof(int64_t); + int32_t width = (bitWidth >> 3u) / sizeof(int64_t); int32_t remainder = numOfRows % width; int32_t rounds = numOfRows / width; @@ -286,20 +286,11 @@ static void i64VectorSumAVX2(const int64_t* plist, int32_t numOfRows, SAvgRes* p } // let sum up the final results - if (type == TSDB_DATA_TYPE_BIGINT) { - const int64_t* q = (const int64_t*)∑ - pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; + const int64_t* q = (const int64_t*)∑ + pRes->sum.isum += q[0] + q[1] + q[2] + q[3]; - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.isum += plist[j + rounds * width]; - } - } else { - const uint64_t* q = (const uint64_t*)∑ - pRes->sum.usum += q[0] + q[1] + q[2] + q[3]; - - for (int32_t j = 0; j < remainder; ++j) { - pRes->sum.usum += (uint64_t)plist[j + rounds * width]; - } + for (int32_t j = 0; j < remainder; ++j) { + pRes->sum.isum += plist[j + rounds * width]; } #endif @@ -588,14 +579,14 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { const int64_t* plist = (const int64_t*) pCol->pData; // 1. If the CPU supports AVX, let's employ AVX instructions to speedup this loop - if (simdAvailable) { + if (simdAvailable && type == TSDB_DATA_TYPE_BIGINT) { i64VectorSumAVX2(plist, numOfRows, pAvgRes); } else { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { if (type == TSDB_DATA_TYPE_BIGINT) { pAvgRes->sum.isum += plist[i]; } else { - pAvgRes->sum.isum += (uint64_t)plist[i]; + pAvgRes->sum.usum += (uint64_t)plist[i]; } } } From 9e803332fce3d007fee29244b116eec10f8697bc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 23:10:12 +0800 Subject: [PATCH 59/95] refact: adjust error codes --- include/util/taoserror.h | 94 +++++++++++++-------------- source/dnode/mnode/impl/src/mndAcct.c | 6 +- source/util/src/terror.c | 3 - 3 files changed, 50 insertions(+), 53 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2baf4d14e2..490dbf7744 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -40,61 +40,61 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error // rpc -// #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x -// #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x -// #define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) //2.x -// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) //2.x -// #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x -// #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x -// #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x -// #define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) //2.x -// #define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) //2.x -// #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) //2.x +// #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) // 2.x +// #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) // 2.x +// #define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) // 2.x +// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) // 2.x +// #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) // 2.x +// #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) // 2.x +// #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) // 2.x +// #define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) // 2.x +// #define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) // 2.x +// #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) // 2.x #define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B) -// #define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) //2.x -// #define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) //2.x -// #define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) //2.x -// #define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) //2.x -// #define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) //2.x -// #define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) //2.x -// #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) //2.x -#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) -// #define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) //2.x +// #define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) // 2.x +// #define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) // 2.x +// #define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) // 2.x +// #define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) // 2.x +// #define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) // 2.x +// #define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) // 2.x +// #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) // 2.x +#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) // +// #define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) // 2.x #define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) -// #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //2.x -#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) -#define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) -#define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) +// #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) // 2.x +#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) // +#define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) // +#define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) // //common & util -#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) -#define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101) +#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) // +// #define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101) // 2.x #define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0102) // #define TSDB_CODE_COM_INVALID_CFG_MSG TAOS_DEF_ERROR_CODE(0, 0x0103) // 2.x -#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104) -#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105) -#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106) -#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0107) -#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0108) -#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0109) -#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010A) +#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104) // +#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105) // internal +#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106) // internal +#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0107) // internal +#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0108) // internal +#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0109) // internal +#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010A) // internal -#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0110) -#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0111) -#define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0112) -#define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0113) -#define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0114) -#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0115) +#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0110) // +#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0111) // internal +#define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0112) // +// #define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0113) +// #define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0114) +#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0115) // #define TSDB_CODE_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0116) // -#define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0117) -#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0118) -#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0119) -#define TSDB_CODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x011A) -#define TSDB_CODE_INVALID_JSON_FORMAT TAOS_DEF_ERROR_CODE(0, 0x011B) -#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x011C) -#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x011D) -#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x011E) -#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F) +#define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0117) // internal +#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0118) // +#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0119) // internal +#define TSDB_CODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x011A) // internal +#define TSDB_CODE_INVALID_JSON_FORMAT TAOS_DEF_ERROR_CODE(0, 0x011B) // internal +#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x011C) // internal +#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x011D) // internal +#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x011E) // internal +#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F) // internal #define TSDB_CODE_COMPRESS_ERROR TAOS_DEF_ERROR_CODE(0, 0x0120) #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121) // diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index ce2ec7b23a..148e21b507 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -220,7 +220,7 @@ static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) { return -1; } - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_OPS_NOT_SUPPORT; mError("failed to process create acct request since %s", terrstr()); return -1; } @@ -230,7 +230,7 @@ static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) { return -1; } - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_OPS_NOT_SUPPORT; mError("failed to process create acct request since %s", terrstr()); return -1; } @@ -240,7 +240,7 @@ static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) { return -1; } - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_OPS_NOT_SUPPORT; mError("failed to process create acct request since %s", terrstr()); return -1; } \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2110c0b3aa..18eeca030e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -55,7 +55,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT, "Conn read timeout") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") TAOS_DEFINE_ERROR(TSDB_CODE_OPS_NOT_SUPPORT, "Operation not supported") -TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, "Memory corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_MEMORY, "Out of Memory") TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, "Data file corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") @@ -68,8 +67,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, "Ref is not there") TAOS_DEFINE_ERROR(TSDB_CODE_APP_ERROR, "Unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RANGE, "Out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_SHM_MEM, "Out of Shared memory") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SHM_ID, "Invalid SHM ID") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG_LEN, "Invalid message len") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PTR, "Invalid pointer") From 3d28255dfae2ea98a1aa720502eafd5f2794a3b8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Dec 2022 23:19:57 +0800 Subject: [PATCH 60/95] change rpc log level --- source/libs/transport/src/transCli.c | 3 ++- source/libs/transport/src/transSvr.c | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a9afbd7ba8..4df2790959 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1519,7 +1519,8 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || - code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_VND_STOPPED) { + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT || + code == TSDB_CODE_VND_STOPPED) { tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 71e2e30511..73c6dc4011 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -250,9 +250,9 @@ static bool uvHandleReq(SSvrConn* pConn) { transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, transMsg.code, (int)(cost)); } else { - tGWarn("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d, cost:%dus", - transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, - transMsg.code, (int)(cost)); + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d, cost:%dus", + transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, + transMsg.code, (int)(cost)); } } From 213b97b79ac2722db50cc029f363a88a84f0ac7d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Dec 2022 23:20:19 +0800 Subject: [PATCH 61/95] feat(api): taos_affected_rows64 add and modfiy tmsg.h --- include/libs/executor/dataSinkMgt.h | 2 +- source/client/src/clientMsgHandler.c | 2 +- source/dnode/vnode/src/tq/tqExec.c | 2 +- source/libs/command/src/command.c | 2 +- source/libs/command/src/explain.c | 2 +- source/libs/qworker/src/qwMsg.c | 2 +- source/libs/qworker/src/qworker.c | 6 +++--- source/libs/scheduler/src/schJob.c | 2 +- source/libs/scheduler/src/schRemote.c | 2 +- source/libs/stream/src/streamDispatch.c | 4 ++-- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 8a02f372d1..ed7cbc8125 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -68,7 +68,7 @@ typedef struct SInputData { typedef struct SOutputData { int32_t numOfBlocks; - int32_t numOfRows; + int64_t numOfRows; // int32_t changed to int64_t int32_t numOfCols; int8_t compressed; char* pData; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 4bd74a842f..d5a567075c 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -450,7 +450,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->precision = 0; (*pRsp)->compressed = 0; (*pRsp)->compLen = 0; - (*pRsp)->numOfRows = htonl(pBlock->info.rows); + (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 2514190035..093186ebbb 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -25,7 +25,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->precision = precision; pRetrieve->compressed = 0; pRetrieve->completed = 1; - pRetrieve->numOfRows = htonl(pBlock->info.rows); + pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index d58c4dc6d3..7661944379 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -36,7 +36,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->precision = 0; (*pRsp)->compressed = 0; (*pRsp)->compLen = 0; - (*pRsp)->numOfRows = htonl(pBlock->info.rows); + (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 410e62a18b..dd48023679 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1615,7 +1615,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { } rsp->completed = 1; - rsp->numOfRows = htonl(rowNum); + rsp->numOfRows = htobe64((int64_t)rowNum); int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index d9a7cea411..7154e22300 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -37,7 +37,7 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) rsp->precision = input->precision; rsp->compressed = input->compressed; rsp->compLen = htonl(len); - rsp->numOfRows = htonl(input->numOfRows); + rsp->numOfRows = htobe64(input->numOfRows); rsp->numOfCols = htonl(input->numOfCols); rsp->numOfBlocks = htonl(input->numOfBlocks); } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 0890d10b65..fe2acb805b 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -275,7 +275,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, QW_ERR_RET(code); } - QW_TASK_DLOG("no more data in sink and query end, fetched blocks %d rows %d", pOutput->numOfBlocks, + QW_TASK_DLOG("no more data in sink and query end, fetched blocks %d rows %"PRId64, pOutput->numOfBlocks, pOutput->numOfRows); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); @@ -320,7 +320,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, pOutput->numOfBlocks++; if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { - QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %d", pOutput->numOfBlocks, + QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %" PRId64, pOutput->numOfBlocks, pOutput->numOfRows); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); break; @@ -332,7 +332,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, } if (pOutput->numOfRows >= QW_MIN_RES_ROWS) { - QW_TASK_DLOG("task fetched blocks %d rows %d reaches the min rows", pOutput->numOfBlocks, pOutput->numOfRows); + QW_TASK_DLOG("task fetched blocks %d rows %" PRId64 " reaches the min rows", pOutput->numOfBlocks, pOutput->numOfRows); break; } } diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index ea49b0c4e0..b9062859c8 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -526,7 +526,7 @@ int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { void schProcessOnDataFetched(SSchJob *pJob) { schPostJobRes(pJob, SCH_OP_FETCH); } int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { - SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); + SCH_TASK_DLOG("got explain rsp, rows:%" PRId64 ", complete:%d", htobe64(pRsp->numOfRows), pRsp->completed); atomic_store_64(&pJob->resNumOfRows, htobe64(pRsp->numOfRows)); atomic_store_ptr(&pJob->fetchRes, pRsp); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 76ac0a2e9f..30622f96bb 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -114,7 +114,7 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); } - SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed); + SCH_TASK_DLOG("got fetch rsp, rows:%" PRId64 ", complete:%d", htobe64(rsp->numOfRows), rsp->completed); msg = NULL; schProcessOnDataFetched(pJob); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 2c36c299ee..4e0b0630bc 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -112,7 +112,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->compressed = 0; pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; - pRetrieve->numOfRows = htonl(pBlock->info.rows); + pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); pRetrieve->numOfCols = htonl(numOfCols); pRetrieve->skey = htobe64(pBlock->info.window.skey); pRetrieve->ekey = htobe64(pBlock->info.window.ekey); @@ -189,7 +189,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->compressed = 0; pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; - pRetrieve->numOfRows = htonl(pBlock->info.rows); + pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); pRetrieve->skey = htobe64(pBlock->info.window.skey); pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); From c20e6078259ba0090da7a0759f09acdb3d1a9f5e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 2 Dec 2022 23:32:31 +0800 Subject: [PATCH 62/95] fix: compile error --- source/dnode/mnode/impl/test/acct/acct.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp index 46a9a465eb..1f59a7fcca 100644 --- a/source/dnode/mnode/impl/test/acct/acct.cpp +++ b/source/dnode/mnode/impl/test/acct/acct.cpp @@ -32,7 +32,7 @@ TEST_F(MndTestAcct, 01_Create_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); } TEST_F(MndTestAcct, 02_Alter_Acct) { @@ -42,7 +42,7 @@ TEST_F(MndTestAcct, 02_Alter_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); } TEST_F(MndTestAcct, 03_Drop_Acct) { @@ -52,5 +52,5 @@ TEST_F(MndTestAcct, 03_Drop_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); } From de0022b473a2f19093ee7cbccbf3f5df36c35370 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Dec 2022 23:38:05 +0800 Subject: [PATCH 63/95] fix(query): fix error in record the created time. --- source/libs/executor/src/executor.c | 4 ++-- source/libs/executor/src/executorimpl.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 60080d204b..4ad81f609a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -207,7 +207,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n } setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTaskInfo->cost.created = taosGetTimestampMs(); + pTaskInfo->cost.created = taosGetTimestampUs(); pTaskInfo->execModel = OPTR_EXEC_MODEL_QUEUE; pTaskInfo->pRoot = createRawScanOperatorInfo(readers, pTaskInfo); if (NULL == pTaskInfo->pRoot) { @@ -711,7 +711,7 @@ static void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) { SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; if (pSummary->pRecoder != NULL) { qDebug( - "%s :cost summary: idle in queue:%.2f ms, elapsed time:%.2f ms, extract tableList:%.2f ms, " + "%s :cost summary: idle:%.2f ms, elapsed time:%.2f ms, extract tableList:%.2f ms, " "createGroupIdMap:%.2f ms, total blocks:%d, " "load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, GET_TASKID(pTaskInfo), idleTime / 1000.0, pSummary->elapsedTime / 1000.0, pSummary->extractListTime, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a489cd1ac8..b408e32061 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1826,7 +1826,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->schemaInfo.dbname = strdup(dbFName); - pTaskInfo->cost.created = taosGetTimestampMs(); pTaskInfo->id.queryId = queryId; pTaskInfo->execModel = model; pTaskInfo->pTableInfoList = tableListCreate(); @@ -2333,6 +2332,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead goto _complete; } + (*pTaskInfo)->cost.created = taosGetTimestampUs(); return TSDB_CODE_SUCCESS; _complete: From 69eae060665f184729ad0a6d72af9d36377bf4e1 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Fri, 2 Dec 2022 23:45:55 +0800 Subject: [PATCH 64/95] enh(taosAdapter): support loongArch64 --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index cc46ef9938..2a7b76d768 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG ff7de07 + GIT_TAG e07f41b SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From e97c8f637ba4506078e1018be1c241346f9b11da Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 3 Dec 2022 08:25:57 +0800 Subject: [PATCH 65/95] enh: logging sync log buffer rollback events --- source/libs/sync/inc/syncPipeline.h | 2 +- source/libs/sync/src/syncPipeline.c | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index 4208d40a69..8c7edf85ff 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -107,7 +107,7 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode); // private SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex index, bool* pInBuf); int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf); -int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SyncIndex toIndex); +int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index f0be976402..fcb0d63209 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -310,7 +310,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt ASSERT(pEntry->index == pExist->index); if (pEntry->term != pExist->term) { - (void)syncLogBufferRollback(pBuf, index); + (void)syncLogBufferRollback(pBuf, pNode, index); } else { sTrace("vgId:%d, duplicate log entry received. index: %" PRId64 ", term: %" PRId64 ". log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", @@ -971,14 +971,18 @@ void syncLogBufferDestroy(SSyncLogBuffer* pBuf) { return; } -int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SyncIndex toIndex) { +int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex) { ASSERT(pBuf->commitIndex < toIndex && toIndex <= pBuf->endIndex); + sInfo("vgId:%d, rollback sync log buffer. toindex: %" PRId64 ", buffer: [%" PRId64 " %" PRId64 " %" PRId64 + ", %" PRId64 ")", + pNode->vgId, toIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); + SyncIndex index = pBuf->endIndex - 1; while (index >= toIndex) { SSyncRaftEntry* pEntry = pBuf->entries[index % pBuf->size].pItem; if (pEntry != NULL) { - syncEntryDestroy(pEntry); + (void)syncEntryDestroy(pEntry); pEntry = NULL; memset(&pBuf->entries[index % pBuf->size], 0, sizeof(pBuf->entries[0])); } @@ -996,7 +1000,7 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { ASSERT(lastVer == pBuf->matchIndex); SyncIndex index = pBuf->endIndex - 1; - (void)syncLogBufferRollback(pBuf, pBuf->matchIndex + 1); + (void)syncLogBufferRollback(pBuf, pNode, pBuf->matchIndex + 1); sInfo("vgId:%d, reset sync log buffer. buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); From 2ce07c6ec1720c28dd6278020b86f70d716447b0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 3 Dec 2022 08:51:58 +0800 Subject: [PATCH 66/95] fix: get result of tmsPutToQueue for FpCommitCb --- include/libs/sync/sync.h | 4 ++-- source/dnode/mnode/impl/src/mndSync.c | 9 ++++++--- source/dnode/vnode/src/vnd/vnodeSync.c | 13 +++++++------ source/libs/sync/src/syncPipeline.c | 9 +++++---- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 4741f7f3d8..1fa862b145 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -138,8 +138,8 @@ typedef struct SSnapshotMeta { typedef struct SSyncFSM { void* data; - void (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); - void (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + int32_t (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + int32_t (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 10cd6416b4..2e3c04eb78 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -72,7 +72,7 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { return code; } -void mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { SMnode *pMnode = pFsm->data; SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSdbRaw *pRaw = pMsg->pCont; @@ -114,12 +114,15 @@ void mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *p mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); } } + + return 0; } -void mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - mndProcessWriteMsg(pFsm, pMsg, pMeta); +int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { + int32_t code = mndProcessWriteMsg(pFsm, pMsg, pMeta); rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; + return code; } int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index bd8b00edf0..55ba0d3e6f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -295,7 +295,7 @@ static int32_t vnodeSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot) return 0; } -static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { SVnode *pVnode = pFsm->data; pMsg->info.conn.applyIndex = pMeta->index; pMsg->info.conn.applyTerm = pMeta->term; @@ -306,17 +306,18 @@ static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbM pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, pMsg->info.conn.applyIndex, pMeta->isWeak, pMeta->code, pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); - tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); + return tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); } -static void vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - vnodeSyncApplyMsg(pFsm, pMsg, pMeta); +static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { + return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); } -static void vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { if (pMeta->isWeak == 1) { - vnodeSyncApplyMsg(pFsm, pMsg, pMeta); + return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); } + return 0; } static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index fcb0d63209..d5d77274a9 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -457,9 +457,9 @@ int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, Syn cbMeta.flag = -1; (void)syncRespMgrGetAndDel(pNode->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); - pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); + int32_t code = pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); ASSERT(rpcMsg.pCont == NULL); - return 0; + return code; } int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf) { @@ -516,8 +516,9 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm } if (syncLogFsmExecute(pNode, pFsm, role, term, pEntry) != 0) { - sError("vgId:%d, failed to execute sync log entry in FSM. log index:%" PRId64 ", term:%" PRId64 "", vgId, - pEntry->index, pEntry->term); + sError("vgId:%d, failed to commit sync log entry. index:%" PRId64 ", term:%" PRId64 + ", role: %d, current term: %" PRId64, + vgId, pEntry->index, pEntry->term, role, term); goto _out; } pBuf->commitIndex = index; From 81dc25819d6480199c915f13587c5f70415b1e4a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 3 Dec 2022 09:42:58 +0800 Subject: [PATCH 67/95] feat(api): add int64 affect row fix build error --- source/libs/qworker/src/qworker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index fe2acb805b..e4c3c95b9c 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -327,7 +327,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, } if (0 == ctx->level) { - QW_TASK_DLOG("task fetched blocks %d rows %d, level %d", pOutput->numOfBlocks, pOutput->numOfRows, ctx->level); + QW_TASK_DLOG("task fetched blocks %d rows %"PRId64", level %d", pOutput->numOfBlocks, pOutput->numOfRows, ctx->level); break; } From 6cb65fabdc907105f7477325f0d516417236c5bb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 3 Dec 2022 09:43:37 +0800 Subject: [PATCH 68/95] fix: fix stable not exist crash issue --- source/libs/catalog/src/ctgCache.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 4626d4527d..6d28420f79 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -439,6 +439,12 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** ctx->tbInfo.tbType, dbFName); ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + taosMemoryFreeClear(*pTableMeta); + ctgDebug("stb 0x%" PRIx64 " meta not in cache", ctx->tbInfo.suid); + return TSDB_CODE_SUCCESS; + } STableMeta *stbMeta = tbCache->pMeta; if (stbMeta->suid != ctx->tbInfo.suid) { From d9c261cd13a5d34300b195ae6afccdbd1a96d540 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 3 Dec 2022 10:03:18 +0800 Subject: [PATCH 69/95] refact: remove some error codes --- include/util/taoserror.h | 56 +++++++++++---- source/client/src/clientEnv.c | 6 +- source/client/src/clientHb.c | 20 +++--- source/client/src/clientImpl.c | 18 ++--- source/client/src/clientMsgHandler.c | 2 +- source/client/src/clientRawBlockWrite.c | 16 ++--- source/client/src/clientSml.c | 10 +-- source/client/src/clientStmt.c | 28 ++++---- source/common/src/tdataformat.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 4 +- source/dnode/mnode/sdb/src/sdbFile.c | 2 +- source/dnode/mnode/sdb/src/sdbHash.c | 4 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +- source/libs/catalog/src/catalog.c | 2 +- source/libs/catalog/src/ctgRemote.c | 4 +- source/libs/command/src/command.c | 2 +- source/libs/command/src/explain.c | 50 ++++++------- source/libs/executor/src/dataDeleter.c | 4 +- source/libs/executor/src/dataDispatcher.c | 12 ++-- source/libs/executor/src/dataInserter.c | 10 +-- source/libs/executor/src/exchangeoperator.c | 8 +-- source/libs/executor/src/executil.c | 4 +- source/libs/executor/src/executor.c | 20 +++--- source/libs/executor/src/executorimpl.c | 18 ++--- source/libs/executor/src/groupoperator.c | 4 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/sysscanoperator.c | 4 +- source/libs/executor/src/timewindowoperator.c | 42 +++++------ source/libs/executor/src/tsort.c | 2 +- source/libs/function/src/udfd.c | 2 +- source/libs/index/src/indexFilter.c | 28 ++++---- source/libs/index/src/indexTfile.c | 2 +- source/libs/nodes/src/nodesToSQLFuncs.c | 4 +- source/libs/nodes/src/nodesUtilFuncs.c | 2 +- source/libs/parser/src/parInsertSml.c | 6 +- source/libs/parser/src/parInsertSql.c | 6 +- source/libs/parser/src/parInsertStmt.c | 4 +- source/libs/parser/src/parInsertUtil.c | 18 ++--- source/libs/parser/src/parTranslater.c | 2 +- source/libs/parser/src/parUtil.c | 2 +- .../libs/parser/test/mockCatalogService.cpp | 2 +- source/libs/qcom/src/queryUtil.c | 8 +-- source/libs/qcom/src/querymsg.c | 8 +-- source/libs/qworker/src/qwDbg.c | 18 ++--- source/libs/qworker/src/qwMsg.c | 28 ++++---- source/libs/qworker/src/qwUtil.c | 14 ++-- source/libs/qworker/src/qworker.c | 22 +++--- source/libs/scalar/src/filter.c | 30 ++++---- source/libs/scalar/src/scalar.c | 46 ++++++------ source/libs/scalar/src/sclvector.c | 8 +-- source/libs/scheduler/src/schFlowCtrl.c | 8 +-- source/libs/scheduler/src/schJob.c | 52 +++++++------- source/libs/scheduler/src/schRemote.c | 70 +++++++++---------- source/libs/scheduler/src/schTask.c | 24 +++---- source/libs/scheduler/src/schUtil.c | 2 +- source/libs/scheduler/src/scheduler.c | 6 +- source/libs/sync/src/syncRaftLog.c | 2 +- source/libs/wal/src/walMeta.c | 8 +-- source/libs/wal/src/walMgmt.c | 2 +- source/libs/wal/src/walRead.c | 8 +-- source/util/src/terror.c | 12 ---- tests/parallel_test/cases.task | 1 - 65 files changed, 421 insertions(+), 402 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 490dbf7744..e8e06f4af1 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -126,11 +126,11 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_DB_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0209) #define TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH TAOS_DEF_ERROR_CODE(0, 0x020A) #define TSDB_CODE_TSC_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x020B) -#define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) +// #define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) // 2.x #define TSDB_CODE_TSC_QUERY_CACHE_ERASED TAOS_DEF_ERROR_CODE(0, 0x020E) #define TSDB_CODE_TSC_QUERY_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x020F) #define TSDB_CODE_TSC_SORTED_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0210) -#define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) +// #define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) // 2.x #define TSDB_CODE_TSC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0212) #define TSDB_CODE_TSC_DISCONNECTED TAOS_DEF_ERROR_CODE(0, 0x0213) #define TSDB_CODE_TSC_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0214) @@ -158,9 +158,17 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022F) // mnode-common +// #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) // 2.x +// #define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0301) // 2.x +// #define TSDB_CODE_MND_ACTION_NEED_REPROCESSEDTAOS_DEF_ERROR_CODE(0, 0x0302) // 2.x #define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) -#define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0304) +// #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0304) // 2.x #define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0305) +// #define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0306) // 2.x +// #define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0307) // 2.x +// #define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0308) // 2.x +// #define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x0309) // 2.x +// #define TSDB_CODE_MND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x030A) // 2.x #define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x030B) #define TSDB_CODE_MND_INVALID_QUERY_ID TAOS_DEF_ERROR_CODE(0, 0x030C) #define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x030D) @@ -174,7 +182,7 @@ int32_t* taosGetErrno(); // mnode-sdb #define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) -#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) // internal +// #define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) // 2.x #define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0322) #define TSDB_CODE_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0323) #define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0325) @@ -217,22 +225,31 @@ int32_t* taosGetErrno(); // mnode-stable-part1 #define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) +// #define TSDB_CODE_MND_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0361) // 2.x #define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0362) +// #define TSDB_CODE_MND_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0363) // 2.x #define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x0364) #define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x0365) +// #define TSDB_CODE_MND_TOO_MANY_TIMESERIES TAOS_DEF_ERROR_CODE(0, 0x0366) // 2.x +// #define TSDB_CODE_MND_NOT_SUPER_TABLE TAOS_DEF_ERROR_CODE(0, 0x0367) // 2.x +// #define TSDB_CODE_MND_COL_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x0368) // 2.x #define TSDB_CODE_MND_TAG_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0369) #define TSDB_CODE_MND_TAG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036A) #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x036B) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036C) +// #define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) // 2.x #define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x036E) #define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) // mnode-func #define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0370) +// #define TSDB_CODE_MND_INVALID_FUNC_LEN TAOS_DEF_ERROR_CODE(0, 0x0371) // 2.x #define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x0372) #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0373) #define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0374) #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) +// #define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) // 2.x +// #define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) // 2.x #define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x0378) #define TSDB_CODE_MND_INVALID_FUNC_RETRIEVE TAOS_DEF_ERROR_CODE(0, 0x0379) @@ -390,8 +407,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) #define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) #define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) -#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) -#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) +// #define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) // 2.x +// #define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) // 2.x #define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060A) #define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060B) #define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060C) @@ -405,6 +422,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) +// #define TSDB_CODE_TDB_INCOMPLETE_DFILESET TAOS_DEF_ERROR_CODE(0, 0x0617) // 2.x #define TSDB_CODE_TDB_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0618) #define TSDB_CODE_TDB_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0619) #define TSDB_CODE_TDB_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x061A) @@ -415,8 +433,9 @@ int32_t* taosGetErrno(); // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) #define TSDB_CODE_QRY_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0701) -#define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) -#define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) +// #define TSDB_CODE_QRY_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0702) // 2.x +// #define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) // 2.x +// #define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) // 2.x #define TSDB_CODE_QRY_DUP_JOIN_KEY TAOS_DEF_ERROR_CODE(0, 0x0705) #define TSDB_CODE_QRY_EXCEED_TAGS_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0706) #define TSDB_CODE_QRY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0707) @@ -428,6 +447,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070D) #define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070E) #define TSDB_CODE_QRY_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x070F) +// #define TSDB_CODE_QRY_INVALID_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0710) // 2.x +// #define TSDB_CODE_QRY_RESULT_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0711) // 2.x #define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0720) #define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0721) #define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0722) @@ -463,7 +484,17 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_TABLE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080D) // sync +// #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x +// #define TSDB_CODE_SYN_NOT_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0901) // 2.x +// #define TSDB_CODE_SYN_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0902) // 2.x #define TSDB_CODE_SYN_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0903) +// #define TSDB_CODE_SYN_TOO_MANY_FWDINFO TAOS_DEF_ERROR_CODE(0, 0x0904) // 2.x +// #define TSDB_CODE_SYN_MISMATCHED_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0905) // 2.x +// #define TSDB_CODE_SYN_MISMATCHED_CLUSTERID TAOS_DEF_ERROR_CODE(0, 0x0906) // 2.x +// #define TSDB_CODE_SYN_MISMATCHED_SIGNATURE TAOS_DEF_ERROR_CODE(0, 0x0907) // 2.x +// #define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) // 2.x +// #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) // 2.x +// #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) // 2.x #define TSDB_CODE_SYN_IS_LEADER TAOS_DEF_ERROR_CODE(0, 0x090B) #define TSDB_CODE_SYN_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x090C) #define TSDB_CODE_SYN_ONE_REPLICA TAOS_DEF_ERROR_CODE(0, 0x090D) @@ -481,7 +512,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) #define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) #define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) -#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) +// #define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) #define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) #define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) #define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) @@ -492,16 +523,17 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TQ_NO_COMMITTED_OFFSET TAOS_DEF_ERROR_CODE(0, 0x0A0D) // wal -#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) +// #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) // 2.x #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) #define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) #define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) -#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) +// #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) // 2.x #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) #define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006) #define TSDB_CODE_WAL_LOG_INCOMPLETE TAOS_DEF_ERROR_CODE(0, 0x1007) // tfs +// #define TSDB_CODE_FS_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x2200) // 2.x #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) #define TSDB_CODE_FS_TOO_MANY_MOUNT TAOS_DEF_ERROR_CODE(0, 0x2202) #define TSDB_CODE_FS_DUP_PRIMARY TAOS_DEF_ERROR_CODE(0, 0x2203) @@ -510,7 +542,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FS_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2206) #define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) #define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) -#define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x220F) +// #define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x220F) // 2.x // catalog #define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 6f1414b72d..f4b18a373b 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -244,14 +244,14 @@ void destroyTscObj(void *pObj) { void *createTscObj(const char *user, const char *auth, const char *db, int32_t connType, SAppInstInfo *pAppInfo) { STscObj *pObj = (STscObj *)taosMemoryCalloc(1, sizeof(STscObj)); if (NULL == pObj) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pObj->pRequests = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); if (NULL == pObj->pRequests) { taosMemoryFree(pObj); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -281,7 +281,7 @@ int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, ri void *createRequest(uint64_t connId, int32_t type, int64_t reqid) { SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj)); if (NULL == pRequest) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6bdc835217..47ed2cf035 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -69,7 +69,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog } else { SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == vgInfo) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } @@ -82,7 +82,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (NULL == vgInfo->vgHash) { taosMemoryFree(vgInfo); tscError("hash init[%d] failed", rsp->vgNum); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } @@ -92,7 +92,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog tscError("hash push failed, errno:%d", errno); taosHashCleanup(vgInfo->vgHash); taosMemoryFree(vgInfo); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } } @@ -366,7 +366,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { desc.subDesc = taosArrayInit(desc.subPlanNum, sizeof(SQuerySubDesc)); if (NULL == desc.subDesc) { releaseRequest(*rid); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } code = schedulerGetTasksStatus(pRequest->body.queryJob, desc.subDesc); @@ -394,14 +394,14 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); if (NULL == pTscObj) { tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SQueryHbReqBasic *hbBasic = (SQueryHbReqBasic *)taosMemoryCalloc(1, sizeof(SQueryHbReqBasic)); if (NULL == hbBasic) { tscError("calloc %d failed", (int32_t)sizeof(SQueryHbReqBasic)); releaseTscObj(connKey->tscRid); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } hbBasic->connId = pTscObj->connId; @@ -419,7 +419,7 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { tscWarn("taosArrayInit %d queryDesc failed", numOfQueries); releaseTscObj(connKey->tscRid); taosMemoryFree(hbBasic); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = hbBuildQueryDesc(hbBasic, pTscObj); @@ -613,7 +613,7 @@ static FORCE_INLINE void hbMgrInitHandle() { SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { SClientHbBatchReq *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); @@ -737,7 +737,7 @@ static void *hbThreadFunc(void *param) { int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); taosArrayPush(mgr, &pAppHbMgr); @@ -748,7 +748,7 @@ static void *hbThreadFunc(void *param) { SMsgSendInfo *pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 846accf51b..920bac0e2b 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -166,7 +166,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, tscError("0x%" PRIx64 " failed to prepare sql string buffer, %s", (*pRequest)->self, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } strntolower((*pRequest)->sqlstr, sql, (int32_t)sqlLen); @@ -179,7 +179,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, if (pParam == NULL) { destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } tsem_init(&pParam->sem, 0, 0); @@ -198,7 +198,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, taosMemoryFree(param); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } (*pRequest)->allocatorRefId = -1; @@ -209,7 +209,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -609,7 +609,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } default: tscError("unknown query policy: %d", tsQueryPolicy); - return TSDB_CODE_TSC_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosArrayDestroy(pDbVgList); @@ -670,7 +670,7 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* } default: tscError("unknown query policy: %d", tsQueryPolicy); - return TSDB_CODE_TSC_APP_ERROR; + return TSDB_CODE_APP_ERROR; } _return: @@ -1136,7 +1136,7 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { int32_t tblNum = taosArrayGetSize(pRequest->tableList); if (dbNum <= 0 && tblNum <= 0) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); @@ -1231,7 +1231,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t SAppInstInfo* pAppInfo, int connType) { STscObj* pTscObj = createTscObj(user, auth, db, connType, pAppInfo); if (NULL == pTscObj) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return pTscObj; } @@ -1268,7 +1268,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 4bd74a842f..7d0d9574f6 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -86,7 +86,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { /*assert(connectRsp.epSet.numOfEps > 0);*/ if (connectRsp.epSet.numOfEps == 0) { - setErrno(pRequest, TSDB_CODE_MND_APP_ERROR); + setErrno(pRequest, TSDB_CODE_APP_ERROR); tsem_post(&pRequest->body.rspSem); goto End; } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 5583c8ae00..150194aa27 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1388,7 +1388,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } dst->vg = vgData; @@ -1579,7 +1579,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } dst->vg = vgData; @@ -1726,7 +1726,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; void* tmp = taosMemoryRealloc(vgData.data, totalLen); if (tmp == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } vgData.data = tmp; @@ -1737,7 +1737,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t totalLen = sizeof(SSubmitReq) + submitLen; void* tmp = taosMemoryCalloc(1, totalLen); if (tmp == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } vgData.data = tmp; @@ -1840,7 +1840,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { while (vData) { SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } dst->vg = vData->vg; @@ -2029,7 +2029,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; void* tmp = taosMemoryRealloc(vgData.data, totalLen); if (tmp == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } vgData.data = tmp; @@ -2040,7 +2040,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) int32_t totalLen = sizeof(SSubmitReq) + submitLen; void* tmp = taosMemoryCalloc(1, totalLen); if (tmp == NULL) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } vgData.data = tmp; @@ -2146,7 +2146,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) while (vData) { SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } dst->vg = vData->vg; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 719a6f82d2..a4e943da32 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1417,7 +1417,7 @@ static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *col SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (!kvHash) { uError("SML:smlDealCols failed to allocate memory"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); @@ -2092,7 +2092,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int l cols = taosArrayInit(16, POINTER_BYTES); if (cols == NULL) { uError("SML:0x%" PRIx64 " smlParseInfluxLine failed to allocate memory", info->id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } else { // if dataFormat is false, cols do not need to save data, there is another new memory to save data cols = info->colsContainer; @@ -2121,7 +2121,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int l if (!tinfo) { smlDestroyCols(cols); if (info->dataFormat) taosArrayDestroy(cols); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); oneTable = &tinfo; @@ -2192,13 +2192,13 @@ static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { int ret = TSDB_CODE_SUCCESS; SSmlTableInfo *tinfo = smlBuildTableInfo(); if (!tinfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SArray *cols = taosArrayInit(16, POINTER_BYTES); if (cols == NULL) { uError("SML:0x%" PRIx64 " smlParseTelnetLine failed to allocate memory", info->id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 291e0f6ae3..82ea9e0d8f 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -83,7 +83,7 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { } break; default: - code = TSDB_CODE_TSC_APP_ERROR; + code = TSDB_CODE_APP_ERROR; break; } @@ -118,7 +118,7 @@ int32_t stmtBackupQueryFields(STscStmt* pStmt) { pRes->fields = taosMemoryMalloc(size); pRes->userFields = taosMemoryMalloc(size); if (NULL == pRes->fields || NULL == pRes->userFields) { - STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pRes->fields, pStmt->exec.pRequest->body.resInfo.fields, size); memcpy(pRes->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size); @@ -136,7 +136,7 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { pStmt->exec.pRequest->body.resInfo.fields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { - STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pStmt->exec.pRequest->body.resInfo.fields, pRes->fields, size); } @@ -144,7 +144,7 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { pStmt->exec.pRequest->body.resInfo.userFields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { - STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pStmt->exec.pRequest->body.resInfo.userFields, pRes->userFields, size); } @@ -463,7 +463,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { tscError("table [%s, %" PRIx64 ", %" PRIx64 "] found in exec blockHash, but not in sql blockHash", pStmt->bInfo.tbFName, uid, cacheUid); - STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } pStmt->bInfo.needParse = false; @@ -512,7 +512,7 @@ int32_t stmtResetStmt(STscStmt* pStmt) { pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; STMT_ERR_RET(terrno); } @@ -527,13 +527,13 @@ TAOS_STMT* stmtInit(STscObj* taos, int64_t reqid) { pStmt = taosMemoryCalloc(1, sizeof(STscStmt)); if (NULL == pStmt) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pStmt); return NULL; } @@ -618,7 +618,7 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } tscDebug("start to bind stmt tag values"); @@ -641,7 +641,7 @@ int stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } STMT_ERR_RET(qBuildStmtTagFields(*pDataBlock, pStmt->bInfo.boundTags, fieldNum, fields)); @@ -659,7 +659,7 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } STMT_ERR_RET(qBuildStmtColFields(*pDataBlock, fieldNum, fields)); @@ -733,7 +733,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } if (colIdx < 0) { @@ -745,7 +745,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { } else { if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) { tscError("bind column index not in sequence"); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } pStmt->bInfo.sBindLastIdx = colIdx; @@ -897,7 +897,7 @@ _return: if (TSDB_CODE_SUCCESS == code && autoCreateTbl) { if (NULL == pRsp) { tscError("no submit resp got for auto create table"); - code = TSDB_CODE_TSC_APP_ERROR; + code = TSDB_CODE_APP_ERROR; } else { code = stmtUpdateTableUid(pStmt, pRsp); } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 728f669fc8..fc49c03379 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -518,7 +518,7 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { SRowIter *pIter = taosMemoryCalloc(1, sizeof(*pIter)); if (pIter == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d42b66455f..fe76ba31d7 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -542,7 +542,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } default: mError("invalid kv key:%d", kv->key); - hbRsp.status = TSDB_CODE_MND_APP_ERROR; + hbRsp.status = TSDB_CODE_APP_ERROR; break; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 161bf7536f..b4ea8ee125 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -276,7 +276,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg } if (createReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return NULL; } @@ -376,7 +376,7 @@ static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *p } if (alterReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return NULL; } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6558a98aaa..318a746bb0 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -415,7 +415,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { break; } } else { - code = TSDB_CODE_SDB_APP_ERROR; + code = TSDB_CODE_APP_ERROR; taosHashCancelIterate(hash, ppRow); break; } diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index c44b659ef5..32b34ea3a3 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -108,7 +108,7 @@ static SHashObj *sdbGetHash(SSdb *pSdb, int32_t type) { SHashObj *hash = pSdb->hashObjs[type]; if (hash == NULL) { - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return NULL; } @@ -302,7 +302,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { terrno = TSDB_CODE_SDB_OBJ_DROPPING; break; default: - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; break; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 003b387838..191fc1b49c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -75,7 +75,7 @@ static void tqPushEntryFree(void* data) { STQ* tqOpen(const char* path, SVnode* pVnode) { STQ* pTq = taosMemoryCalloc(1, sizeof(STQ)); if (pTq == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pTq->path = strdup(path); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d9f0c55d24..8211e58279 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1311,7 +1311,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte char* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num); if (buf == NULL) { cleanupBlockOrderSupporter(&sup); - return TSDB_CODE_TDB_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; @@ -1354,7 +1354,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); if (ret != TSDB_CODE_SUCCESS) { cleanupBlockOrderSupporter(&sup); - return TSDB_CODE_TDB_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t numOfTotal = 0; @@ -3812,7 +3812,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL tsdbReaderClose(p); *ppReader = NULL; - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 4e0c0e90d1..55703002b8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -107,7 +107,7 @@ int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItem SArray *pArray = taosArrayInit(pMapData->nItem, itemSize); if (pArray == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3a398d1551..1d74957718 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -619,7 +619,7 @@ int32_t catalogInit(SCatalogCfg* cfg) { gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == gCtgMgmt.queue.head) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); - CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } gCtgMgmt.queue.tail = gCtgMgmt.queue.head; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 753c4ce917..7cc6c90d30 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -384,13 +384,13 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, SArr SMsgSendInfo* msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == msgSendInfo) { qError("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - CTG_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCtgTaskCallbackParam* param = taosMemoryCalloc(1, sizeof(SCtgTaskCallbackParam)); if (NULL == param) { qError("calloc %d failed", (int32_t)sizeof(SCtgTaskCallbackParam)); - CTG_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } param->reqType = msgType; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index d58c4dc6d3..6713d0cbe9 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -497,7 +497,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); char* buf2 = taosMemoryMalloc(SHOW_CREATE_TB_RESULT_FIELD2_LEN); if (NULL == buf2) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 03c7249294..f9b166de04 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -77,19 +77,19 @@ int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, d SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); if (NULL == ctx) { qError("calloc SExplainCtx failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); if (NULL == rows) { qError("taosArrayInit SQueryExplainRowInfo failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE); if (NULL == tbuf) { qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } ctx->mode = mode; @@ -235,14 +235,14 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo } default: qError("not supported physical node type %d", pNode->type); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } if (pPhysiChildren) { *pChildren = nodesMakeList(); if (NULL == *pChildren) { qError("nodesMakeList failed"); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -260,7 +260,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S *pExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainExecInfo)); if (NULL == (*pExecInfo)) { qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SExplainRsp *rsp = NULL; @@ -272,7 +272,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S rsp = taosArrayGet(group->nodeExecInfo, group->nodeIdx++); if (group->physiPlanExecIdx >= rsp->numOfPlans) { qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); @@ -281,7 +281,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S rsp = taosArrayGet(group->nodeExecInfo, i); if (group->physiPlanExecIdx >= rsp->numOfPlans) { qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); @@ -297,13 +297,13 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai if (NULL == pNode) { *pResNode = NULL; qError("physical node is NULL"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SExplainResNode *resNode = taosMemoryCalloc(1, sizeof(SExplainResNode)); if (NULL == resNode) { qError("calloc SPhysiNodeExplainRes failed"); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = 0; @@ -378,7 +378,7 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t row.buf = taosMemoryMalloc(len); if (NULL == row.buf) { qError("taosMemoryMalloc %d failed", len); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(row.buf, tbuf, len); @@ -389,7 +389,7 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t if (NULL == taosArrayPush(ctx->rows, &row)) { qError("taosArrayPush row to explain res rows failed"); taosMemoryFree(row.buf); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; @@ -407,7 +407,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i SPhysiNode *pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } switch (pNode->type) { @@ -793,7 +793,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcStartGroupId, sizeof(pExchNode->srcStartGroupId)); if (NULL == group) { qError("exchange src group %d not in groupHash", pExchNode->srcStartGroupId); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } nodeNum += group->nodeNum; @@ -1585,7 +1585,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } default: qError("not supported physical node type %d", pNode->type); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -1594,7 +1594,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } int32_t code = 0; @@ -1614,7 +1614,7 @@ int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, b SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); if (NULL == group) { qError("group %d not in groupHash", groupId); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } group->singleChannel = singleChannel; @@ -1636,7 +1636,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { int32_t rowNum = taosArrayGetSize(pCtx->rows); if (rowNum <= 0) { qError("empty explain res rows"); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } SSDataBlock *pBlock = createDataBlock(); @@ -1659,7 +1659,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); blockDataDestroy(pBlock); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } rsp->completed = 1; @@ -1698,7 +1698,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (NULL == groupHash) { qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } QRY_ERR_JRET( @@ -1732,7 +1732,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -1800,7 +1800,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t if (NULL == group) { qError("group %d not in groupHash", groupId); tFreeSExplainRsp(pRspMsg); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } taosWLockLatch(&group->lock); @@ -1811,7 +1811,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } group->physiPlanExecNum = pRspMsg->numOfPlans; @@ -1821,7 +1821,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } if (group->physiPlanExecNum != pRspMsg->numOfPlans) { @@ -1830,7 +1830,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } taosArrayPush(group->nodeExecInfo, pRspMsg); diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index c7a2480204..12fb449238 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -135,12 +135,12 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; SDataDeleterBuf* pBuf = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM); if (NULL == pBuf) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } if (!allocBuf(pDeleter, pInput, pBuf)) { taosFreeQitem(pBuf); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } toDataCacheEntry(pDeleter, pInput, pBuf); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index d4248fc420..c45226e02b 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -128,12 +128,12 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM); if (NULL == pBuf) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } if (!allocBuf(pDispatcher, pInput, pBuf)) { taosFreeQitem(pBuf); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } toDataCacheEntry(pDispatcher, pInput, pBuf); @@ -237,8 +237,8 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); if (NULL == dispatcher) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } dispatcher->sink.fPut = putDataBlock; dispatcher->sink.fEndPut = endPut; @@ -254,8 +254,8 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD taosThreadMutexInit(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { taosMemoryFree(dispatcher); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } *pHandle = dispatcher; return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 09ca1d27b9..346fcc9729 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -101,7 +101,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { taosMemoryFreeClear(pMsg); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -304,8 +304,8 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat void* pParam) { SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle)); if (NULL == inserter) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SQueryInserterNode* pInserterNode = (SQueryInserterNode*)pDataSink; @@ -342,8 +342,8 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat if (NULL == inserter->pDataBlocks) { destroyDataSinker((SDataSinkHandle*)inserter); taosMemoryFree(inserter); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 76e27f12fc..3b5928a284 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -424,20 +424,20 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas int32_t msgSize = tSerializeSResFetchReq(NULL, 0, &req); if (msgSize < 0) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pWrapper); return pTaskInfo->code; } void* msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pWrapper); return pTaskInfo->code; } if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pWrapper); taosMemoryFree(msg); return pTaskInfo->code; @@ -453,7 +453,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas taosMemoryFreeClear(msg); taosMemoryFree(pWrapper); qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return pTaskInfo->code; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1241bdf5a4..93c4372586 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1457,7 +1457,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu SqlFunctionCtx* p = NULL; SqlFunctionCtx** pValCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES); if (pValCtx == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < numOfOutput; ++i) { @@ -1605,7 +1605,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)*pCond->numOfCols); if (pCond->colList == NULL || pCond->pSlotList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFreeClear(pCond->colList); taosMemoryFreeClear(pCond->pSlotList); return terrno; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 4ad81f609a..6fca2858dc 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -35,12 +35,12 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { // not handle this in join query qError("join not supported for stream block scan, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pOperator->status = OP_NOT_OPENED; return doSetSMABlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); @@ -76,12 +76,12 @@ static int32_t doSetStreamOpOpen(SOperatorInfo* pOperator, char* id) { if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { // not handle this in join query qError("join not supported for stream block scan, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pOperator->status = OP_NOT_OPENED; return doSetStreamOpOpen(pOperator->pDownstream[0], id); @@ -95,12 +95,12 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { // not handle this in join query qError("join not supported for stream block scan, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pOperator->status = OP_NOT_OPENED; return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); @@ -139,7 +139,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; @@ -156,7 +156,7 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pBlocks == NULL || numOfBlocks == 0) { @@ -177,7 +177,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pBlocks == NULL || numOfBlocks == 0) { @@ -980,7 +980,7 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s if (pCond->colList == NULL || pCond->pSlotList == NULL) { taosMemoryFreeClear(pCond->colList); taosMemoryFreeClear(pCond->pSlotList); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b408e32061..bd22e864cd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -675,7 +675,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY); } } } else if (pQueryAttr->stableQuery && (!pQueryAttr->tsCompQuery) && (!pQueryAttr->diffQuery)) { // stable aggregate, not interval aggregate or normal column aggregate @@ -726,7 +726,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY); } } } @@ -2196,12 +2196,12 @@ static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanI if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { qError("join not supported for stream block scan"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return extractTbscanInStreamOpTree(pOperator->pDownstream[0], ppInfo); } else { @@ -2219,13 +2219,13 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return 0; } else { ASSERT(0); - terrno = TSDB_CODE_QRY_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return -1; } } else { if (LIST_LENGTH(pNode->pChildren) != 1) { ASSERT(0); - terrno = TSDB_CODE_QRY_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return -1; } SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pNode->pChildren, 0); @@ -2253,7 +2253,7 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa if (pTableScanInfo->dataReader == NULL) { ASSERT(0); qError("failed to create data reader"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } // TODO: set uid and ts to data reader return 0; @@ -2428,7 +2428,7 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInf code = getOperatorExplainExecInfo(operatorInfo->pDownstream[i], pExecInfoList); if (code != TSDB_CODE_SUCCESS) { // taosMemoryFreeClear(*pRes); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -2445,7 +2445,7 @@ int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResul int32_t size = pAggSup->resultRowSize; if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } *pResult = (SResultRow*)value; ASSERT(*pResult); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 5d34064b7e..2cd1bd7dec 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -310,7 +310,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.id.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } int32_t rowIndex = j - num; @@ -327,7 +327,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.id.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } int32_t rowIndex = pBlock->info.rows - num; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0d245f0815..04f5f4ecfe 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2183,7 +2183,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 32905ca897..a88f673e0b 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1335,7 +1335,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -1881,7 +1881,7 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)); if (pCond->colList == NULL || pCond->pSlotList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 2d0893b97b..d9a011a892 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -648,7 +648,7 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); @@ -927,7 +927,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } TSKEY ekey = ascScan ? win.ekey : win.skey; int32_t forwardRows = @@ -943,7 +943,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } // window start key interpolation @@ -967,7 +967,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } ekey = ascScan ? nextWin.ekey : nextWin.skey; @@ -1150,7 +1150,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); @@ -1175,7 +1175,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); @@ -1855,7 +1855,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } // pInfo->numOfRows data belong to the current session window @@ -1874,7 +1874,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); @@ -2164,7 +2164,7 @@ static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, S int32_t code = setOutputBuf(pInfo->pState, &parentWin, &pCurResult, pWinRes->groupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup); if (code != TSDB_CODE_SUCCESS || pCurResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } num++; @@ -2403,7 +2403,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p int32_t code = setOutputBuf(pInfo->pState, &nextWin, &pResult, groupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } if (IS_FINAL_OP(pInfo)) { @@ -2574,7 +2574,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { for (int32_t i = 0; i < chIndex + 1 - size; i++) { SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0); if (!pChildOp) { - T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } SStreamIntervalOperatorInfo* pTmpInfo = pChildOp->info; pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; @@ -3000,7 +3000,7 @@ static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindo SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset); if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, false); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput); @@ -3109,13 +3109,13 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData pAggSup->pResultRows, pStUpdated, pStDeleted); // coverity scan error if (!winInfo.pOutputBuf) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted); saveSessionOutputBuf(pAggSup, &winInfo); @@ -3123,7 +3123,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(winInfo, pStUpdated); if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { @@ -3447,7 +3447,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0); if (!pChildOp) { - T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } taosArrayPush(pInfo->pChildren, &pChildOp); } @@ -3890,14 +3890,14 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } saveSessionOutputBuf(pAggSup, &curWin.winInfo); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { code = saveResult(curWin.winInfo, pSeUpdated); if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } @@ -4448,7 +4448,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } TSKEY ekey = ascScan ? win.ekey : win.skey; @@ -4465,7 +4465,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } // window start key interpolation @@ -4494,7 +4494,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } ekey = ascScan ? nextWin.ekey : nextWin.skey; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 37688c0b53..30911887bb 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -156,7 +156,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSource SSortSource* pSource = taosMemoryCalloc(1, sizeof(SSortSource)); if (pSource == NULL) { taosArrayDestroy(pPageIdList); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pSource->src.pBlock = pBlock; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 9d249e0831..afadf7401d 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -419,7 +419,7 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { if (connectRsp.epSet.numOfEps == 0) { - msgInfo->code = TSDB_CODE_MND_APP_ERROR; + msgInfo->code = TSDB_CODE_APP_ERROR; goto _return; } diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 72828e1daa..5a86bc8678 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -195,7 +195,7 @@ static FORCE_INLINE int32_t sifGetValueFromNode(SNode *node, char **value) { } char *tv = taosMemoryCalloc(1, valLen + 1); if (tv == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(tv, pData, valLen); @@ -259,7 +259,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pFilter); indexError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); - SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SIF_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } break; } @@ -269,7 +269,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SIFParam *res = (SIFParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { indexError("no result for node, type:%d, node:%p", nodeType(node), node); - SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SIF_ERR_RET(TSDB_CODE_APP_ERROR); } *param = *res; break; @@ -300,7 +300,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); if (NULL == paramList) { - SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SIF_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR && @@ -319,7 +319,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); // if (paramList[0].colValType == TSDB_DATA_TYPE_JSON && // ((SOperatorNode *)(node))->opType == OP_TYPE_JSON_CONTAINS) { - // return TSDB_CODE_QRY_OUT_OF_MEMORY; + // return TSDB_CODE_OUT_OF_MEMORY; //} } *params = paramList; @@ -335,7 +335,7 @@ static int32_t sifInitParamList(SIFParam **params, SNodeList *nodeList, SIFCtx * SIFParam *tParams = taosMemoryCalloc(nodeList->length, sizeof(SIFParam)); if (tParams == NULL) { indexError("failed to calloc, nodeList: %p", nodeList); - SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SIF_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SListCell *cell = nodeList->pHead; @@ -464,7 +464,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, right->colValType, left->colName, strlen(left->colName), right->condValue, strlen(right->condValue)); if (tm == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); @@ -722,7 +722,7 @@ static EDealRes sifWalkFunction(SNode *pNode, void *context) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } return DEAL_RES_CONTINUE; @@ -740,7 +740,7 @@ static EDealRes sifWalkLogic(SNode *pNode, void *context) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } return DEAL_RES_CONTINUE; @@ -756,7 +756,7 @@ static EDealRes sifWalkOper(SNode *pNode, void *context) { return DEAL_RES_ERROR; } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -807,7 +807,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { if (NULL == ctx.pRes) { indexError("index-filter failed to taosHashInit"); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx); @@ -821,7 +821,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (res == NULL) { indexError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); - SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SIF_ERR_RET(TSDB_CODE_APP_ERROR); } if (res->result != NULL) { taosArrayAddAll(pDst->result, res->result); @@ -844,7 +844,7 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { indexError("index-filter failed to taosHashInit"); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx); @@ -856,7 +856,7 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (res == NULL) { indexError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); - SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SIF_ERR_RET(TSDB_CODE_APP_ERROR); } *status = res->status; sifFreeParam(res); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index e002ff9c32..b34d05d297 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -269,7 +269,7 @@ static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { if (ret != 0) { taosArrayDestroy(offsets); indexError("failed to find target tablelist"); - return TSDB_CODE_TDB_FILE_CORRUPTED; + return TSDB_CODE_FILE_CORRUPTED; } } taosArrayDestroy(offsets); diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 9325d02886..0181da92a9 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -132,7 +132,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { char *t = nodesGetStrValueFromNode(colNode); if (NULL == t) { nodesError("fail to get str value from valueNode"); - NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + NODES_ERR_RET(TSDB_CODE_APP_ERROR); } int32_t tlen = strlen(t); @@ -229,5 +229,5 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } nodesError("nodesNodeToSQL unknown node = %s", nodesNodeName(pNode->type)); - NODES_RET(TSDB_CODE_QRY_APP_ERROR); + NODES_RET(TSDB_CODE_APP_ERROR); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index f363cd0f10..de51e9d9f3 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1563,7 +1563,7 @@ int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { pNode->datum.p = (char*)value; break; default: - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index e76ca7751d..358baa74cb 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -114,7 +114,7 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS if (!isOrdered) { pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SBoundIdxInfo* pColIdx = pColList->colIdxInfo; for (col_id_t i = 0; i < pColList->numOfBound; ++i) { @@ -150,11 +150,11 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p SMsgBuf* msg) { SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); if (!pTagArray) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } *tagName = taosArrayInit(8, TSDB_COL_NAME_LEN); if (!*tagName) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 064e376894..df7a3953f2 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -236,7 +236,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, b if (!isOrdered) { pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SBoundIdxInfo* pColIdx = pColList->colIdxInfo; for (col_id_t i = 0; i < pColList->numOfBound; ++i) { @@ -1261,7 +1261,7 @@ static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize, // do nothing, if allocate more memory failed pDataBlock->nAllocSize = nAllocSizeOld; *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -1524,7 +1524,7 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); if (NULL == tags) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index e0e191b7c9..4ed72e6c14 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -50,7 +50,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch int32_t code = TSDB_CODE_SUCCESS; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); @@ -339,7 +339,7 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pDataBlock->pTableMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pTableMeta->tableType != TSDB_CHILD_TABLE) { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 0600accd6d..73cedfeb3d 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -198,7 +198,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } dataBuf->nAllocSize = (uint32_t)defaultSize; @@ -212,7 +212,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->pData = taosMemoryMalloc(dataBuf->nAllocSize); if (dataBuf->pData == NULL) { taosMemoryFreeClear(dataBuf); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); @@ -249,7 +249,7 @@ int32_t insBuildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbR memset(pBlocks->pData + pBlocks->size, 0, pBlocks->nAllocSize - pBlocks->size); } else { pBlocks->nAllocSize -= len + pBlocks->rowSize; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -350,7 +350,7 @@ static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; @@ -518,7 +518,7 @@ static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* p if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; @@ -668,7 +668,7 @@ int32_t insMergeTableDataBlocks(SHashObj* pHashObj, SArray** pVgDataBlocks) { insDestroyBlockArrayList(pVnodeDataBlockList); taosMemoryFreeClear(dataBuf->pData); taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -721,7 +721,7 @@ int32_t insAllocateMemForSize(STableDataBlocks* pDataBlock, int32_t allSize) { } else { // do nothing, if allocate more memory failed pDataBlock->nAllocSize = nAllocSizeOld; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -938,13 +938,13 @@ int32_t insBuildOutput(SHashObj* pVgroupsHashObj, SArray* pVgDataBlocks, SArray* size_t numOfVg = taosArrayGetSize(pVgDataBlocks); *pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); if (NULL == *pDataBlocks) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (size_t i = 0; i < numOfVg; ++i) { STableDataBlocks* src = taosArrayGetP(pVgDataBlocks, i); SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bca139664d..049d18e20a 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5682,7 +5682,7 @@ static int32_t readFromFile(char* pName, int32_t* len, char** buf) { if (s != *len) { taosCloseFile(&tfile); taosMemoryFreeClear(*buf); - return TSDB_CODE_TSC_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosCloseFile(&tfile); return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index d9fc83c20f..fa091901b6 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -377,7 +377,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi int32_t valLen = (int32_t)strlen(jsonValue); char* tmp = taosMemoryCalloc(1, valLen * TSDB_NCHAR_SIZE); if (!tmp) { - retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; + retCode = TSDB_CODE_OUT_OF_MEMORY; goto end; } val.type = TSDB_DATA_TYPE_NCHAR; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index bed8f82d3a..9cc55a7cd5 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -432,7 +432,7 @@ class MockCatalogServiceImpl { int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); dst->reset((STableMeta*)taosMemoryCalloc(1, len)); if (!dst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(dst->get(), src, len); return TSDB_CODE_SUCCESS; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 6eadf80e3d..938be3fcbd 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -159,7 +159,7 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra if (NULL == pMsg) { qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); destroySendMsgInfo(pInfo); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -441,7 +441,7 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { int32_t metaSize = sizeof(STableMeta) + numOfField * sizeof(SSchema); *pDst = taosMemoryMalloc(metaSize); if (NULL == *pDst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(*pDst, pSrc, metaSize); return TSDB_CODE_SUCCESS; @@ -455,7 +455,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { *pDst = taosMemoryMalloc(sizeof(*pSrc)); if (NULL == *pDst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(*pDst, pSrc, sizeof(*pSrc)); if (pSrc->vgHash) { @@ -463,7 +463,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { HASH_ENTRY_LOCK); if (NULL == (*pDst)->vgHash) { taosMemoryFreeClear(*pDst); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SVgroupInfo* vgInfo = NULL; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index fadc39f21d..2e4b000761 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -34,7 +34,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == pOut->dbVgroup) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pOut->dbVgroup->vgVersion = usedbRsp->vgVersion; @@ -52,7 +52,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup->vgHash = taosHashInit(usedbRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == pOut->dbVgroup->vgHash) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < usedbRsp->vgNum; ++i) { @@ -61,7 +61,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { qDebug("the %dth vgroup, id %d, epNum %d, current %s port %d", i, pVgInfo->vgId, pVgInfo->epSet.numOfEps, pVgInfo->epSet.eps[pVgInfo->epSet.inUse].fqdn, pVgInfo->epSet.eps[pVgInfo->epSet.inUse].port); if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -387,7 +387,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta * STableMeta *pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == pTableMeta) { qError("calloc size[%d] failed", metaSize); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pTableMeta->vgId = isStb ? 0 : msg->vgId; diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index a122508f98..db6e5b19fb 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -28,57 +28,57 @@ int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, return TSDB_CODE_SUCCESS; } - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } switch (oriStatus) { case JOB_TASK_STATUS_NULL: if (newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_INIT) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_INIT: if (newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_FAIL) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_EXEC: if (newStatus != JOB_TASK_STATUS_PART_SUCC && newStatus != JOB_TASK_STATUS_SUCC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_DROP) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_PART_SUCC: if (newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_SUCC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_DROP) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_SUCC: if (newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_FAIL) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_FAIL: if (newStatus != JOB_TASK_STATUS_DROP) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_DROP: if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_PART_SUCC) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; default: QW_TASK_ELOG("invalid task origStatus:%s", jobTaskStatusStr(oriStatus)); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 7dc13ce5f3..d77dc2b3f3 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -17,7 +17,7 @@ int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **r (SRetrieveTableRsp *)(rpcMalloc ? rpcReallocCont(*rsp, msgSize) : taosMemoryRealloc(*rsp, msgSize)); if (NULL == pRsp) { qError("rpcMallocCont %d failed", msgSize); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } if (NULL == *rsp) { @@ -78,18 +78,18 @@ int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t c int32_t msgSize = tSerializeSQueryTableRsp(NULL, 0, &rsp); if (msgSize < 0) { qError("tSerializeSQueryTableRsp failed"); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } void *pRsp = rpcMallocCont(msgSize); if (NULL == pRsp) { qError("rpcMallocCont %d failed", msgSize); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSQueryTableRsp(pRsp, msgSize, &rsp) < 0) { qError("tSerializeSQueryTableRsp %d failed", msgSize); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg rpcRsp = { @@ -212,19 +212,19 @@ int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); if (msgSize < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = rpcMallocCont(msgSize); if (NULL == msg) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); rpcFreeCont(msg); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg pNewMsg = { @@ -250,7 +250,7 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { SQueryContinueReq *req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); if (NULL == req) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq)); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } req->header.vgId = mgmt->nodeId; @@ -291,19 +291,19 @@ int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); if (msgSize < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = rpcMallocCont(msgSize); if (NULL == msg) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); rpcFreeCont(msg); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg brokenMsg = { @@ -327,17 +327,17 @@ int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SRpcHandleInfo * int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = rpcMallocCont(msgSize); if (NULL == msg) { QW_SCH_ELOG("calloc %d failed", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); rpcFreeCont(msg); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg brokenMsg = { diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 86fd1d533c..ce3b493638 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -76,7 +76,7 @@ int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) { taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == newSch.tasksHash) { QW_SCH_ELOG("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } QW_LOCK(QW_WRITE, &mgmt->schLock); @@ -87,7 +87,7 @@ int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) { QW_SCH_ELOG("taosHashPut new sch to scheduleHash failed, errno:%d", errno); taosHashCleanup(newSch.tasksHash); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } taosHashCleanup(newSch.tasksHash); @@ -114,7 +114,7 @@ int32_t qwAcquireSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQW QW_RET(TSDB_CODE_QRY_SCH_NOT_EXIST); } else { QW_SCH_ELOG("unknown notExistOpt:%d", nOpt); - QW_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_RET(TSDB_CODE_APP_ERROR); } } @@ -171,7 +171,7 @@ int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, i } } else { QW_TASK_ELOG("taosHashPut to tasksHash failed, error:%x - %s", code, tstrerror(code)); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } QW_UNLOCK(QW_WRITE, &sch->tasksLock); @@ -251,7 +251,7 @@ int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, SQWTaskCtx **ctx) { } } else { QW_TASK_ELOG("taosHashPut to ctxHash failed, error:%x", code); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -363,7 +363,7 @@ int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { if (taosHashRemove(sch->tasksHash, id, sizeof(id))) { QW_TASK_ELOG_E("taosHashRemove task from hash failed"); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } QW_TASK_DLOG_E("task status dropped"); @@ -509,7 +509,7 @@ int32_t qwOpenRef(void) { if (gQwMgmt.qwRef < 0) { taosWUnLockLatch(&gQwMgmt.lock); qError("init qworker ref failed"); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } } taosWUnLockLatch(&gQwMgmt.lock); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9a318df324..f4fb3d91fd 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -216,7 +216,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) if (NULL == hbInfo->rsp.taskStatus) { QW_UNLOCK(QW_READ, &sch->tasksLock); QW_ELOG("taosArrayInit taskStatus failed, num:%d", taskNum); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } void *key = NULL; @@ -463,7 +463,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } default: QW_TASK_ELOG("invalid phase %s", qwPhaseStr(phase)); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } if (ctx->rspCode) { @@ -508,7 +508,7 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { if (QW_PHASE_POST_FETCH == phase) { QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase)); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } // qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code); @@ -629,7 +629,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { if (NULL == sinkHandle || NULL == pTaskInfo) { QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); @@ -1059,7 +1059,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { if (NULL == sinkHandle || NULL == pTaskInfo) { QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } ctx.taskHandle = pTaskInfo; @@ -1097,7 +1097,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorker)); atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->cfg.maxSchedulerNum = QW_DEFAULT_SCHEDULER_NUMBER; @@ -1109,20 +1109,20 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S if (NULL == mgmt->schHash) { taosMemoryFreeClear(mgmt); qError("init %d scheduler hash failed", mgmt->cfg.maxSchedulerNum); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->ctxHash = taosHashInit(mgmt->cfg.maxTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == mgmt->ctxHash) { qError("init %d task ctx hash failed", mgmt->cfg.maxTaskNum); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->timer = taosTmrInit(0, 0, 0, "qworker"); if (NULL == mgmt->timer) { qError("init timer failed, error:%s", tstrerror(terrno)); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->nodeType = nodeType; @@ -1145,7 +1145,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void *)param, mgmt->timer); if (NULL == mgmt->hbTimer) { qError("start hb timer failed"); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } *qWorkerMgmt = mgmt; @@ -1288,7 +1288,7 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64 if (NULL == sinkHandle || NULL == pTaskInfo) { QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } ctx->level = plan->level; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 1cf4bd9923..59e39e3f6f 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -833,7 +833,7 @@ int32_t filterGetRangeRes(void *h, SFilterRange *ra) { if (num == 0) { qError("no range result"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -1053,12 +1053,12 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *fid) { if (node == NULL) { fltError("empty node"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) { - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } int32_t type; @@ -1125,7 +1125,7 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, int32_t paramNum = scalarGetOperatorParamNum(optr); if (1 != paramNum) { fltError("invalid right field in unit, operator:%s, rightType:%d", operatorTypeStr(optr), u->right.type); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } } @@ -1208,7 +1208,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { } else { void *data = taosMemoryCalloc(1, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); // reserved space for simple_copy if (NULL == data) { - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(data, nodesGetValueFromNode(valueNode), tDataTypes[type].bytes); filterAddField(info, NULL, (void **)&data, FLD_TYPE_VALUE, &right, len, true, NULL); @@ -1515,7 +1515,7 @@ EDealRes fltTreeToGroup(SNode *pNode, void *pContext) { return DEAL_RES_IGNORE_CHILD; } - ctx->code = TSDB_CODE_QRY_APP_ERROR; + ctx->code = TSDB_CODE_APP_ERROR; fltError("invalid condition type, type:%d", node->condType); @@ -1946,7 +1946,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { FLT_ERR_RET(scalarGenerateSetFromList((void **)&fi->data, fi->desc, type)); if (fi->data == NULL) { fltError("failed to convert in param"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); @@ -2014,7 +2014,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); if (len < 0) { qError("filterInitValFieldData taosUcs4ToMbs error 1"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } varDataSetLen(newValData, len); varDataCopy(fi->data, newValData); @@ -3721,12 +3721,12 @@ int32_t fltAddValueNodeToConverList(SFltTreeStat *stat, SValueNode *pNode) { if (NULL == stat->nodeList) { stat->nodeList = taosArrayInit(10, POINTER_BYTES); if (NULL == stat->nodeList) { - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } if (NULL == taosArrayPush(stat->nodeList, &pNode)) { - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; @@ -3847,7 +3847,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { if (NULL == node->pRight) { if (scalarGetOperatorParamNum(node->opType) > 1) { fltError("invalid operator, pRight:%p, nodeType:%d, opType:%d", node->pRight, nodeType(node), node->opType); - stat->code = TSDB_CODE_QRY_APP_ERROR; + stat->code = TSDB_CODE_APP_ERROR; return DEAL_RES_ERROR; } @@ -3908,7 +3908,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { if (OP_TYPE_IN == node->opType && QUERY_NODE_NODE_LIST != nodeType(node->pRight)) { fltError("invalid IN operator node, rightType:%d", nodeType(node->pRight)); - stat->code = TSDB_CODE_QRY_APP_ERROR; + stat->code = TSDB_CODE_APP_ERROR; return DEAL_RES_ERROR; } @@ -3994,7 +3994,7 @@ int32_t fltGetDataFromSlotId(void *param, int32_t id, void **data) { if (id < 0 || id >= numOfCols || id >= taosArrayGetSize(pDataBlock)) { fltError("invalid slot id, id:%d, numOfCols:%d, arraySize:%d", id, numOfCols, (int32_t)taosArrayGetSize(pDataBlock)); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, id); @@ -4024,14 +4024,14 @@ int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pInfo, uint32_t options) int32_t code = 0; if (pNode == NULL || pInfo == NULL) { fltError("invalid param"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } if (*pInfo == NULL) { *pInfo = taosMemoryCalloc(1, sizeof(SFilterInfo)); if (NULL == *pInfo) { fltError("taosMemoryCalloc %d failed", (int32_t)sizeof(SFilterInfo)); - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 55f33b7a3e..2bb38c1678 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -83,7 +83,7 @@ int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockL SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); if (NULL == pLeft) { sclError("calloc %d failed", (int32_t)sizeof(SScalarParam)); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } pLeft->numOfRows = pb->info.rows; @@ -104,7 +104,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); if (NULL == pObj) { sclError("taosHashInit failed, size:%d", 256); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); @@ -162,7 +162,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { if (taosHashPut(pObj, buf, (size_t)len, NULL, 0)) { sclError("taosHashPut to set failed"); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } colDataDestroy(out.columnData); @@ -224,7 +224,7 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { *res = taosMemoryMalloc(pNode->node.resType.bytes); if (NULL == (*res)) { sclError("malloc %d failed", pNode->node.resType.bytes); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(*res, nodesGetValueFromNode(pNode), pNode->node.resType.bytes); @@ -362,7 +362,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t taosHashCleanup(param->pHashFilter); param->pHashFilter = NULL; sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } param->colAlloced = false; break; @@ -370,7 +370,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t case QUERY_NODE_COLUMN: { if (NULL == ctx->pBlockList) { sclError("invalid node type for constant calculating, type:%d, src:%p", nodeType(node), ctx->pBlockList); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCL_ERR_RET(TSDB_CODE_APP_ERROR); } SColumnNode *ref = (SColumnNode *)node; @@ -417,7 +417,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { sclError("no result for node, type:%d, node:%p", nodeType(node), node); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCL_ERR_RET(TSDB_CODE_APP_ERROR); } *param = *res; param->colAlloced = false; @@ -459,7 +459,7 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList *pParamList, SScalarC SScalarParam *paramList = taosMemoryCalloc(*paramNum, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)((*paramNum) * sizeof(SScalarParam))); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (pParamList) { @@ -548,7 +548,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal SScalarParam *paramList = taosMemoryCalloc(paramNum, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)(paramNum * sizeof(SScalarParam))); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } sclSetOperatorValueType(node, ctx); @@ -987,7 +987,7 @@ EDealRes sclRewriteNullInOptr(SNode **pNode, SScalarCtx *ctx, EOperatorType opTy SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -999,7 +999,7 @@ EDealRes sclRewriteNullInOptr(SNode **pNode, SScalarCtx *ctx, EOperatorType opTy SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1126,7 +1126,7 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1178,7 +1178,7 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1218,7 +1218,7 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1270,7 +1270,7 @@ EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1329,7 +1329,7 @@ EDealRes sclWalkFunction(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1346,7 +1346,7 @@ EDealRes sclWalkLogic(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1364,7 +1364,7 @@ EDealRes sclWalkOperator(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1412,7 +1412,7 @@ EDealRes sclWalkTarget(SNode *pNode, SScalarCtx *ctx) { SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); if (NULL == res) { sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); - ctx->code = TSDB_CODE_QRY_APP_ERROR; + ctx->code = TSDB_CODE_APP_ERROR; return DEAL_RES_ERROR; } @@ -1434,7 +1434,7 @@ EDealRes sclWalkCaseWhen(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1485,7 +1485,7 @@ int32_t sclCalcConstants(SNode *pNode, bool dual, SNode **pRes) { ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); @@ -1603,7 +1603,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); @@ -1613,7 +1613,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (NULL == res) { sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); - SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCL_ERR_JRET(TSDB_CODE_APP_ERROR); } if (1 == res->numOfRows) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 6074c72424..a1995bdf50 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -395,7 +395,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { func = varToTimestamp; } else { sclError("invalid convert outType:%d", pCtx->outType); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pCtx->pOut->numOfRows = pCtx->pIn->numOfRows; @@ -440,7 +440,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { if (len < 0) { sclError("castConvert taosUcs4ToMbs error 1"); taosMemoryFreeClear(tmp); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } tmp[len] = 0; @@ -642,7 +642,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { } } else { sclError("not supported input type:%d", pCtx->inType); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -859,7 +859,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } default: sclError("invalid convert output type:%d", cCtx.outType); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 8980d08e89..5e4fe4b8a1 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -63,7 +63,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { taosHashInit(pJob->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == pJob->flowCtrl) { SCH_JOB_ELOG("taosHashInit %d flowCtrl failed", pJob->taskNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_SET_JOB_NEED_FLOW_CTRL(pJob); @@ -122,7 +122,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { } SCH_TASK_ELOG("taosHashPut flowCtrl failed, size:%d", (int32_t)sizeof(nctrl)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, @@ -156,13 +156,13 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { ctrl->taskList = taosArrayInit(pLevel->taskNum, POINTER_BYTES); if (NULL == ctrl->taskList) { SCH_TASK_ELOG("taosArrayInit taskList failed, size:%d", (int32_t)pLevel->taskNum); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } } if (NULL == taosArrayPush(ctrl->taskList, &pTask)) { SCH_TASK_ELOG("taosArrayPush to taskList failed, size:%d", (int32_t)taosArrayGetSize(ctrl->taskList)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } *enough = false; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 0eb29a3667..af2aeb6461 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -89,34 +89,34 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { switch (oriStatus) { case JOB_TASK_STATUS_NULL: if (newStatus != JOB_TASK_STATUS_INIT) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_INIT: if (newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_DROP) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_EXEC: if (newStatus != JOB_TASK_STATUS_PART_SUCC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_DROP) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_PART_SUCC: if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC && newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_SUCC: case JOB_TASK_STATUS_FAIL: if (newStatus != JOB_TASK_STATUS_DROP) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; @@ -125,7 +125,7 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { default: SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus)); - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } if (oriStatus != atomic_val_compare_exchange_8(&pJob->status, oriStatus, newStatus)) { @@ -168,7 +168,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { pTask->children = taosArrayInit(childNum, POINTER_BYTES); if (NULL == pTask->children) { SCH_TASK_ELOG("taosArrayInit %d children failed", childNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -182,7 +182,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { if (NULL == taosArrayPush(pTask->children, childTask)) { SCH_TASK_ELOG("taosArrayPush childTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("children info, the %d child TID 0x%" PRIx64, n, (*childTask)->taskId); @@ -197,7 +197,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { pTask->parents = taosArrayInit(parentNum, POINTER_BYTES); if (NULL == pTask->parents) { SCH_TASK_ELOG("taosArrayInit %d parents failed", parentNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } else { if (0 != pLevel->level) { @@ -216,7 +216,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { if (NULL == taosArrayPush(pTask->parents, parentTask)) { SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("parents info, the %d parent TID 0x%" PRIx64, n, (*parentTask)->taskId); @@ -278,13 +278,13 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { HASH_NO_LOCK); if (NULL == planToTask) { SCH_JOB_ELOG("taosHashInit %d failed", SCHEDULE_DEFAULT_MAX_TASK_NUM); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } pJob->levels = taosArrayInit(levelNum, sizeof(SSchLevel)); if (NULL == pJob->levels) { SCH_JOB_ELOG("taosArrayInit %d failed", levelNum); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pJob->levelNum = levelNum; @@ -300,7 +300,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { for (int32_t i = 0; i < levelNum; ++i) { if (NULL == taosArrayPush(pJob->levels, &level)) { SCH_JOB_ELOG("taosArrayPush level failed, level:%d", i); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pLevel = taosArrayGet(pJob->levels, i); @@ -323,7 +323,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { pLevel->subTasks = taosArrayInit(taskNum, sizeof(SSchTask)); if (NULL == pLevel->subTasks) { SCH_JOB_ELOG("taosArrayInit %d failed", taskNum); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } for (int32_t n = 0; n < taskNum; ++n) { @@ -335,7 +335,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SSchTask *pTask = taosArrayPush(pLevel->subTasks, &task); if (NULL == pTask) { SCH_TASK_ELOG("taosArrayPush task to level failed, level:%d, taskIdx:%d", pLevel->level, n); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schInitTask(pJob, pTask, plan, pLevel)); @@ -344,12 +344,12 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &pTask, POINTER_BYTES)) { SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } if (0 != taosHashPut(pJob->taskList, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES)) { SCH_TASK_ELOG("taosHashPut to taskList failed, taskIdx:%d", n); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } ++pJob->taskNum; @@ -702,7 +702,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pJob->attr.explainMode = pReq->pDag->explainInfo.mode; @@ -728,7 +728,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { HASH_ENTRY_LOCK); if (NULL == pJob->taskList) { SCH_JOB_ELOG("taosHashInit %d taskList failed", pReq->pDag->numOfSubplans); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schValidateAndBuildJob(pReq->pDag, pJob)); @@ -741,7 +741,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { HASH_ENTRY_LOCK); if (NULL == pJob->execTasks) { SCH_JOB_ELOG("taosHashInit %d execTasks failed", pReq->pDag->numOfSubplans); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } tsem_init(&pJob->rspSem, 0, 0); @@ -865,8 +865,8 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq if (SCH_OP_NULL != atomic_val_compare_exchange_32(&pJob->opStatus.op, SCH_OP_NULL, type)) { SCH_JOB_ELOG("job already in %s operation", schGetOpStr(pJob->opStatus.op)); SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); - SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + schDirectPostJobRes(pReq, TSDB_CODE_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); @@ -879,8 +879,8 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq if (SCH_OP_NULL != atomic_val_compare_exchange_32(&pJob->opStatus.op, SCH_OP_NULL, type)) { SCH_JOB_ELOG("job already in %s operation", schGetOpStr(pJob->opStatus.op)); SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); - SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + schDirectPostJobRes(pReq, TSDB_CODE_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); @@ -894,7 +894,7 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq if (!SCH_JOB_NEED_FETCH(pJob)) { SCH_JOB_ELOG("no need to fetch data, status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } if (status != JOB_TASK_STATUS_PART_SUCC) { @@ -911,7 +911,7 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq return TSDB_CODE_SUCCESS; default: SCH_JOB_ELOG("unknown operation type %d", type); - SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } if (schJobNeedToStop(pJob, &status)) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1a6d7df349..fa0e7ff554 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -371,7 +371,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SExplainRsp rsp = {0}; if (tDeserializeSExplainRsp(msg, msgSize, &rsp)) { tFreeSExplainRsp(&rsp); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schProcessExplainRsp(pJob, pTask, &rsp)); @@ -506,7 +506,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->queryId = pJob->queryId; @@ -523,7 +523,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo SSchHbCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->head.isHbParam = true; @@ -543,7 +543,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); if (NULL == param) { qError("calloc SSchTaskCallbackParam failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->pTrans = trans->pTrans; @@ -558,7 +558,7 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 SMsgSendInfo *msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == msgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } msgSendInfo->paramFreeFp = taosMemoryFree; @@ -621,7 +621,7 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { break; default: qError("unknown msg type for callback, msgType:%d", msgType); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } return TSDB_CODE_SUCCESS; @@ -632,7 +632,7 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { SSchHbCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->head.isHbParam = true; @@ -662,7 +662,7 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { pDst->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pDst->args) { qError("taosHashInit %d RpcCtx failed", 1); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SRpcCtxVal dst = {0}; @@ -679,7 +679,7 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { if (taosHashPut(pDst->args, msgType, sizeof(*msgType), &dst, sizeof(dst))) { qError("taosHashPut msg %d to rpcCtx failed", *msgType); (*pSrc->freeFunc)(dst.val); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pIter = taosHashIterate(pSrc->args, pIter); @@ -706,19 +706,19 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } int32_t msgType = TDMT_SCH_QUERY_HEARTBEAT_RSP; @@ -735,7 +735,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo}; if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schMakeBrokenLinkVal(pJob, pTask, &pCtx->brokenVal, true)); @@ -780,7 +780,7 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; @@ -790,7 +790,7 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SRpcCtxVal ctxVal = {.val = pExplainMsgSendInfo, .clone = schCloneSMsgSendInfo}; if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schMakeBrokenLinkVal(pJob, pTask, &pCtx->brokenVal, false)); @@ -815,7 +815,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SSchHbCallbackParam *dst = taosMemoryMalloc(sizeof(SSchHbCallbackParam)); if (NULL == dst) { qError("malloc SSchHbCallbackParam failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(dst, pSrc, sizeof(*dst)); @@ -827,7 +827,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SSchTaskCallbackParam *dst = taosMemoryMalloc(sizeof(SSchTaskCallbackParam)); if (NULL == dst) { qError("malloc SSchTaskCallbackParam failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(dst, pSrc, sizeof(*dst)); @@ -842,7 +842,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { SMsgSendInfo *pDst = taosMemoryMalloc(sizeof(*pSrc)); if (NULL == pDst) { qError("malloc SMsgSendInfo for rpcCtx failed, len:%d", (int32_t)sizeof(*pSrc)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pDst, pSrc, sizeof(*pSrc)); @@ -959,17 +959,17 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { qError("calloc hb req %d failed", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } int64_t transporterId = 0; @@ -1016,7 +1016,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(msg, pTask->msg, msgSize); @@ -1037,7 +1037,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } tSerializeSVDeleteReq(msg, msgSize, &req); @@ -1067,19 +1067,19 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSSubQueryMsg(NULL, 0, &qMsg); if (msgSize < 0) { SCH_TASK_ELOG("tSerializeSSubQueryMsg get size, msgSize:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSubQueryMsg(msg, msgSize, &qMsg) < 0) { SCH_TASK_ELOG("tSerializeSSubQueryMsg failed, msgSize:%d", msgSize); taosMemoryFree(msg); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } persistHandle = true; @@ -1098,18 +1098,18 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSResFetchReq(NULL, 0, &req); if (msgSize < 0) { SCH_TASK_ELOG("tSerializeSResFetchReq get size, msgSize:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { SCH_TASK_ELOG("tSerializeSResFetchReq %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } break; } @@ -1126,19 +1126,19 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); if (msgSize < 0) { SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); taosMemoryFree(msg); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } break; } @@ -1154,16 +1154,16 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { SCH_JOB_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_JOB_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { SCH_JOB_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } persistHandle = true; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index c8b8db367f..0f15002d9b 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -73,7 +73,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * taosHashInit(pTask->maxExecTimes, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pTask->profile.execTime = taosArrayInit(pTask->maxExecTimes, sizeof(int64_t)); if (NULL == pTask->execNodes || NULL == pTask->profile.execTime) { - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); @@ -112,7 +112,7 @@ int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad if (taosHashPut(pTask->execNodes, &execId, sizeof(execId), &nodeInfo, sizeof(nodeInfo))) { SCH_TASK_ELOG("taosHashPut nodeInfo to execNodes failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("task execNode added, execId:%d, handle:%p", execId, nodeInfo.handle); @@ -530,7 +530,7 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { } SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("task added to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); @@ -555,7 +555,7 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { } SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } *moved = true; @@ -582,7 +582,7 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { } SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } *moved = true; @@ -607,7 +607,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { } SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } *moved = true; @@ -712,7 +712,7 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId, @@ -740,13 +740,13 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { pTask->candidateAddrs = taosArrayInit(SCHEDULE_DEFAULT_MAX_NODE_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->candidateAddrs) { SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCHEDULE_DEFAULT_MAX_NODE_NUM); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (pTask->plan->execNode.epSet.numOfEps > 0) { if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("use execNode in plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps); @@ -756,7 +756,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_DATA_BIND_TASK(pTask)) { SCH_TASK_ELOG("no execNode specifed for data src task, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_ERR_RET(schSetAddrsFromNodeList(pJob, pTask)); @@ -1173,7 +1173,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { SSchTimerParam *param = taosMemoryMalloc(sizeof(SSchTimerParam)); if (NULL == param) { SCH_TASK_ELOG("taosMemoryMalloc %d failed", (int)sizeof(SSchTimerParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->rId = pJob->refId; @@ -1184,7 +1184,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { pTask->delayTimer = taosTmrStart(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer); if (NULL == pTask->delayTimer) { SCH_TASK_ELOG("start delay timer failed, handle:%p", schMgmt.timer); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index a5a15cd1c6..0582184730 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -235,7 +235,7 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { if (NULL == hb) { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_LOCK(SCH_WRITE, &hb->lock); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 91c3b5d7a1..1926c26dc9 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -39,19 +39,19 @@ int32_t schedulerInit() { schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); if (schMgmt.jobRef < 0) { qError("init schduler jobRef failed, num:%u", schMgmt.cfg.maxJobNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } schMgmt.hbConnections = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == schMgmt.hbConnections) { qError("taosHashInit hb connections failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } schMgmt.timer = taosTmrInit(0, 0, 0, "scheduler"); if (NULL == schMgmt.timer) { qError("init timer failed, error:%s", tstrerror(terrno)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (taosGetSystemUUID((char *)&schMgmt.sId, sizeof(schMgmt.sId))) { diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index cbcf111a85..f51fe344df 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -41,7 +41,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->pCache = taosLRUCacheInit(30 * 1024 * 1024, 1, .5); if (pLogStore->pCache == NULL) { taosMemoryFree(pLogStore); - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index d77acbbb6f..64dde15e76 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -105,7 +105,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { void* ptr = taosMemoryRealloc(buf, capacity); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } buf = ptr; @@ -632,7 +632,7 @@ int walRollFileInfo(SWal* pWal) { // TODO: change to emplace back SWalFileInfo* pNewInfo = taosMemoryMalloc(sizeof(SWalFileInfo)); if (pNewInfo == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } pNewInfo->firstVer = pWal->vers.lastVer + 1; @@ -664,7 +664,7 @@ char* walMetaSerialize(SWal* pWal) { if (pFiles) { cJSON_Delete(pFiles); } - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } cJSON_AddItemToObject(pRoot, "meta", pMeta); @@ -895,7 +895,7 @@ int walLoadMeta(SWal* pWal) { int size = (int)fileSize; char* buf = taosMemoryMalloc(size + 5); if (buf == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } memset(buf, 0, size + 5); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 5d0f020b02..a683ceb2b1 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -168,7 +168,7 @@ _err: } int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { - if (pWal == NULL) return TSDB_CODE_WAL_APP_ERROR; + if (pWal == NULL) return TSDB_CODE_APP_ERROR; if (pWal->cfg.level == pCfg->level && pWal->cfg.fsyncPeriod == pCfg->fsyncPeriod) { wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->cfg.vgId, pWal->cfg.level, diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1350ca0c37..76266d338b 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -48,7 +48,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { pReader->pHead = taosMemoryMalloc(sizeof(SWalCkHead)); if (pReader->pHead == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pReader); return NULL; } @@ -280,7 +280,7 @@ static int32_t walFetchBodyNew(SWalReader *pRead) { if (pRead->capacity < pReadHead->bodyLen) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } pRead->pHead = ptr; @@ -437,7 +437,7 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { if (pRead->capacity < pReadHead->bodyLen) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(*ppHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } *ppHead = ptr; @@ -546,7 +546,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReader->pHead->head.bodyLen); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; taosThreadMutexUnlock(&pReader->mutex); return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 18eeca030e..1cea9b8e19 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -109,11 +109,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_PASS_LENGTH, "Invalid password") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_DB_LENGTH, "Database name too long") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH, "Table name too long") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_CONNECTION, "Invalid connection") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_OUT_OF_MEMORY, "System out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CACHE_ERASED, "Query cache erased") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CANCELLED, "Query terminated") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SORTED_RES_TOO_MANY, "Result set too large to be sorted") // too many result for ordered super table projection query -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, "Application error") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, "Disconnected from service") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, "No write permission") @@ -142,7 +140,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NOT_STABLE_ERROR, "Table is not a super // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, "Invalid message connection") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, "Data expired") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, "Invalid query id") @@ -157,7 +154,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_DISABLED, "User is disabled") // mnode-sdb TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_ALREADY_THERE, "Object already there") -TAOS_DEFINE_ERROR(TSDB_CODE_SDB_APP_ERROR, "Unexpected generic error in sdb") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_TABLE_TYPE, "Invalid table type") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_NOT_THERE, "Object not there") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_KEY_TYPE, "Invalid key type") @@ -335,8 +331,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, "Table already exists" TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, "Invalid configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, "Tsdb init failed") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISK_PERMISSIONS, "No permission for disk files") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED, "Data file(s) corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY, "Out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE, "Tag too old") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE, "Timestamp data out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, "Submit message is messed up") @@ -360,8 +354,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE, "Table already exists // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG, "Invalid message") // failed to validate the sql expression msg by vnode -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, "System out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, "Unexpected generic error in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, "Duplicated join key") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready") @@ -426,7 +418,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_INVALID_CONFIG, "TQ invalid config") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_INIT_FAILED, "TQ init falied") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_DISK_PERMISSIONS, "TQ no disk permissions") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FILE_CORRUPTED, "TQ file corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_TQ_OUT_OF_MEMORY, "TQ out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FILE_ALREADY_EXISTS, "TQ file already exists") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FAILED_TO_CREATE_DIR, "TQ failed to create dir") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_META_NO_SUCH_KEY, "TQ meta no such key") @@ -437,11 +428,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND, "TQ table schema not f TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no commited offset") // wal -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "WAL unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_INCOMPLETE, "WAL log incomplete") @@ -455,7 +444,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_MOUNT_AT_TIER, "tfs no mount at tier" TAOS_DEFINE_ERROR(TSDB_CODE_FS_FILE_ALREADY_EXISTS, "tfs file already exists") TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, "tfs invalid level") TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, "tfs no valid disk") -TAOS_DEFINE_ERROR(TSDB_CODE_FS_APP_ERROR, "tfs out of memory") // catalog TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INTERNAL_ERROR, "catalog internal error") diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index f62c3f7f58..28ba7ea60f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -113,7 +113,6 @@ ,,y,script,./test.sh -f tsim/parser/first_last.sim ,,y,script,./test.sh -f tsim/parser/fill_stb.sim ,,y,script,./test.sh -f tsim/parser/interp.sim -#,,y,script,./test.sh -f tsim/parser/limit2.sim ,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim ,,y,script,./test.sh -f tsim/parser/function.sim ,,y,script,./test.sh -f tsim/parser/groupby-basic.sim From 658cb393a775207824a663fc87415442252620f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:11:20 +0800 Subject: [PATCH 70/95] enh(query): improve the performance and add some todo --- source/libs/executor/src/executil.c | 1 + source/libs/scalar/src/scalar.c | 5 +---- source/util/src/tpagedbuf.c | 12 ++++++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1241bdf5a4..b9681a4fde 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -129,6 +129,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in void* pData = NULL; pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES); + // todo avoid repeated malloc memory size_t keyLen = 0; int32_t iter = 0; while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 55f33b7a3e..b60b651ebc 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -165,10 +165,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - colDataDestroy(out.columnData); - taosMemoryFreeClear(out.columnData); - out.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); - + colInfoDataCleanup(out.columnData, out.numOfRows); cell = cell->pNext; } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 0f4c51060b..ad08b15c25 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -30,6 +30,7 @@ struct SDiskbasedBuf { TdFilePtr pFile; int32_t allocateId; // allocated page id char* path; // file path + char* prefix; // file name prefix int32_t pageSize; // current used page size int32_t inMemPages; // numOfPages that are allocated in memory SList* freePgList; // free page list @@ -48,6 +49,12 @@ struct SDiskbasedBuf { }; static int32_t createDiskFile(SDiskbasedBuf* pBuf) { + if (pBuf->path[0] == '\0') { // prepare the file name when needed it + char path[PATH_MAX] = {0}; + taosGetTmpfilePath(pBuf->prefix, "paged-buf", path); + pBuf->path = taosMemoryStrDup(path); + } + pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); if (pBuf->pFile == NULL) { @@ -356,10 +363,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem pPBuf->assistBuf = taosMemoryMalloc(pPBuf->pageSize + 2); // EXTRA BYTES pPBuf->all = taosHashInit(10, fn, true, false); - - char path[PATH_MAX] = {0}; - taosGetTmpfilePath(dir, "paged-buf", path); - pPBuf->path = strdup(path); + pPBuf->prefix = (char*) dir; pPBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); From b6199d8d99f9c527d027af8ffe1552e913e762ed Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:37:42 +0800 Subject: [PATCH 71/95] enh(query): update the aligned bits for colInfodata --- source/common/src/tdatablock.c | 2 +- source/os/src/osMemory.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c53e771f42..c253e842d6 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -19,7 +19,7 @@ #include "tlog.h" #include "tname.h" -#define MALLOC_ALIGN_BYTES 32 +#define MALLOC_ALIGN_BYTES 256 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 2f30e8977a..230ca5b968 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -351,7 +351,8 @@ void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { ASSERT(0); #else #if defined(LINUX) - return memalign(alignment, size); + void* p = memalign(alignment, size); + return p; #else return taosMemoryMalloc(size); #endif From 85c1e3aabf9a8e6c04eff24ea4bcaa5a1410f0f9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:41:56 +0800 Subject: [PATCH 72/95] refactor: add an assert. --- source/os/src/osMemory.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 230ca5b968..b2d1dcf58e 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -352,6 +352,8 @@ void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #else #if defined(LINUX) void* p = memalign(alignment, size); + ASSERT((((uint64_t)p) & (alignment - 1)) == 0x0); + return p; #else return taosMemoryMalloc(size); From fa43f86fe1af85ccc050c3ecb6232c548ef2a7fb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 12:43:03 +0800 Subject: [PATCH 73/95] refactor: add an assert. --- source/common/src/tdatablock.c | 3 ++- source/os/src/osMemory.c | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c253e842d6..2832e6d16e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1191,7 +1191,6 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* int32_t oldLen = BitmapLen(existedRows); pColumn->nullbitmap = tmp; memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); - ASSERT(pColumn->info.bytes); // make sure the allocated memory is MALLOC_ALIGN_BYTES aligned @@ -1207,6 +1206,8 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* } pColumn->pData = tmp; + ASSERT((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) == 0x0); + if (clearPayload) { memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); } diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index b2d1dcf58e..230ca5b968 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -352,8 +352,6 @@ void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #else #if defined(LINUX) void* p = memalign(alignment, size); - ASSERT((((uint64_t)p) & (alignment - 1)) == 0x0); - return p; #else return taosMemoryMalloc(size); From c3267d5294357bdcc2bb20c2c6eb7efe8fd0a45f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 3 Dec 2022 15:08:45 +0800 Subject: [PATCH 74/95] fix: fix db release issue --- source/libs/catalog/src/ctgCache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 6d28420f79..f41058584f 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -440,7 +440,7 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); if (NULL == tbCache) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); taosMemoryFreeClear(*pTableMeta); ctgDebug("stb 0x%" PRIx64 " meta not in cache", ctx->tbInfo.suid); return TSDB_CODE_SUCCESS; @@ -520,7 +520,7 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache); if (NULL == tbCache) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid); return TSDB_CODE_SUCCESS; } From a3200b555cc992bcd06a7d45f93e89a0d0201d72 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 17:03:31 +0800 Subject: [PATCH 75/95] fix(query): fix null ptr access. --- source/util/src/tpagedbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index ad08b15c25..37874ae250 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -49,7 +49,7 @@ struct SDiskbasedBuf { }; static int32_t createDiskFile(SDiskbasedBuf* pBuf) { - if (pBuf->path[0] == '\0') { // prepare the file name when needed it + if (pBuf->path == NULL) { // prepare the file name when needed it char path[PATH_MAX] = {0}; taosGetTmpfilePath(pBuf->prefix, "paged-buf", path); pBuf->path = taosMemoryStrDup(path); From 0a219ec88ff30f7eb87faa7bda28b8a14040fe6e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 3 Dec 2022 17:38:38 +0800 Subject: [PATCH 76/95] fix(case): taos shell modify Query OK output string --- tests/system-test/0-others/taosShell.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index e3095e8b93..bcf27800f2 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -62,7 +62,7 @@ def taos_command (buildPath, key, value, expectString, cfgDir, sqlString='', key print ('taos login success! Here can run sql, taos> ') if len(sqlString) != 0: child.sendline (sqlString) - w = child.expect(["Query OK", taosExpect.TIMEOUT, taosExpect.EOF], timeout=10) + w = child.expect(["Query OK", "Create OK", "Insert OK", "Drop OK", taosExpect.TIMEOUT, taosExpect.EOF], timeout=10) if w == 0: return "TAOS_OK" else: @@ -233,7 +233,7 @@ class TDTestCase: tdLog.printNoPrefix("================================ parameter: -s") newDbName="dbss" keyDict['s'] = "\"create database " + newDbName + "\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s fail") @@ -246,17 +246,17 @@ class TDTestCase: tdLog.exit("create db fail after taos -s %s fail"%(keyDict['s'])) keyDict['s'] = "\"create table " + newDbName + ".stb (ts timestamp, c int) tags (t int)\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s create table fail") keyDict['s'] = "\"create table " + newDbName + ".ctb0 using " + newDbName + ".stb tags (0) " + newDbName + ".ctb1 using " + newDbName + ".stb tags (1)\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s create table fail") keyDict['s'] = "\"insert into " + newDbName + ".ctb0 values('2021-04-01 08:00:00.000', 10)('2021-04-01 08:00:01.000', 20) " + newDbName + ".ctb1 values('2021-04-01 08:00:00.000', 11)('2021-04-01 08:00:01.000', 21)\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s insert data fail") @@ -401,27 +401,27 @@ class TDTestCase: tdLog.printNoPrefix("================================ parameter: -w") newDbName="dbw" keyDict['s'] = "\"create database " + newDbName + "\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w fail") keyDict['s'] = "\"create table " + newDbName + ".ntb (ts timestamp, c binary(128))\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w create table fail") keyDict['s'] = "\"insert into " + newDbName + ".ntb values('2021-04-01 08:00:00.001', 'abcd0123456789')('2021-04-01 08:00:00.002', 'abcd012345678901234567890123456789') \"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w insert data fail") keyDict['s'] = "\"insert into " + newDbName + ".ntb values('2021-04-01 08:00:00.003', 'aaaaaaaaaaaaaaaaaaaa')('2021-04-01 08:00:01.004', 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb') \"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w insert data fail") keyDict['s'] = "\"insert into " + newDbName + ".ntb values('2021-04-01 08:00:00.005', 'cccccccccccccccccccc')('2021-04-01 08:00:01.006', 'dddddddddddddddddddddddddddddddddddddddd') \"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w insert data fail") From a976d3a4c10ed6fdad29976cdb8c9b713764af49 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 3 Dec 2022 17:48:47 +0800 Subject: [PATCH 77/95] fix(case): pexpect module expect return index --- tests/system-test/0-others/taosShell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index bcf27800f2..5c7bb0443a 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -63,7 +63,7 @@ def taos_command (buildPath, key, value, expectString, cfgDir, sqlString='', key if len(sqlString) != 0: child.sendline (sqlString) w = child.expect(["Query OK", "Create OK", "Insert OK", "Drop OK", taosExpect.TIMEOUT, taosExpect.EOF], timeout=10) - if w == 0: + if w == 0 or w == 1 or w == 2: return "TAOS_OK" else: print(1) From 392fe988a0f4de1726475be1568242d28864d8ba Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 20:16:56 +0800 Subject: [PATCH 78/95] fix(query): remove invalid assert for mac&win. --- source/common/src/tdatablock.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 2832e6d16e..dfd0b68039 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1206,7 +1206,11 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* } pColumn->pData = tmp; + + // todo remove it soon +#if defined LINUX ASSERT((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) == 0x0); +#endif if (clearPayload) { memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); From a7807d3ee4a08cd98c16779b7569a632c5340c41 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 3 Dec 2022 20:53:41 +0800 Subject: [PATCH 79/95] fix: synchronize syncPropose for executing blocking msgs in vnodeProposeMsg --- include/common/tmsg.h | 4 +++ include/util/taoserror.h | 1 + source/dnode/vnode/src/vnd/vnodeSvr.c | 3 +- source/dnode/vnode/src/vnd/vnodeSync.c | 45 +++++++++++++++++++------- source/libs/sync/src/syncMain.c | 6 +++- source/libs/sync/src/syncPipeline.c | 9 ++++-- source/util/src/terror.c | 1 + 7 files changed, 54 insertions(+), 15 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0a082a37e4..f554989edc 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -66,6 +66,10 @@ extern int32_t tMsgDict[]; typedef uint16_t tmsg_t; +static inline bool vnodeIsMsgBlock(tmsg_t type) { + return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || + (type == TDMT_VND_UPDATE_TAG_VAL); +} /* ------------------------ OTHER DEFINITIONS ------------------------ */ // IE type #define TSDB_IE_TYPE_SEC 1 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 490dbf7744..e900eff3fd 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -381,6 +381,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_COL_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x0527) #define TSDB_CODE_VND_NO_AVAIL_BUFPOOL TAOS_DEF_ERROR_CODE(0, 0x0528) #define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529) +#define TSDB_CODE_VND_DUP_REQUEST TAOS_DEF_ERROR_CODE(0, 0x0530) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fc6b126cc5..ee2e8f6f6d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -181,14 +181,15 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp if (version <= pVnode->state.applied) { vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, pVnode->state.applied); + terrno = TSDB_CODE_VND_DUP_REQUEST; pRsp->info.handle = NULL; return -1; } vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); - ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); + ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); pVnode->state.applied = version; pVnode->state.applyTerm = pMsg->info.conn.applyTerm; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 20ae16ae43..1c2139c95d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -18,14 +18,15 @@ #define BATCH_ENABLE 0 -static inline bool vnodeIsMsgBlock(tmsg_t type) { - return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || - (type == TDMT_VND_UPDATE_TAG_VAL); -} - static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; } static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { + const STraceId *trace = &pMsg->info.traceId; + vGTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); + tsem_wait(&pVnode->syncSem); +} + +static inline void vnodeWaitBlockMsgOld(SVnode *pVnode, const SRpcMsg *pMsg) { if (vnodeIsMsgBlock(pMsg->msgType)) { const STraceId *trace = &pMsg->info.traceId; taosThreadMutexLock(&pVnode->lock); @@ -115,21 +116,31 @@ static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) static void inline vnodeProposeBatchMsg(SVnode *pVnode, SRpcMsg **pMsgArr, bool *pIsWeakArr, int32_t *arrSize) { if (*arrSize <= 0) return; + SRpcMsg *pLastMsg = pMsgArr[*arrSize - 1]; + taosThreadMutexLock(&pVnode->lock); int32_t code = syncProposeBatch(pVnode->sync, pMsgArr, pIsWeakArr, *arrSize); + bool wait = (code == 0 && vnodeIsBlockMsg(pLastMsg->msgType)); + if (wait) { + ASSERT(!pVnode->blocked); + pVnode->blocked = true; + } + taosThreadMutexUnlock(&pVnode->lock); + if (code > 0) { for (int32_t i = 0; i < *arrSize; ++i) { vnodeHandleWriteMsg(pVnode, pMsgArr[i]); } - } else if (code == 0) { - vnodeWaitBlockMsg(pVnode, pMsgArr[*arrSize - 1]); - } else { + } else if (code < 0) { if (terrno != 0) code = terrno; for (int32_t i = 0; i < *arrSize; ++i) { vnodeHandleProposeError(pVnode, pMsgArr[i], code); } } + if (wait) vnodeWaitBlockMsg(pVnode, pLastMsg); + pLastMsg = NULL; + for (int32_t i = 0; i < *arrSize; ++i) { SRpcMsg *pMsg = pMsgArr[i]; const STraceId *trace = &pMsg->info.traceId; @@ -206,16 +217,23 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) #else static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) { + taosThreadMutexLock(&pVnode->lock); int32_t code = syncPropose(pVnode->sync, pMsg, isWeak); + bool wait = (code == 0 && vnodeIsMsgBlock(pMsg->msgType)); + if (wait) { + ASSERT(!pVnode->blocked); + pVnode->blocked = true; + } + taosThreadMutexUnlock(&pVnode->lock); + if (code > 0) { vnodeHandleWriteMsg(pVnode, pMsg); - } else if (code == 0) { - vnodeWaitBlockMsg(pVnode, pMsg); - } else { + } else if (code < 0) { if (terrno != 0) code = terrno; vnodeHandleProposeError(pVnode, pMsg, code); } + if (wait) vnodeWaitBlockMsg(pVnode, pMsg); return code; } @@ -274,6 +292,11 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { vGTrace("vgId:%d, msg:%p get from vnode-apply queue, type:%s handle:%p index:%" PRId64, vgId, pMsg, TMSG_INFO(pMsg->msgType), pMsg->info.handle, pMsg->info.conn.applyIndex); + if (vnodeIsMsgBlock(pMsg->msgType)) { + vTrace("vgId:%d, blocking msg obtained from apply-queue. index:%" PRId64 ", term: %" PRId64 ", type: %s", vgId, + pMsg->info.conn.applyIndex, pMsg->info.conn.applyTerm, TMSG_INFO(pMsg->msgType)); + } + SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info}; if (rsp.code == 0) { if (vnodeProcessWriteMsg(pVnode, pMsg, pMsg->info.conn.applyIndex, &rsp) < 0) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index abfd8043c9..1fe2af696c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2638,7 +2638,11 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn (*pRetIndex) = index; } - return syncNodeAppend(ths, pEntry); + int32_t code = syncNodeAppend(ths, pEntry); + if (code < 0 && ths->vgId != 1 && vnodeIsMsgBlock(pEntry->originalRpcType)) { + ASSERT(false && "failed to append blocking msg"); + } + return code; } return -1; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index d5d77274a9..74a19608dc 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "syncPipeline.h" +#include "syncCommit.h" #include "syncIndexMgr.h" #include "syncInt.h" #include "syncRaftEntry.h" @@ -442,6 +443,11 @@ int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, Syn return 0; } + if (pNode->vgId != 1 && vnodeIsMsgBlock(pEntry->originalRpcType)) { + sTrace("vgId:%d, blocking msg ready to execute. index:%" PRId64 ", term: %" PRId64 ", type: %s", pNode->vgId, + pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType)); + } + SRpcMsg rpcMsg = {0}; syncEntry2OriginalRpc(pEntry, &rpcMsg); @@ -514,9 +520,8 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm } continue; } - if (syncLogFsmExecute(pNode, pFsm, role, term, pEntry) != 0) { - sError("vgId:%d, failed to commit sync log entry. index:%" PRId64 ", term:%" PRId64 + sError("vgId:%d, failed to execute sync log entry. index:%" PRId64 ", term:%" PRId64 ", role: %d, current term: %" PRId64, vgId, pEntry->index, pEntry->term, role, term); goto _out; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 18eeca030e..5d8a18fc46 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -326,6 +326,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_NOT_EXISTS, "Table column not exis TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_SUBSCRIBED, "Table column is subscribed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_AVAIL_BUFPOOL, "No availabe buffer pool") TAOS_DEFINE_ERROR(TSDB_CODE_VND_STOPPED, "Vnode stopped") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_DUP_REQUEST, "Duplicate write request") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") From 803762c4e0573e2488bea350859a6413a8af957b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 22:57:00 +0800 Subject: [PATCH 80/95] enh(query): avoid repeatly load del file. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 65 ++++++++++--------- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 3 +- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d9f0c55d24..25b6fc2a9a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -170,7 +170,9 @@ struct STsdbReader { SIOCostSummary cost; STSchema* pSchema; // the newest version schema STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times - SDataFReader* pFileReader; + SDataFReader* pFileReader; // the file reader + SDelFReader* pDelFReader; // the del file reader + SArray* pDelIdx; // del file block index; SVersionRange verRange; SBlockInfoBuf blockInfoBuf; int32_t step; @@ -2531,42 +2533,17 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* } int32_t code = 0; - STsdb* pTsdb = pReader->pTsdb; - SArray* pDelData = taosArrayInit(4, sizeof(SDelData)); SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; - if (pDelFile) { - SDelFReader* pDelFReader = NULL; - code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - SArray* aDelIdx = taosArrayInit(4, sizeof(SDelIdx)); - if (aDelIdx == NULL) { - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } - - // TODO: opt the perf of read del index - code = tsdbReadDelIdx(pDelFReader, aDelIdx); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(aDelIdx); - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } - + if (pDelFile && taosArrayGetSize(pReader->pDelIdx) > 0) { SDelIdx idx = {.suid = pReader->suid, .uid = pBlockScanInfo->uid}; - SDelIdx* pIdx = taosArraySearch(aDelIdx, &idx, tCmprDelIdx, TD_EQ); + SDelIdx* pIdx = taosArraySearch(pReader->pDelIdx, &idx, tCmprDelIdx, TD_EQ); if (pIdx != NULL) { - code = tsdbReadDelData(pDelFReader, pIdx, pDelData); + code = tsdbReadDelData(pReader->pDelFReader, pIdx, pDelData); } - taosArrayDestroy(aDelIdx); - tsdbDelFReaderClose(&pDelFReader); - if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -2662,6 +2639,27 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { } taosArrayDestroy(pIndexList); + + SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; + if (pReader->pDelFReader == NULL && pDelFile != NULL) { + int32_t code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pReader->pTsdb); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pReader->pDelIdx = taosArrayInit(4, sizeof(SDelIdx)); + if (pReader->pDelIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + code = tsdbReadDelIdx(pReader->pDelFReader, pReader->pDelIdx); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pReader->pDelIdx); + return code; + } + } + return TSDB_CODE_SUCCESS; } @@ -3912,6 +3910,15 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbDataFReaderClose(&pReader->pFileReader); } + if (pReader->pDelFReader != NULL) { + tsdbDelFReaderClose(&pReader->pDelFReader); + } + + if (pReader->pDelIdx != NULL) { + taosArrayDestroy(pReader->pDelIdx); + pReader->pDelIdx = NULL; + } + tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap, pReader->idStr); taosMemoryFree(pReader->status.uidCheckInfo.tableUidList); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 294a4bd3e4..1d09bd4936 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1477,9 +1477,8 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { } taosMemoryFree(pReader); } - *ppReader = NULL; -_exit: + *ppReader = NULL; return code; } From fca20668f56fe712b4806ce3414d3bffef2a9248 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Dec 2022 23:52:38 +0800 Subject: [PATCH 81/95] fix(query): check for 0 tables in tsdbreader --- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 25b6fc2a9a..fece5df28e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3960,6 +3960,9 @@ static bool doTsdbNextDataBlock(STsdbReader* pReader) { blockDataCleanup(pBlock); SReaderStatus* pStatus = &pReader->status; + if (taosHashGetSize(pStatus->pTableMap) == 0){ + return false; + } if (pStatus->loadFromFile) { int32_t code = buildBlockFromFiles(pReader); @@ -3977,8 +3980,6 @@ static bool doTsdbNextDataBlock(STsdbReader* pReader) { buildBlockFromBufferSequentially(pReader); return pBlock->info.rows > 0; } - - return false; } bool tsdbNextDataBlock(STsdbReader* pReader) { From 21ff35ee421dde49efe4df311d3fc9961e559251 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sun, 4 Dec 2022 08:13:49 +0800 Subject: [PATCH 82/95] fix: truncate WAL on need while syncLogBufferRollback --- source/libs/sync/src/syncPipeline.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 74a19608dc..5d6905f99e 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -984,6 +984,7 @@ int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex ", %" PRId64 ")", pNode->vgId, toIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); + // trunc buffer SyncIndex index = pBuf->endIndex - 1; while (index >= toIndex) { SSyncRaftEntry* pEntry = pBuf->entries[index % pBuf->size].pItem; @@ -997,6 +998,17 @@ int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex pBuf->endIndex = toIndex; pBuf->matchIndex = TMIN(pBuf->matchIndex, index); ASSERT(index + 1 == toIndex); + + // trunc wal + SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); + if (lastVer >= toIndex && pNode->pLogStore->syncLogTruncate(pNode->pLogStore, toIndex) < 0) { + sError("vgId:%d, failed to truncate log store since %s. from index:%" PRId64 "", pNode->vgId, terrstr(), toIndex); + return -1; + } + lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); + ASSERT(toIndex == lastVer + 1); + + syncLogBufferValidate(pBuf); return 0; } From 575c5e89fc0bd5d61567bac561e15cc3cc054846 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 4 Dec 2022 09:49:27 +0800 Subject: [PATCH 83/95] refactor(sync): add config item in sync --- include/common/tglobal.h | 13 +++++++++---- include/libs/sync/sync.h | 2 +- source/common/src/tglobal.c | 25 ++++++++++++++++++++----- source/libs/sync/src/syncMain.c | 9 ++++++--- 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 005cf36d5e..92672311d0 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -64,6 +64,11 @@ extern int32_t tsNumOfSnodeStreamThreads; extern int32_t tsNumOfSnodeWriteThreads; extern int64_t tsRpcQueueMemoryAllowed; +// sync raft +extern int32_t tsElectInterval; +extern int32_t tsHeartbeatInterval; +extern int32_t tsHeartbeatTimeout; + // monitor extern bool tsEnableMonitor; extern int32_t tsMonitorInterval; @@ -126,9 +131,9 @@ extern char tsUdfdResFuncs[]; extern char tsUdfdLdLibPath[]; // schemaless -extern char tsSmlChildTableName[]; -extern char tsSmlTagName[]; -extern bool tsSmlDataFormat; +extern char tsSmlChildTableName[]; +extern char tsSmlTagName[]; +extern bool tsSmlDataFormat; extern int32_t tsSmlBatchSize; // wal @@ -146,7 +151,7 @@ extern int32_t tsUptimeInterval; extern int32_t tsRpcRetryLimit; extern int32_t tsRpcRetryInterval; -//#define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) +// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 1fa862b145..7f9b240f63 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -43,7 +43,7 @@ extern "C" { #define SYNC_MAX_RETRY_BACKOFF 5 #define SYNC_LOG_REPL_RETRY_WAIT_MS 100 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000 -#define SYNC_HEART_TIMEOUT_MS 1000 * 8 +#define SYNC_HEART_TIMEOUT_MS 1000 * 15 #define SYNC_HEARTBEAT_SLOW_MS 1500 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2e8cb5e5f7..3bcfddb8b2 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -55,6 +55,11 @@ int32_t tsNumOfQnodeFetchThreads = 1; int32_t tsNumOfSnodeStreamThreads = 4; int32_t tsNumOfSnodeWriteThreads = 1; +// sync raft +int32_t tsElectInterval = 25 * 1000; +int32_t tsHeartbeatInterval = 1000; +int32_t tsHeartbeatTimeout = 20 * 1000; + // monitor bool tsEnableMonitor = true; int32_t tsMonitorInterval = 30; @@ -74,8 +79,8 @@ char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. // If set to empty system will generate table name using MD5 hash. // true means that the name and order of cols in each line are the same(only for influx protocol) -bool tsSmlDataFormat = false; -int32_t tsSmlBatchSize = 10000; +bool tsSmlDataFormat = false; +int32_t tsSmlBatchSize = 10000; // query int32_t tsQueryPolicy = 1; @@ -198,9 +203,7 @@ int32_t taosSetTfsCfg(SConfig *pCfg) { int32_t taosSetTfsCfg(SConfig *pCfg); #endif -struct SConfig *taosGetCfg() { - return tsCfg; -} +struct SConfig *taosGetCfg() { return tsCfg; } static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { @@ -423,6 +426,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; @@ -728,6 +735,10 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; + tsElectInterval = cfgGetItem(pCfg, "syncElectInterval")->i32; + tsHeartbeatInterval = cfgGetItem(pCfg, "syncHeartbeatInterval")->i32; + tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32; + tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; @@ -737,6 +748,10 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsWalFsyncDataSizeLimit = cfgGetItem(pCfg, "walFsyncDataSizeLimit")->i64; + tsElectInterval = cfgGetItem(pCfg, "syncElectInterval")->i32; + tsHeartbeatInterval = cfgGetItem(pCfg, "syncHeartbeatInterval")->i32; + tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32; + tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); tstrncpy(tsUdfdLdLibPath, cfgGetItem(pCfg, "udfdLdLibPath")->str, sizeof(tsUdfdLdLibPath)); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 1fe2af696c..a00b639a9c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -22,8 +22,8 @@ #include "syncEnv.h" #include "syncIndexMgr.h" #include "syncInt.h" -#include "syncPipeline.h" #include "syncMessage.h" +#include "syncPipeline.h" #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" @@ -35,6 +35,7 @@ #include "syncTimeout.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "tglobal.h" #include "tref.h" static void syncNodeEqPingTimer(void* param, void* tmrId); @@ -1115,7 +1116,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->hbrSlowNum = 0; pSyncNode->tmrRoutineNum = 0; - sNTrace(pSyncNode, "sync open, node:%p", pSyncNode); + sNInfo(pSyncNode, "sync open, node:%p", pSyncNode); + sTrace("vgId:%d, tsElectInterval:%d, tsHeartbeatInterval:%d, tsHeartbeatTimeout:%d", pSyncNode->vgId, tsElectInterval, + tsHeartbeatInterval, tsHeartbeatTimeout); return pSyncNode; @@ -1229,7 +1232,7 @@ void syncHbTimerDataFree(SSyncHbTimerData* pData) { taosMemoryFree(pData); } void syncNodeClose(SSyncNode* pSyncNode) { if (pSyncNode == NULL) return; - sNTrace(pSyncNode, "sync close, data:%p", pSyncNode); + sNInfo(pSyncNode, "sync close, node:%p", pSyncNode); int32_t ret = raftStoreClose(pSyncNode->pRaftStore); ASSERT(ret == 0); From 8aad71c92843a3f181811a8c64f123e5c2b10c1e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Dec 2022 10:07:08 +0800 Subject: [PATCH 84/95] more code --- source/dnode/vnode/src/vnd/vnodeCommit.c | 1 + source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 79e4f46241..6ef5116fdc 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -232,6 +232,7 @@ int vnodeAsyncCommit(SVnode *pVnode) { pInfo->info.state.committed = pVnode->state.applied; pInfo->info.state.commitTerm = pVnode->state.applyTerm; pInfo->info.state.commitID = pVnode->state.commitID; + pInfo->pVnode = pVnode; vnodeScheduleTask(vnodeCommitTask, pInfo); _exit: diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index afb4edab7a..daeda530fd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -313,7 +313,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp // commit if need if (vnodeShouldCommit(pVnode)) { vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); +#if 1 + vnodeSyncCommit(pVnode); +#else vnodeAsyncCommit(pVnode); +#endif // start a new one if (vnodeBegin(pVnode) < 0) { From 0dce1e1047d869c5f25e8474d176a2b52742ad01 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Dec 2022 11:15:15 +0800 Subject: [PATCH 85/95] async commit --- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index daeda530fd..d8c8a3e1b2 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -313,7 +313,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp // commit if need if (vnodeShouldCommit(pVnode)) { vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); -#if 1 +#if 0 vnodeSyncCommit(pVnode); #else vnodeAsyncCommit(pVnode); From 0d687a533efbad0c51cad17eabbf37efd6f5023a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 4 Dec 2022 11:35:54 +0800 Subject: [PATCH 86/95] fix(query): check for null ptr. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 34 ++++++++++++++------------ 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index fece5df28e..7f513d2b2d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2534,6 +2534,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* int32_t code = 0; SArray* pDelData = taosArrayInit(4, sizeof(SDelData)); + ASSERT(pReader->pReadSnap != NULL); SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; if (pDelFile && taosArrayGetSize(pReader->pDelIdx) > 0) { @@ -2640,23 +2641,26 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { taosArrayDestroy(pIndexList); - SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; - if (pReader->pDelFReader == NULL && pDelFile != NULL) { - int32_t code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pReader->pTsdb); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + if (pReader->pReadSnap != NULL) { - pReader->pDelIdx = taosArrayInit(4, sizeof(SDelIdx)); - if (pReader->pDelIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } + SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; + if (pReader->pDelFReader == NULL && pDelFile != NULL) { + int32_t code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pReader->pTsdb); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - code = tsdbReadDelIdx(pReader->pDelFReader, pReader->pDelIdx); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pReader->pDelIdx); - return code; + pReader->pDelIdx = taosArrayInit(4, sizeof(SDelIdx)); + if (pReader->pDelIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + code = tsdbReadDelIdx(pReader->pDelFReader, pReader->pDelIdx); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pReader->pDelIdx); + return code; + } } } From 69850a1a2b1e011130133f16866f248261c7f53d Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Dec 2022 12:28:36 +0800 Subject: [PATCH 87/95] fix: dataDir process of multi-tier storage conform to 2.6 --- source/util/src/tconfig.c | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 586586f742..cb47838b1b 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -294,8 +294,8 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, SDiskCfg cfg = {0}; tstrncpy(cfg.dir, value, sizeof(cfg.dir)); - cfg.level = atoi(level); - cfg.primary = atoi(primary); + cfg.level = level ? atoi(level) : 0; + cfg.primary = primary ? atoi(primary) : 1; void *ret = taosArrayPush(pItem->array, &cfg); if (ret == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -660,13 +660,12 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_VAR); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } uInfo("load from env variables cfg success"); @@ -703,13 +702,12 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } uInfo("load from env cmd cfg success"); @@ -768,13 +766,12 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_FILE); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } taosCloseFile(&pFile); @@ -828,13 +825,12 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } taosCloseFile(&pFile); @@ -993,13 +989,12 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { paGetToken(value2 + vlen2 + 1, &value3, &vlen3); if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_APOLLO_URL); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } + code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } tjsonDelete(pJson); From 4dbc6075a381dae7e0c009afa3d22802c54f1d49 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Dec 2022 12:32:08 +0800 Subject: [PATCH 88/95] fix: dataDir process of multi-tier storage conform to 2.6 --- source/util/src/tconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index cb47838b1b..3d05f80acf 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -891,7 +891,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { // code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); // if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; -// if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { +// if (strcasecmp(name, "dataDir") == 0) { // code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); // if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; // } From f78efdac7ef263e21daddb53def1cd1cd4b8f409 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Dec 2022 14:50:58 +0800 Subject: [PATCH 89/95] fix asan error --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index f3acfca39c..bd616766df 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -170,6 +170,10 @@ int32_t tsdbCommit(STsdb *pTsdb) { // check if (pMemTable->nRow == 0 && pMemTable->nDel == 0) { + taosThreadRwlockWrlock(&pTsdb->rwLock); + pTsdb->imem = NULL; + taosThreadRwlockUnlock(&pTsdb->rwLock); + tsdbUnrefMemTable(pMemTable); goto _exit; } From 2a20347210aa5a70bd4a0e5ccf135bf9197e0549 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 4 Dec 2022 15:14:04 +0800 Subject: [PATCH 90/95] add more code --- source/dnode/vnode/src/inc/vnodeInt.h | 10 ++++++++-- source/dnode/vnode/src/sma/smaCommit.c | 18 +++++++++--------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 18 +++++++++--------- source/dnode/vnode/src/vnd/vnodeCommit.c | 10 +++------- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 010a011896..35c2f39770 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -75,6 +75,7 @@ typedef struct SStreamStateWriter SStreamStateWriter; typedef struct SRSmaSnapReader SRSmaSnapReader; typedef struct SRSmaSnapWriter SRSmaSnapWriter; typedef struct SSnapDataHdr SSnapDataHdr; +typedef struct SCommitInfo SCommitInfo; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" @@ -147,7 +148,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* int tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbPrepareCommit(STsdb* pTsdb); -int32_t tsdbCommit(STsdb* pTsdb); +int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); int32_t tsdbFinishCommit(STsdb* pTsdb); int32_t tsdbRollbackCommit(STsdb* pTsdb); int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); @@ -205,7 +206,7 @@ int32_t smaSyncPreCommit(SSma* pSma); int32_t smaSyncCommit(SSma* pSma); int32_t smaSyncPostCommit(SSma* pSma); int32_t smaPreCommit(SSma* pSma); -int32_t smaCommit(SSma* pSma); +int32_t smaCommit(SSma* pSma, SCommitInfo* pInfo); int32_t smaFinishCommit(SSma* pSma); int32_t smaPostCommit(SSma* pSma); int32_t smaDoRetention(SSma* pSma, int64_t now); @@ -407,6 +408,11 @@ struct SSnapDataHdr { uint8_t data[]; }; +struct SCommitInfo { + SVnodeInfo info; + SVnode* pVnode; +}; + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index a79ae35d79..f038d96c7b 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -23,7 +23,7 @@ static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma); static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma); #endif static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma); -static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo); static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); @@ -67,7 +67,7 @@ int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); * @param pSma * @return int32_t */ -int32_t smaCommit(SSma *pSma) { return tdProcessRSmaAsyncCommitImpl(pSma); } +int32_t smaCommit(SSma *pSma, SCommitInfo *pInfo) { return tdProcessRSmaAsyncCommitImpl(pSma, pInfo); } /** * @brief async commit, only applicable to Rollup SMA @@ -127,8 +127,8 @@ _exit: } int32_t smaFinishCommit(SSma *pSma) { - int32_t code = 0; - SVnode *pVnode = pSma->pVnode; + int32_t code = 0; + SVnode *pVnode = pSma->pVnode; if (VND_RSMA1(pVnode) && (code = tsdbFinishCommit(VND_RSMA1(pVnode))) < 0) { smaError("vgId:%d, failed to finish commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(code)); @@ -387,9 +387,9 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { * @param pSma * @return int32_t */ -static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { - int32_t code = 0; - SVnode *pVnode = pSma->pVnode; +static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo) { + int32_t code = 0; + SVnode *pVnode = pSma->pVnode; #if 0 SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); @@ -399,11 +399,11 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { } #endif - if ((code = tsdbCommit(VND_RSMA1(pVnode))) < 0) { + if ((code = tsdbCommit(VND_RSMA1(pVnode), pInfo)) < 0) { smaError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(code)); goto _exit; } - if ((code = tsdbCommit(VND_RSMA2(pVnode))) < 0) { + if ((code = tsdbCommit(VND_RSMA2(pVnode), pInfo)) < 0) { smaError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(code)); goto _exit; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index bd616766df..906e3b2638 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -93,7 +93,7 @@ typedef struct { SArray *aDelData; // SArray } SCommitter; -static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter); +static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter, SCommitInfo *pInfo); static int32_t tsdbCommitData(SCommitter *pCommitter); static int32_t tsdbCommitDel(SCommitter *pCommitter); static int32_t tsdbCommitCache(SCommitter *pCommitter); @@ -160,7 +160,7 @@ int32_t tsdbPrepareCommit(STsdb *pTsdb) { return 0; } -int32_t tsdbCommit(STsdb *pTsdb) { +int32_t tsdbCommit(STsdb *pTsdb, SCommitInfo *pInfo) { if (!pTsdb) return 0; int32_t code = 0; @@ -179,7 +179,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { } // start commit - code = tsdbStartCommit(pTsdb, &commith); + code = tsdbStartCommit(pTsdb, &commith, pInfo); TSDB_CHECK_CODE(code, lino, _exit); // commit impl @@ -816,7 +816,7 @@ _exit: } // ---------------------------------------------------------------------------- -static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { +static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter, SCommitInfo *pInfo) { int32_t code = 0; int32_t lino = 0; @@ -824,13 +824,13 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { ASSERT(pTsdb->imem && "last tsdb commit incomplete"); pCommitter->pTsdb = pTsdb; - pCommitter->commitID = pTsdb->pVnode->state.commitID; + pCommitter->commitID = pInfo->info.state.commitID; pCommitter->minutes = pTsdb->keepCfg.days; pCommitter->precision = pTsdb->keepCfg.precision; - pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; - pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; - pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; - pCommitter->sttTrigger = pTsdb->pVnode->config.sttTrigger; + pCommitter->minRow = pInfo->info.config.tsdbCfg.minRows; + pCommitter->maxRow = pInfo->info.config.tsdbCfg.maxRows; + pCommitter->cmprAlg = pInfo->info.config.tsdbCfg.compression; + pCommitter->sttTrigger = pInfo->info.config.sttTrigger; pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); if (pCommitter->aTbDataP == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 6ef5116fdc..8401b164ea 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -14,11 +14,7 @@ */ #include "vnd.h" - -typedef struct { - SVnodeInfo info; - SVnode *pVnode; -} SCommitInfo; +#include "vnodeInt.h" #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" @@ -291,11 +287,11 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { TSDB_CHECK_CODE(code, lino, _exit); } - code = tsdbCommit(pVnode->pTsdb); + code = tsdbCommit(pVnode->pTsdb, pInfo); TSDB_CHECK_CODE(code, lino, _exit); if (VND_IS_RSMA(pVnode)) { - code = smaCommit(pVnode->pSma); + code = smaCommit(pVnode->pSma, pInfo); TSDB_CHECK_CODE(code, lino, _exit); } From 1e15e23ccefca0c13bab7507bbddec328f20ccb9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 4 Dec 2022 15:30:06 +0800 Subject: [PATCH 91/95] feat: taosbenchmark supports vgid (#18657) * test: update taos-tools e0f2fe3 * feat: taosbenchmark supports vgid * fix: util func refactor * test: update taos-tools 09372db * test: update taos-tools e62c5ea * test: comment off replica line for ci * test: comment off invalid_commandline.py and demo.py --- cmake/taostools_CMakeLists.txt.in | 2 +- tests/develop-test/5-taos-tools/taosbenchmark/demo.py | 2 +- .../5-taos-tools/taosbenchmark/invalid_commandline.py | 2 +- tests/parallel_test/cases.task | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index db727907b7..2d7bcf6592 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 823fae5 + GIT_TAG e62c5ea SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py index 6be5117b08..25a8d92f7e 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py @@ -28,7 +28,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) + # comment off by Shuduo for CI self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py index 73894d5e33..e5a2551e63 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py @@ -25,7 +25,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) + # comment off by Shuduo for CI self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 28ba7ea60f..c90b2499a4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1022,9 +1022,9 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py -,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py +#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py -,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py +#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py From feceb0f1f45ca8a9e3b56a80a2334a7b9061f026 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Dec 2022 15:31:08 +0800 Subject: [PATCH 92/95] fix: multi-tier storage test case update --- tests/system-test/1-insert/mutil_stage.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 3e2bec130e..8780ecc477 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -147,19 +147,20 @@ class TDTestCase: def __current_cfg(self): cfg_list = [] current_case1 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", + #f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}" ] - current_case2 = [f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] + #current_case2 = [f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] + current_case2 = [] for i in range(9): current_case2.append(f"dataDir {self.taos_data_dir}/{DATA_PRE0}{i+1} {L0} {NON_PRIMARY_DIR}") # TD-17773bug current_case3 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 ", + #f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 ", f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE1}0 {L1} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE2}0 {L2} {NON_PRIMARY_DIR}", From 9b9ebea6f50fcd6a63309d34962ea850cbfff2f8 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Dec 2022 17:49:37 +0800 Subject: [PATCH 93/95] fix: tsdb do retention fix --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 1d09bd4936..c7bce6182a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -750,7 +750,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { // head tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -771,7 +771,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { // data tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -781,7 +781,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = taosFSendFile(pOutFD, PInFD, 0, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage)); + n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -792,7 +792,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { // sma tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -814,7 +814,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) { tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom); tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; From a4adcbbe0df383cb4fef988dedced773aa94aad7 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sun, 4 Dec 2022 18:13:51 +0800 Subject: [PATCH 94/95] vnode/async-commit: adapter vnode-commit to use txn commit --- source/dnode/vnode/src/inc/vnodeInt.h | 6 ++++-- source/dnode/vnode/src/meta/metaCommit.c | 7 ++++--- source/dnode/vnode/src/meta/metaSnapshot.c | 4 ++-- source/dnode/vnode/src/vnd/vnodeCommit.c | 10 ++++++---- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 35c2f39770..d3f10ad529 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -101,8 +101,9 @@ typedef struct STbUidStore STbUidStore; int metaOpen(SVnode* pVnode, SMeta** ppMeta, int8_t rollback); int metaClose(SMeta* pMeta); int metaBegin(SMeta* pMeta, int8_t fromSys); -int metaCommit(SMeta* pMeta); -int metaFinishCommit(SMeta* pMeta); +TXN* metaGetTxn(SMeta* pMeta); +int metaCommit(SMeta* pMeta, TXN* txn); +int metaFinishCommit(SMeta* pMeta, TXN* txn); int metaPrepareAsyncCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); @@ -411,6 +412,7 @@ struct SSnapDataHdr { struct SCommitInfo { SVnodeInfo info; SVnode* pVnode; + TXN* txn; }; #ifdef __cplusplus diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index c46c219ac3..b2cd85e9fa 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -40,9 +40,10 @@ int metaBegin(SMeta *pMeta, int8_t fromSys) { } // commit the meta txn -int metaCommit(SMeta *pMeta) { return tdbCommit(pMeta->pEnv, pMeta->txn); } -int metaFinishCommit(SMeta *pMeta) { return tdbPostCommit(pMeta->pEnv, pMeta->txn); } -int metaPrepareAsyncCommit(SMeta *pMeta) { return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); } +TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; } +int metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); } +int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); } +int metaPrepareAsyncCommit(SMeta *pMeta) { return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); } // abort the meta txn int metaAbort(SMeta *pMeta) { return tdbAbort(pMeta->pEnv, pMeta->txn); } diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 5c5b49ece5..6a4dcf6ead 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -163,9 +163,9 @@ int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) { if (rollback) { ASSERT(0); } else { - code = metaCommit(pWriter->pMeta); + code = metaCommit(pWriter->pMeta, pWriter->pMeta->txn); if (code) goto _err; - code = metaFinishCommit(pWriter->pMeta); + code = metaFinishCommit(pWriter->pMeta, pWriter->pMeta->txn); if (code) goto _err; } taosMemoryFree(pWriter); diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 8401b164ea..4bb2b62f28 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -189,10 +189,11 @@ _err: static void vnodePrepareCommit(SVnode *pVnode) { tsem_wait(&pVnode->canCommit); + tsdbPrepareCommit(pVnode->pTsdb); + metaPrepareAsyncCommit(pVnode->pMeta); + vnodeBufPoolUnRef(pVnode->inUse); pVnode->inUse = NULL; - - tsdbPrepareCommit(pVnode->pTsdb); } static int32_t vnodeCommitTask(void *arg) { int32_t code = 0; @@ -229,6 +230,7 @@ int vnodeAsyncCommit(SVnode *pVnode) { pInfo->info.state.commitTerm = pVnode->state.applyTerm; pInfo->info.state.commitID = pVnode->state.commitID; pInfo->pVnode = pVnode; + pInfo->txn = metaGetTxn(pVnode->pMeta); vnodeScheduleTask(vnodeCommitTask, pInfo); _exit: @@ -282,7 +284,7 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { TSDB_CHECK_CODE(code, lino, _exit); // commit each sub-system - if (metaCommit(pVnode->pMeta) < 0) { + if (metaCommit(pVnode->pMeta, pInfo->txn) < 0) { code = TSDB_CODE_FAILED; TSDB_CHECK_CODE(code, lino, _exit); } @@ -314,7 +316,7 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { TSDB_CHECK_CODE(code, lino, _exit); } - if (metaFinishCommit(pVnode->pMeta) < 0) { + if (metaFinishCommit(pVnode->pMeta, pInfo->txn) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } From cf0995f749832bdef213ac8571b9486542d68bf5 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Dec 2022 20:26:58 +0800 Subject: [PATCH 95/95] feat: async commit for rsma --- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/sma/smaCommit.c | 7 ++++++- source/dnode/vnode/src/vnd/vnodeCommit.c | 5 ++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index d3f10ad529..8cf212cb1d 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -206,7 +206,7 @@ int32_t smaBegin(SSma* pSma); int32_t smaSyncPreCommit(SSma* pSma); int32_t smaSyncCommit(SSma* pSma); int32_t smaSyncPostCommit(SSma* pSma); -int32_t smaPreCommit(SSma* pSma); +int32_t smaPrepareAsyncCommit(SSma* pSma); int32_t smaCommit(SSma* pSma, SCommitInfo* pInfo); int32_t smaFinishCommit(SSma* pSma); int32_t smaPostCommit(SSma* pSma); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index f038d96c7b..9748963722 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -59,7 +59,7 @@ int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaSyncPostCommitImpl(p * @param pSma * @return int32_t */ -int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); } +int32_t smaPrepareAsyncCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); } /** * @brief async commit, only applicable to Rollup SMA @@ -378,6 +378,11 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); #endif + // all rsma results are written completely + STsdb *pTsdb = NULL; + if ((pTsdb = VND_RSMA1(pSma->pVnode))) tsdbPrepareCommit(pTsdb); + if ((pTsdb = VND_RSMA2(pSma->pVnode))) tsdbPrepareCommit(pTsdb); + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 4bb2b62f28..f9a598fec6 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -191,6 +191,8 @@ static void vnodePrepareCommit(SVnode *pVnode) { tsdbPrepareCommit(pVnode->pTsdb); metaPrepareAsyncCommit(pVnode->pMeta); + smaPrepareAsyncCommit(pVnode->pSma); + vnodeBufPoolUnRef(pVnode->inUse); pVnode->inUse = NULL; @@ -280,9 +282,6 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { // walBeginSnapshot(pVnode->pWal, pVnode->state.applied); syncBeginSnapshot(pVnode->sync, pVnode->state.applied); - code = smaPreCommit(pVnode->pSma); - TSDB_CHECK_CODE(code, lino, _exit); - // commit each sub-system if (metaCommit(pVnode->pMeta, pInfo->txn) < 0) { code = TSDB_CODE_FAILED;