From 9e7f860d7c38ed7898891a9279403a03a8c1a1ff Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 12 Jun 2023 16:40:24 +0800 Subject: [PATCH 01/34] enh(tdb/recycle): first round implemenation of page recycling --- source/libs/tdb/src/db/tdbBtree.c | 11 + source/libs/tdb/src/db/tdbDb.c | 6 + source/libs/tdb/src/db/tdbPager.c | 56 +- source/libs/tdb/src/inc/tdbInt.h | 3 + source/libs/tdb/test/CMakeLists.txt | 4 + source/libs/tdb/test/tdbPageRecycleTest.cpp | 674 ++++++++++++++++++++ 6 files changed, 753 insertions(+), 1 deletion(-) create mode 100644 source/libs/tdb/test/tdbPageRecycleTest.cpp diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c49b5726b6..bb02db8bb8 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -860,6 +860,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx if (!TDB_BTREE_PAGE_IS_LEAF(pNews[0])) { ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno; + } else { + // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pNews[0])); + tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pNews[0]), pTxn); } } @@ -870,9 +873,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } for (pageIdx = 0; pageIdx < nOlds; ++pageIdx) { + // printf("tdb/balance: btree balance old pgno: %d.\n", TDB_PAGE_PGNO(pOlds[pageIdx])); + if (pageIdx >= nNews) { + // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pOlds[pageIdx])); + tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pOlds[pageIdx]), pTxn); + } tdbPagerReturnPage(pBt->pPager, pOlds[pageIdx], pTxn); } for (; pageIdx < nNews; ++pageIdx) { + // printf("tdb/balance: btree balance new pgno: %d.\n", TDB_PAGE_PGNO(pNews[pageIdx])); tdbPagerReturnPage(pBt->pPager, pNews[pageIdx], pTxn); } @@ -2113,6 +2122,8 @@ int tdbBtcDelete(SBTC *pBtc) { return -1; } + // printf("tdb/btc-delete: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pBtc->pPage)); + ret = tdbBtreeBalance(pBtc); if (ret < 0) { tdbError("tdb/btc-delete: btree balance failed with ret: %d.", ret); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 952c49db73..fe9d51dc82 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -70,6 +70,11 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i if (ret < 0) { return -1; } + + ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback); + if (ret < 0) { + return -1; + } #endif *ppDb = pDb; @@ -82,6 +87,7 @@ int tdbClose(TDB *pDb) { if (pDb) { #ifdef USE_MAINDB if (pDb->pMainDb) tdbTbClose(pDb->pMainDb); + if (pDb->pFreeDb) tdbTbClose(pDb->pFreeDb); #endif for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) { diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 5ea9be63db..5f187d339e 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -316,6 +316,10 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { return -1; } + if (!TDB_PAGE_TOTAL_CELLS(pPage) && TDB_PAGE_PGNO(pPage) > 1) { + tdbDebug("pager/commit: %p, %d/%d, txnId:%" PRId64, pPager, pPager->dbOrigSize, pPager->dbFileSize, pTxn->txnId); + } + ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); @@ -695,9 +699,59 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) { // TDB_PAGE_PGNO(pPage), pPage); } +int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) { + int code = 0; + + code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn); + if (code < 0) { + return -1; + } + + return code; +} + +static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) { + int code = 0; + TBC *pCur; + + if (!pPager->pEnv->pFreeDb) { + return 0; + } + + code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, NULL); + if (code < 0) { + return 0; + } + + code = tdbTbcMoveToFirst(pCur); + if (code) { + tdbTbcClose(pCur); + return 0; + } + + void *pKey = NULL; + int nKey = 0; + + code = tdbTbcGet(pCur, (const void **)&pKey, &nKey, NULL, NULL); + if (code < 0) { + tdbTbcClose(pCur); + return 0; + } + + *pPgno = *(SPgno *)pKey; + + code = tdbTbcDelete(pCur); + if (code < 0) { + tdbTbcClose(pCur); + return 0; + } + tdbTbcClose(pCur); + return 0; +} + static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { // TODO: Allocate a page from the free list - return 0; + return tdbPagerRemoveFreePage(pPager, ppgno); } static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) { diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 7a0bcc00a4..e65edb4afe 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -198,6 +198,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn); int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn); void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); +int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn); int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); int tdbPagerRestoreJournals(SPager *pPager); int tdbPagerRollback(SPager *pPager); @@ -373,6 +374,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { #ifdef USE_MAINDB #define TDB_MAINDB_NAME "main.tdb" +#define TDB_FREEDB_NAME "_free.db" #endif struct STDB { @@ -386,6 +388,7 @@ struct STDB { SPager **pgrHash; #ifdef USE_MAINDB TTB *pMainDb; + TTB *pFreeDb; #endif int64_t txnId; }; diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index fd4d7c101d..4715ccbd41 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -14,3 +14,7 @@ target_link_libraries(tdbExOVFLTest tdb gtest gtest_main) add_executable(tdbPageDefragmentTest "tdbPageDefragmentTest.cpp") target_link_libraries(tdbPageDefragmentTest tdb gtest gtest_main) +# page recycling testing +add_executable(tdbPageRecycleTest "tdbPageRecycleTest.cpp") +target_link_libraries(tdbPageRecycleTest tdb gtest gtest_main) + diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp new file mode 100644 index 0000000000..39e89aaf3d --- /dev/null +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -0,0 +1,674 @@ +#include + +#define ALLOW_FORBID_FUNC +#include "os.h" +#include "tdb.h" + +#include +#include +#include +#include +#include "tlog.h" + +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void clearPool(SPoolMem *pPool) { + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); + } while (1); + + assert(pPool->size == 0); +} + +static void closePool(SPoolMem *pPool) { + clearPool(pPool); + taosMemoryFree(pPool); +} + +static void *poolMalloc(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} + +static void generateBigVal(char *val, int valLen) { + for (int i = 0; i < valLen; ++i) { + char c = char(i & 0xff); + if (c == 0) { + c = 1; + } + val[i] = c; + } +} + +static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { + TDB *pEnv = NULL; + + int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0); + if (ret) { + pEnv = NULL; + } + + return pEnv; +} + +static void insertOfp(void) { + int ret = 0; + + taosRemoveDir("tdb"); + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + char const *key = "key123456789"; + ret = tdbTbInsert(pDb, key, strlen(key), val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); +} + +TEST(TdbPageRecycleTest, DISABLED_TbInsertTest) { + // TEST(TdbPageRecycleTest, TbInsertTest) { + // ofp inserting + insertOfp(); +} + +TEST(TdbPageRecycleTest, DISABLED_TbGetTest) { + // TEST(TdbPageRecycleTest, TbGetTest) { + insertOfp(); + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // int ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + int ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + { // Query the data + void *pVal = NULL; + int vLen; + + // char const *key = "key1"; + char const *key = "key123456789"; + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, valLen); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + + tdbFree(pVal); + } +} + +TEST(TdbPageRecycleTest, DISABLED_TbDeleteTest) { + // TEST(TdbPageRecycleTest, TbDeleteTest) { + int ret = 0; + + taosRemoveDir("tdb"); + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[((4083 - 4 - 3 - 2) + 1) * 2]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + { // insert the generated big data + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the data + void *pVal = NULL; + int vLen; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, valLen); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + + tdbFree(pVal); + } + /* open to debug committed file +tdbCommit(pEnv, &txn); +tdbTxnClose(&txn); + +++txnid; +tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); +tdbBegin(pEnv, &txn); + */ + { // upsert the data + ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the upserted data + void *pVal = NULL; + int vLen; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, strlen("value1")); + GTEST_ASSERT_EQ(memcmp("value1", pVal, vLen), 0); + + tdbFree(pVal); + } + + { // delete the data + ret = tdbTbDelete(pDb, "key1", strlen("key1"), txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the deleted data + void *pVal = NULL; + int vLen = -1; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == -1); + GTEST_ASSERT_EQ(ret, -1); + + GTEST_ASSERT_EQ(vLen, -1); + GTEST_ASSERT_EQ(pVal, nullptr); + + tdbFree(pVal); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); +} + +TEST(TdbPageRecycleTest, DISABLED_simple_insert1) { + // TEST(TdbPageRecycleTest, simple_insert1) { + int ret; + TDB *pEnv; + TTB *pDb; + tdb_cmpr_fn_t compFunc; + int nData = 1; + TXN *txn; + int const pageSize = 4096; + + taosRemoveDir("tdb"); + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + { + char key[64]; + // char val[(4083 - 4 - 3 - 2)]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 1; iData <= nData; iData++) { + sprintf(key, "key0"); + sprintf(val, "value%d", iData); + + // ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + // GTEST_ASSERT_EQ(ret, 0); + + // generate value payload + int valLen = sizeof(val) / sizeof(val[0]); + for (int i = 6; i < valLen; ++i) { + char c = char(i & 0xff); + if (c == 0) { + c = 1; + } + val[i] = c; + } + + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + { // Query the data + void *pVal = NULL; + int vLen; + + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + // sprintf(val, "value%d", i); + + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, sizeof(val) / sizeof(val[0])); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + } + + tdbFree(pVal); + } + + { // Iterate to query the DB data + TBC *pDBC; + void *pKey = NULL; + void *pVal = NULL; + int vLen, kLen; + int count = 0; + + ret = tdbTbcOpen(pDb, &pDBC, NULL); + GTEST_ASSERT_EQ(ret, 0); + + tdbTbcMoveToFirst(pDBC); + + for (;;) { + ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; + + count++; + } + + GTEST_ASSERT_EQ(count, nData); + + tdbTbcClose(pDBC); + + tdbFree(pKey); + tdbFree(pVal); + } + } + + ret = tdbTbDrop(pDb); + GTEST_ASSERT_EQ(ret, 0); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); +} + +// TEST(TdbPageRecycleTest, DISABLED_seq_insert) { +TEST(TdbPageRecycleTest, seq_insert) { + int ret = 0; + TDB *pEnv = NULL; + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc; + int nData = 256; + TXN *txn = NULL; + int const pageSize = 4 * 1024; + + taosRemoveDir("tdb"); + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // 1, insert nData kv + { + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 0; iData < nData; ++iData) { + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); + GTEST_ASSERT_EQ(ret, 0); + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // 2, delete nData/2 records + + closePool(pPool); + } + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} + +// TEST(TdbPageRecycleTest, DISABLED_seq_delete) { +TEST(TdbPageRecycleTest, seq_delete) { + int ret = 0; + TDB *pEnv = NULL; + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc; + int nData = 256; + TXN *txn = NULL; + int const pageSize = 4 * 1024; + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // 2, delete nData/2 records + { + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 0; iData < nData; iData++) { + // if (iData % 2 == 0) continue; + + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + { // delete the data + ret = tdbTbDelete(pDb, key, strlen(key), txn); + GTEST_ASSERT_EQ(ret, 0); + } + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + } + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} + +// TEST(TdbPageRecycleTest, DISABLED_recycly_insert) { +TEST(TdbPageRecycleTest, recycly_insert) { + int ret = 0; + TDB *pEnv = NULL; + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + int nData = 256; + TXN *txn = NULL; + int const pageSize = 4 * 1024; + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // 3, insert 32k records + { + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = nData; iData < nData + nData; iData++) { + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); + GTEST_ASSERT_EQ(ret, 0); + + if (pPool->size >= poolLimit) { + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + } + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} From 621d4b20c0361efb5d24261c700db89f70f8c76e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 12 Jun 2023 16:43:49 +0800 Subject: [PATCH 02/34] tdb/pager: remove debug log --- source/libs/tdb/src/db/tdbPager.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 5f187d339e..8984de6476 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -316,10 +316,6 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { return -1; } - if (!TDB_PAGE_TOTAL_CELLS(pPage) && TDB_PAGE_PGNO(pPage) > 1) { - tdbDebug("pager/commit: %p, %d/%d, txnId:%" PRId64, pPager, pPager->dbOrigSize, pPager->dbFileSize, pTxn->txnId); - } - ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); From 8fee813de633881fa4c998139ec3b9bf86f08d1c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 13 Jun 2023 14:35:49 +0800 Subject: [PATCH 03/34] tdb/alloc-page: new param pTxn to fix memory leaking --- source/libs/tdb/src/db/tdbPager.c | 24 ++++++++++++++++-------- source/libs/tdb/src/inc/tdbInt.h | 6 +++--- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 8984de6476..a1d57db8d3 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -338,10 +338,13 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { if (pTxn->jPageSet) { hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } + + tdbTrace("tdb/pager-commit: remove page: %p %d from dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); + tdbPCacheRelease(pPager->pCache, pPage, pTxn); } - tdbTrace("pager/commit reset dirty tree: %p", &pPager->rbt); + tdbTrace("tdb/pager-commit reset dirty tree: %p", &pPager->rbt); tRBTreeCreate(&pPager->rbt, pageCmpFn); // sync the db file @@ -629,6 +632,8 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) { return 0; } +static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno, TXN *pTxn); + int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn) { SPage *pPage; @@ -643,7 +648,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa // alloc new page if (pgno == 0) { loadPage = 0; - ret = tdbPagerAllocPage(pPager, &pgno); + ret = tdbPagerAllocPage(pPager, &pgno, pTxn); if (ret < 0) { tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); return -1; @@ -706,7 +711,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) { return code; } -static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) { +static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) { int code = 0; TBC *pCur; @@ -714,13 +719,14 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) { return 0; } - code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, NULL); + code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn); if (code < 0) { return 0; } code = tdbTbcMoveToFirst(pCur); if (code) { + tdbError("tdb/remove-free-page: moveto first failed with ret: %d.", code); tdbTbcClose(pCur); return 0; } @@ -730,6 +736,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) { code = tdbTbcGet(pCur, (const void **)&pKey, &nKey, NULL, NULL); if (code < 0) { + tdbError("tdb/remove-free-page: tbc get failed with ret: %d.", code); tdbTbcClose(pCur); return 0; } @@ -738,6 +745,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) { code = tdbTbcDelete(pCur); if (code < 0) { + tdbError("tdb/remove-free-page: tbc delete failed with ret: %d.", code); tdbTbcClose(pCur); return 0; } @@ -745,9 +753,9 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) { return 0; } -static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { +static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno, TXN *pTxn) { // TODO: Allocate a page from the free list - return tdbPagerRemoveFreePage(pPager, ppgno); + return tdbPagerRemoveFreePage(pPager, ppgno, pTxn); } static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) { @@ -755,13 +763,13 @@ static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) { return 0; } -int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) { +static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno, TXN *pTxn) { int ret; *ppgno = 0; // Try to allocate from the free list of the pager - ret = tdbPagerAllocFreePage(pPager, ppgno); + ret = tdbPagerAllocFreePage(pPager, ppgno, pTxn); if (ret < 0) { return -1; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index e65edb4afe..bd680da09e 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -199,9 +199,9 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initP TXN *pTxn); void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn); -int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); -int tdbPagerRestoreJournals(SPager *pPager); -int tdbPagerRollback(SPager *pPager); +// int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); +int tdbPagerRestoreJournals(SPager *pPager); +int tdbPagerRollback(SPager *pPager); // tdbPCache.c ==================================== #define TDB_PCACHE_PAGE \ From 40b741dfee02b21267c42fd242ac2f60a2bc7206 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 20 Jun 2023 07:52:45 +0800 Subject: [PATCH 04/34] tdb/pager: comment out error log --- source/libs/tdb/src/db/tdbPager.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index a1d57db8d3..9c00a82826 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -736,7 +736,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) { code = tdbTbcGet(pCur, (const void **)&pKey, &nKey, NULL, NULL); if (code < 0) { - tdbError("tdb/remove-free-page: tbc get failed with ret: %d.", code); + // tdbError("tdb/remove-free-page: tbc get failed with ret: %d.", code); tdbTbcClose(pCur); return 0; } @@ -754,7 +754,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) { } static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno, TXN *pTxn) { - // TODO: Allocate a page from the free list + // Allocate a page from the free list return tdbPagerRemoveFreePage(pPager, ppgno, pTxn); } From 4e3df6606bb3c03370722c6afc04cd4fab3b5108 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 20 Jun 2023 15:06:17 +0800 Subject: [PATCH 05/34] tdb/btree: recyle pNews 0 --- source/libs/tdb/src/db/tdbBtree.c | 8 ++++---- source/libs/tdb/src/db/tdbPager.c | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index bb02db8bb8..3afdb9a84f 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -860,10 +860,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx if (!TDB_BTREE_PAGE_IS_LEAF(pNews[0])) { ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno; - } else { - // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pNews[0])); - tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pNews[0]), pTxn); - } + } // else { + // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pNews[0])); + tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pNews[0]), pTxn); + //} } for (int i = 0; i < 3; i++) { diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 9c00a82826..4e29ca45ca 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -703,8 +703,10 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) { int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) { int code = 0; + // tdbError("tdb/insert-free-page: tbc get page: %d.", pgno); code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn); if (code < 0) { + tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); return -1; } @@ -742,6 +744,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) { } *pPgno = *(SPgno *)pKey; + // tdbError("tdb/remove-free-page: tbc get page: %d.", *pPgno); code = tdbTbcDelete(pCur); if (code < 0) { @@ -760,6 +763,7 @@ static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno, TXN *pTxn) { static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) { *ppgno = ++pPager->dbFileSize; + // tdbError("tdb/alloc-new-page: %d.", *ppgno); return 0; } From 76a734c53a38cbb710ffe7519ec1e4fee4a22b55 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 20 Jun 2023 15:07:49 +0800 Subject: [PATCH 06/34] tdb/test: fix recycle testing cases --- source/libs/tdb/test/tdbPageRecycleTest.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp index 39e89aaf3d..e4787fcc70 100644 --- a/source/libs/tdb/test/tdbPageRecycleTest.cpp +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -464,13 +464,15 @@ TEST(TdbPageRecycleTest, DISABLED_simple_insert1) { GTEST_ASSERT_EQ(ret, 0); } +static const int nDataConst = 256 * 19; + // TEST(TdbPageRecycleTest, DISABLED_seq_insert) { TEST(TdbPageRecycleTest, seq_insert) { int ret = 0; TDB *pEnv = NULL; TTB *pDb = NULL; tdb_cmpr_fn_t compFunc; - int nData = 256; + int nData = nDataConst; TXN *txn = NULL; int const pageSize = 4 * 1024; @@ -480,11 +482,13 @@ TEST(TdbPageRecycleTest, seq_insert) { ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); GTEST_ASSERT_EQ(ret, 0); + printf("tdb opened\n"); // Create a database compFunc = tKeyCmpr; ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); GTEST_ASSERT_EQ(ret, 0); + printf("tb opened\n"); // 1, insert nData kv { char key[64]; @@ -542,7 +546,7 @@ TEST(TdbPageRecycleTest, seq_delete) { TDB *pEnv = NULL; TTB *pDb = NULL; tdb_cmpr_fn_t compFunc; - int nData = 256; + int nData = nDataConst; TXN *txn = NULL; int const pageSize = 4 * 1024; @@ -614,7 +618,7 @@ TEST(TdbPageRecycleTest, recycly_insert) { TDB *pEnv = NULL; TTB *pDb = NULL; tdb_cmpr_fn_t compFunc = tKeyCmpr; - int nData = 256; + int nData = nDataConst; TXN *txn = NULL; int const pageSize = 4 * 1024; @@ -639,7 +643,8 @@ TEST(TdbPageRecycleTest, recycly_insert) { // start a transaction tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - for (int iData = nData; iData < nData + nData; iData++) { + // for (int iData = nData; iData < nData + nData; iData++) { + for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%03d", iData); sprintf(val, "value%03d", iData); From fe197ccf9c734ef20a5669a822129401903dba25 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sun, 25 Jun 2023 10:46:28 +0800 Subject: [PATCH 07/34] tdb/ofp-test: fix memory leaks --- source/libs/tdb/test/tdbExOVFLTest.cpp | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp index b16bc643d3..325703c946 100644 --- a/source/libs/tdb/test/tdbExOVFLTest.cpp +++ b/source/libs/tdb/test/tdbExOVFLTest.cpp @@ -190,6 +190,15 @@ static void insertOfp(void) { // commit current transaction tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } // TEST(TdbOVFLPagesTest, DISABLED_TbInsertTest) { @@ -233,6 +242,13 @@ TEST(TdbOVFLPagesTest, TbGetTest) { tdbFree(pVal); } + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } // TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) { @@ -334,6 +350,15 @@ tdbBegin(pEnv, &txn); // commit current transaction tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } // TEST(tdb_test, DISABLED_simple_insert1) { @@ -407,6 +432,8 @@ TEST(tdb_test, simple_insert1) { tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); + closePool(pPool); + { // Query the data void *pVal = NULL; int vLen; From 204999d57ecc91f28890a9371a5d42b10f3be587 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 26 Jun 2023 08:10:45 +0800 Subject: [PATCH 08/34] tdb/test: refactor page recycling test cases --- source/libs/tdb/src/db/tdbBtree.c | 8 +- source/libs/tdb/test/tdbPageRecycleTest.cpp | 88 ++++----------------- 2 files changed, 16 insertions(+), 80 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 3afdb9a84f..64ae8d1c3f 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -860,10 +860,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx if (!TDB_BTREE_PAGE_IS_LEAF(pNews[0])) { ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno; - } // else { - // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pNews[0])); + } + tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pNews[0]), pTxn); - //} } for (int i = 0; i < 3; i++) { @@ -873,15 +872,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } for (pageIdx = 0; pageIdx < nOlds; ++pageIdx) { - // printf("tdb/balance: btree balance old pgno: %d.\n", TDB_PAGE_PGNO(pOlds[pageIdx])); if (pageIdx >= nNews) { - // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pOlds[pageIdx])); tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pOlds[pageIdx]), pTxn); } tdbPagerReturnPage(pBt->pPager, pOlds[pageIdx], pTxn); } for (; pageIdx < nNews; ++pageIdx) { - // printf("tdb/balance: btree balance new pgno: %d.\n", TDB_PAGE_PGNO(pNews[pageIdx])); tdbPagerReturnPage(pBt->pPager, pNews[pageIdx], pTxn); } diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp index e4787fcc70..05b19fc4eb 100644 --- a/source/libs/tdb/test/tdbPageRecycleTest.cpp +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -464,31 +464,25 @@ TEST(TdbPageRecycleTest, DISABLED_simple_insert1) { GTEST_ASSERT_EQ(ret, 0); } -static const int nDataConst = 256 * 19; +static void clearDb(char const *db) { taosRemoveDir(db); } -// TEST(TdbPageRecycleTest, DISABLED_seq_insert) { -TEST(TdbPageRecycleTest, seq_insert) { +static void insertDb(int nData) { int ret = 0; TDB *pEnv = NULL; TTB *pDb = NULL; tdb_cmpr_fn_t compFunc; - int nData = nDataConst; TXN *txn = NULL; int const pageSize = 4 * 1024; - taosRemoveDir("tdb"); - // Open Env ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); GTEST_ASSERT_EQ(ret, 0); - printf("tdb opened\n"); // Create a database compFunc = tKeyCmpr; ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); GTEST_ASSERT_EQ(ret, 0); - printf("tb opened\n"); // 1, insert nData kv { char key[64]; @@ -540,13 +534,11 @@ TEST(TdbPageRecycleTest, seq_insert) { system("ls -l ./tdb"); } -// TEST(TdbPageRecycleTest, DISABLED_seq_delete) { -TEST(TdbPageRecycleTest, seq_delete) { +static void deleteDb(int nData) { int ret = 0; TDB *pEnv = NULL; TTB *pDb = NULL; tdb_cmpr_fn_t compFunc; - int nData = nDataConst; TXN *txn = NULL; int const pageSize = 4 * 1024; @@ -612,68 +604,16 @@ TEST(TdbPageRecycleTest, seq_delete) { system("ls -l ./tdb"); } -// TEST(TdbPageRecycleTest, DISABLED_recycly_insert) { -TEST(TdbPageRecycleTest, recycly_insert) { - int ret = 0; - TDB *pEnv = NULL; - TTB *pDb = NULL; - tdb_cmpr_fn_t compFunc = tKeyCmpr; - int nData = nDataConst; - TXN *txn = NULL; - int const pageSize = 4 * 1024; +static const int nDataConst = 256 * 19; - // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0); - GTEST_ASSERT_EQ(ret, 0); - - // Create a database - ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); - GTEST_ASSERT_EQ(ret, 0); - - // 3, insert 32k records - { - char key[64]; - char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) - int64_t poolLimit = 4096; - SPoolMem *pPool; - - // open the pool - pPool = openPool(); - - // start a transaction - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - - // for (int iData = nData; iData < nData + nData; iData++) { - for (int iData = 0; iData < nData; iData++) { - sprintf(key, "key%03d", iData); - sprintf(val, "value%03d", iData); - - ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); - GTEST_ASSERT_EQ(ret, 0); - - if (pPool->size >= poolLimit) { - tdbCommit(pEnv, txn); - tdbPostCommit(pEnv, txn); - - // start a new transaction - clearPool(pPool); - - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - } - } - - tdbCommit(pEnv, txn); - tdbPostCommit(pEnv, txn); - - closePool(pPool); - } - - // Close a database - tdbTbClose(pDb); - - // Close Env - ret = tdbClose(pEnv); - GTEST_ASSERT_EQ(ret, 0); - - system("ls -l ./tdb"); +// TEST(TdbPageRecycleTest, DISABLED_seq_insert) { +TEST(TdbPageRecycleTest, seq_insert) { + clearDb("tdb"); + insertDb(nDataConst); } + +// TEST(TdbPageRecycleTest, DISABLED_seq_delete) { +TEST(TdbPageRecycleTest, seq_delete) { deleteDb(nDataConst); } + +// TEST(TdbPageRecycleTest, DISABLED_recycly_insert) { +TEST(TdbPageRecycleTest, recycly_insert) { insertDb(nDataConst); } From f8921199e78f00cd5fa666fb33294eb39ec1c377 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 28 Jun 2023 09:37:26 +0800 Subject: [PATCH 09/34] tdb/test: cases for ofp recycling --- source/libs/tdb/test/tdbPageRecycleTest.cpp | 105 +++++++++++++++++--- 1 file changed, 89 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp index 05b19fc4eb..2f2dd0659b 100644 --- a/source/libs/tdb/test/tdbPageRecycleTest.cpp +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -120,16 +120,6 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in return cret; } -static void generateBigVal(char *val, int valLen) { - for (int i = 0; i < valLen; ++i) { - char c = char(i & 0xff); - if (c == 0) { - c = 1; - } - val[i] = c; - } -} - static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { TDB *pEnv = NULL; @@ -141,11 +131,19 @@ static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) return pEnv; } +static void generateBigVal(char *val, int valLen) { + for (int i = 0; i < valLen; ++i) { + char c = char(i & 0xff); + if (c == 0) { + c = 1; + } + val[i] = c; + } +} + static void insertOfp(void) { int ret = 0; - taosRemoveDir("tdb"); - // open Env int const pageSize = 4096; int const pageNum = 64; @@ -156,7 +154,7 @@ static void insertOfp(void) { TTB *pDb = NULL; tdb_cmpr_fn_t compFunc = tKeyCmpr; // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); - ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); GTEST_ASSERT_EQ(ret, 0); // open the pool @@ -176,22 +174,35 @@ static void insertOfp(void) { // insert the generated big data // char const *key = "key1"; char const *key = "key123456789"; - ret = tdbTbInsert(pDb, key, strlen(key), val, valLen, txn); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); GTEST_ASSERT_EQ(ret, 0); // commit current transaction tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); } +static void clearDb(char const *db) { taosRemoveDir(db); } + TEST(TdbPageRecycleTest, DISABLED_TbInsertTest) { // TEST(TdbPageRecycleTest, TbInsertTest) { // ofp inserting + clearDb("tdb"); insertOfp(); } TEST(TdbPageRecycleTest, DISABLED_TbGetTest) { // TEST(TdbPageRecycleTest, TbGetTest) { + clearDb("tdb"); insertOfp(); // open Env @@ -464,8 +475,6 @@ TEST(TdbPageRecycleTest, DISABLED_simple_insert1) { GTEST_ASSERT_EQ(ret, 0); } -static void clearDb(char const *db) { taosRemoveDir(db); } - static void insertDb(int nData) { int ret = 0; TDB *pEnv = NULL; @@ -617,3 +626,67 @@ TEST(TdbPageRecycleTest, seq_delete) { deleteDb(nDataConst); } // TEST(TdbPageRecycleTest, DISABLED_recycly_insert) { TEST(TdbPageRecycleTest, recycly_insert) { insertDb(nDataConst); } + +// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp) { +TEST(TdbPageRecycleTest, recycly_seq_insert_ofp) { + clearDb("tdb"); + insertOfp(); + system("ls -l ./tdb"); +} + +static void deleteOfp(void) { + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + { // delete the data + char const *key = "key123456789"; + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + ret = tdbTbDrop(pDb); + GTEST_ASSERT_EQ(ret, 0); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); +} + +// TEST(TdbPageRecycleTest, DISABLED_seq_delete_ofp) { +TEST(TdbPageRecycleTest, seq_delete_ofp) { + deleteOfp(); + system("ls -l ./tdb"); +} + +// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp_again) { +TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_again) { + insertOfp(); + system("ls -l ./tdb"); +} From a3c9b17212a1a58aebd00f95905226a88396f339 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 28 Jun 2023 10:46:01 +0800 Subject: [PATCH 10/34] tdb/ofp: recycl ofps --- source/libs/tdb/src/db/tdbBtree.c | 17 +++++++++++++++++ source/libs/tdb/src/inc/tdbInt.h | 15 ++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 64ae8d1c3f..65d1c30328 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1317,6 +1317,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, return -1; } + if (!pDecoder->ofps) { + pDecoder->ofps = taosArrayInit(8, sizeof(SPgno)); + } + taosArrayPush(pDecoder->ofps, &pgno); + ofpCell = tdbPageGetCell(ofp, 0); if (nLeft <= ofp->maxLocal - sizeof(SPgno)) { @@ -2075,6 +2080,14 @@ int tdbBtcDelete(SBTC *pBtc) { tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt); + // recycle ofps if any + if (pBtc->coder.ofps) { + for (int i = 0; i < TARRAY_SIZE(pBtc->coder.ofps); ++i) { + SPgno *pgno = taosArrayGet(pBtc->coder.ofps, i); + tdbPagerInsertFreePage(pBtc->pBt->pPager, *pgno, pBtc->pTxn); + } + } + // update interior page or do balance if (idx == nCells - 1) { if (idx) { @@ -2370,6 +2383,10 @@ int tdbBtcClose(SBTC *pBtc) { tdbTxnClose(pBtc->pTxn); } + if (pBtc->coder.ofps) { + taosArrayDestroy(pBtc->coder.ofps); + } + return 0; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index bd680da09e..7b08da4ca8 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -131,13 +131,14 @@ typedef struct SBtInfo { #define TDB_CELLDECODER_FREE_VAL(pCellDecoder) ((pCellDecoder)->freeKV & TDB_CELLD_F_VAL) typedef struct { - int kLen; - u8 *pKey; - int vLen; - u8 *pVal; - SPgno pgno; - u8 *pBuf; - u8 freeKV; + int kLen; + u8 *pKey; + int vLen; + u8 *pVal; + SPgno pgno; + u8 *pBuf; + u8 freeKV; + SArray *ofps; } SCellDecoder; struct SBTC { From b2c0bcb1e0480d968e45b5cac1c24ce34a4e6959 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 28 Jun 2023 13:35:14 +0800 Subject: [PATCH 11/34] tdb/ofp-recycle: fix mem leaks --- source/libs/tdb/src/db/tdbBtree.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 65d1c30328..ef9aaa4571 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -233,6 +233,7 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { int ret; tdbBtcOpen(&btc, pBt, pTxn); + btc.coder.ofps = taosArrayInit(8, sizeof(SPgno)); tdbTrace("tdb delete, btc: %p, pTxn: %p", &btc, pTxn); @@ -1317,10 +1318,9 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, return -1; } - if (!pDecoder->ofps) { - pDecoder->ofps = taosArrayInit(8, sizeof(SPgno)); + if (pDecoder->ofps) { + taosArrayPush(pDecoder->ofps, &pgno); } - taosArrayPush(pDecoder->ofps, &pgno); ofpCell = tdbPageGetCell(ofp, 0); From fc79074e499c864951f08dae8d648cff389e3a60 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 28 Jun 2023 15:51:33 +0800 Subject: [PATCH 12/34] tdb/ofp: turn ofp recycle off for ci --- source/libs/tdb/src/db/tdbBtree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index ef9aaa4571..7cbca72e71 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -564,6 +564,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } // copy the parent key out if child pages are not leaf page + // childNotLeaf = !(TDB_BTREE_PAGE_IS_LEAF(pOlds[0]) || TDB_BTREE_PAGE_IS_OVFL(pOlds[0])); childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(pOlds[0]); if (childNotLeaf) { for (int i = 0; i < nOlds; i++) { @@ -2084,7 +2085,7 @@ int tdbBtcDelete(SBTC *pBtc) { if (pBtc->coder.ofps) { for (int i = 0; i < TARRAY_SIZE(pBtc->coder.ofps); ++i) { SPgno *pgno = taosArrayGet(pBtc->coder.ofps, i); - tdbPagerInsertFreePage(pBtc->pBt->pPager, *pgno, pBtc->pTxn); + // tdbPagerInsertFreePage(pBtc->pBt->pPager, *pgno, pBtc->pTxn); } } From 0ec80ff47fcb71701954741be4eee4b5c4d84422 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 29 Jun 2023 13:14:45 +0800 Subject: [PATCH 13/34] tdb/ofp-recycle: recycle ofps when dropOfp --- source/libs/tdb/src/db/tdbBtree.c | 51 ++++++----- source/libs/tdb/src/db/tdbPager.c | 23 ++++- source/libs/tdb/src/inc/tdbInt.h | 2 +- source/libs/tdb/test/tdbPageRecycleTest.cpp | 97 +++++++++++++++++++++ 4 files changed, 149 insertions(+), 24 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 7cbca72e71..6921a26f19 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -233,8 +233,9 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { int ret; tdbBtcOpen(&btc, pBt, pTxn); + /* btc.coder.ofps = taosArrayInit(8, sizeof(SPgno)); - + */ tdbTrace("tdb delete, btc: %p, pTxn: %p", &btc, pTxn); // move the cursor @@ -864,7 +865,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno; } - tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pNews[0]), pTxn); + tdbPagerInsertFreePage(pBt->pPager, pNews[0], pTxn); } for (int i = 0; i < 3; i++) { @@ -875,7 +876,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx for (pageIdx = 0; pageIdx < nOlds; ++pageIdx) { if (pageIdx >= nNews) { - tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pOlds[pageIdx]), pTxn); + tdbPagerInsertFreePage(pBt->pPager, pOlds[pageIdx], pTxn); } tdbPagerReturnPage(pBt->pPager, pOlds[pageIdx], pTxn); } @@ -1319,10 +1320,6 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, return -1; } - if (pDecoder->ofps) { - taosArrayPush(pDecoder->ofps, &pgno); - } - ofpCell = tdbPageGetCell(ofp, 0); if (nLeft <= ofp->maxLocal - sizeof(SPgno)) { @@ -1529,8 +1526,8 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * if (pPage->vLen == TDB_VARIANT_LEN) { if (!leaf) { - tdbError("tdb/btree-cell-size: not a leaf page."); - return -1; + tdbError("tdb/btree-cell-size: not a leaf page:%p, pgno:%" PRIu32 ".", pPage, TDB_PAGE_PGNO(pPage)); + // return -1; } nHeader += tdbGetVarInt(pCell + nHeader, &vLen); } else if (leaf) { @@ -1570,8 +1567,27 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * bytes = ofp->maxLocal - sizeof(SPgno); } + SPgno origPgno = pgno; memcpy(&pgno, ofpCell + bytes, sizeof(pgno)); + ret = tdbPagerWrite(pBt->pPager, ofp); + if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); + return -1; + } + // tdbPageDropCell(ofp, 0, pTxn, pBt); + // tdbPageZero(ofp, sizeof(SLeafHdr), tdbBtreeCellSize); + // tdbPageZero(ofp, sizeof(SIntHdr), tdbBtreeCellSize); + // SIntHdr *pIntHdr = (SIntHdr *)(ofp->pData); + // pIntHdr->flags = TDB_FLAG_ADD(0, TDB_BTREE_OVFL); + // pIntHdr->pgno = 0; + // ofp->pPager = NULL; + + tdbPagerInsertFreePage(pBt->pPager, ofp, pTxn); + + // printf("tdb recycle, pTxn: %p, pgno:%u\n", pTxn, pgno); + tdbTrace("tdb recycle, pTxn: %p, pgno:%u", pTxn, origPgno); + tdbPagerReturnPage(pPage->pPager, ofp, pTxn); nLeft -= bytes; @@ -1991,6 +2007,11 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { return -1; } + if (TDB_BTREE_PAGE_IS_OVFL(pBtc->pPage)) { + tdbError("tdb/btc-move-downward: should not be a ovfl page here."); + return -1; + } + if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); pgno = ((SPgno *)pCell)[0]; @@ -2081,14 +2102,6 @@ int tdbBtcDelete(SBTC *pBtc) { tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt); - // recycle ofps if any - if (pBtc->coder.ofps) { - for (int i = 0; i < TARRAY_SIZE(pBtc->coder.ofps); ++i) { - SPgno *pgno = taosArrayGet(pBtc->coder.ofps, i); - // tdbPagerInsertFreePage(pBtc->pBt->pPager, *pgno, pBtc->pTxn); - } - } - // update interior page or do balance if (idx == nCells - 1) { if (idx) { @@ -2384,10 +2397,6 @@ int tdbBtcClose(SBTC *pBtc) { tdbTxnClose(pBtc->pTxn); } - if (pBtc->coder.ofps) { - taosArrayDestroy(pBtc->coder.ofps); - } - return 0; } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 4e29ca45ca..62702cbf40 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -292,7 +292,23 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { */ return 0; } +/* +int tdbPagerCancelDirty(SPager *pPager, SPage *pPage, TXN *pTxn) { + SRBTreeNode *pNode = tRBTreeGet(&pPager->rbt, (SRBTreeNode *)pPage); + if (pNode) { + pPage->isDirty = 0; + tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + if (pTxn->jPageSet) { + hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); + } + + tdbPCacheRelease(pPager->pCache, pPage, pTxn); + } + + return 0; +} +*/ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { SPage *pPage; int ret; @@ -700,8 +716,9 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) { // TDB_PAGE_PGNO(pPage), pPage); } -int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) { - int code = 0; +int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { + int code = 0; + SPgno pgno = TDB_PAGE_PGNO(pPage); // tdbError("tdb/insert-free-page: tbc get page: %d.", pgno); code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn); @@ -710,6 +727,8 @@ int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) { return -1; } + pPage->pPager = NULL; + return code; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 7b08da4ca8..879e6a3a49 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -199,7 +199,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn); int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg, TXN *pTxn); void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); -int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn); +int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn); // int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); int tdbPagerRestoreJournals(SPager *pPager); int tdbPagerRollback(SPager *pPager); diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp index 2f2dd0659b..b4391c4a8c 100644 --- a/source/libs/tdb/test/tdbPageRecycleTest.cpp +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -690,3 +690,100 @@ TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_again) { insertOfp(); system("ls -l ./tdb"); } + +// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp_nocommit) { +TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) { + clearDb("tdb"); + insertOfp(); + system("ls -l ./tdb"); + + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + { // delete the data + char const *key = "key123456789"; + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // 1, insert nData kv + { + int nData = nDataConst; + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + /* + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + */ + for (int iData = 0; iData < nData; ++iData) { + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); + GTEST_ASSERT_EQ(ret, 0); + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + } + + /* + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + char const *key = "key123456789"; + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + */ + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} From 8e491c307fec09033b0ceb7b0cda68f7be58de00 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 30 Jun 2023 08:32:39 +0800 Subject: [PATCH 14/34] tdb/recyle-ofp: nullize ofp's pager to mark uninitailized --- source/libs/tdb/src/db/tdbBtree.c | 37 ++++++++++++++++++++++--------- source/libs/tdb/src/db/tdbPager.c | 5 +++-- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 6921a26f19..382c25bfd5 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1320,6 +1320,10 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, return -1; } + if (pDecoder->ofps) { + taosArrayPush(pDecoder->ofps, &ofp); + } + ofpCell = tdbPageGetCell(ofp, 0); if (nLeft <= ofp->maxLocal - sizeof(SPgno)) { @@ -1354,11 +1358,16 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, int lastKeyPageSpace = 0; // load left key & val to ovpages while (pgno != 0) { + tdbTrace("tdb decode-ofp, pTxn: %p, pgno:%u by cell:%p", pTxn, pgno, pCell); ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { return -1; } + if (pDecoder->ofps) { + taosArrayPush(pDecoder->ofps, &ofp); + } + ofpCell = tdbPageGetCell(ofp, 0); int lastKeyPage = 0; @@ -1567,27 +1576,21 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * bytes = ofp->maxLocal - sizeof(SPgno); } - SPgno origPgno = pgno; + // SPgno origPgno = pgno; memcpy(&pgno, ofpCell + bytes, sizeof(pgno)); - + /* ret = tdbPagerWrite(pBt->pPager, ofp); if (ret < 0) { tdbError("failed to write page since %s", terrstr()); return -1; } - // tdbPageDropCell(ofp, 0, pTxn, pBt); - // tdbPageZero(ofp, sizeof(SLeafHdr), tdbBtreeCellSize); - // tdbPageZero(ofp, sizeof(SIntHdr), tdbBtreeCellSize); + tdbPageDropCell(ofp, 0, pTxn, pBt); + */ // SIntHdr *pIntHdr = (SIntHdr *)(ofp->pData); // pIntHdr->flags = TDB_FLAG_ADD(0, TDB_BTREE_OVFL); // pIntHdr->pgno = 0; // ofp->pPager = NULL; - tdbPagerInsertFreePage(pBt->pPager, ofp, pTxn); - - // printf("tdb recycle, pTxn: %p, pgno:%u\n", pTxn, pgno); - tdbTrace("tdb recycle, pTxn: %p, pgno:%u", pTxn, origPgno); - tdbPagerReturnPage(pPage->pPager, ofp, pTxn); nLeft -= bytes; @@ -2100,6 +2103,9 @@ int tdbBtcDelete(SBTC *pBtc) { return -1; } + // btc.coder.ofps = taosArrayInit(8, sizeof(SPgno)); + pBtc->coder.ofps = taosArrayInit(8, sizeof(SPage *)); + tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt); // update interior page or do balance @@ -2155,6 +2161,17 @@ int tdbBtcDelete(SBTC *pBtc) { } } + SArray *ofps = pBtc->coder.ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + // tdbPagerInsertFreePage(pBtc->pBt->pPager, ofp, pBtc->pTxn); + } + + taosArrayDestroy(ofps); + pBtc->coder.ofps = NULL; + } + return 0; } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 62702cbf40..5bfcdfa344 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -720,7 +720,8 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { int code = 0; SPgno pgno = TDB_PAGE_PGNO(pPage); - // tdbError("tdb/insert-free-page: tbc get page: %d.", pgno); + // memset(pPage->pData, 0, pPage->pageSize); + tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno); code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn); if (code < 0) { tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); @@ -763,7 +764,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) { } *pPgno = *(SPgno *)pKey; - // tdbError("tdb/remove-free-page: tbc get page: %d.", *pPgno); + tdbTrace("tdb/remove-free-page: tbc get page: %d.", *pPgno); code = tdbTbcDelete(pCur); if (code < 0) { From f89b43b64ca16e966f05b4da8e99d00c798de9af Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 30 Jun 2023 10:05:17 +0800 Subject: [PATCH 15/34] tdb/ofp-recycle: new ofps list with pager --- source/libs/tdb/src/db/tdbBtree.c | 40 +++++++++++++++------ source/libs/tdb/src/db/tdbPager.c | 2 ++ source/libs/tdb/src/inc/tdbInt.h | 1 + source/libs/tdb/test/tdbPageRecycleTest.cpp | 21 ----------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 382c25bfd5..8ffb5cd43e 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -234,7 +234,9 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { tdbBtcOpen(&btc, pBt, pTxn); /* - btc.coder.ofps = taosArrayInit(8, sizeof(SPgno)); + btc.coder.ofps = taosArrayInit(8, sizeof(SPage *)); + // btc.coder.ofps = taosArrayInit(8, sizeof(SPgno)); + //pBtc->coder.ofps = taosArrayInit(8, sizeof(SPage *)); */ tdbTrace("tdb delete, btc: %p, pTxn: %p", &btc, pTxn); @@ -256,7 +258,18 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { tdbBtcClose(&btc); return -1; } + /* + SArray *ofps = btc.coder.ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(btc.pBt->pPager, ofp, btc.pTxn); + } + taosArrayDestroy(ofps); + btc.coder.ofps = NULL; + } + */ tdbBtcClose(&btc); return 0; } @@ -1319,11 +1332,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, if (ret < 0) { return -1; } - + /* if (pDecoder->ofps) { taosArrayPush(pDecoder->ofps, &ofp); } - + */ ofpCell = tdbPageGetCell(ofp, 0); if (nLeft <= ofp->maxLocal - sizeof(SPgno)) { @@ -1363,11 +1376,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, if (ret < 0) { return -1; } - + /* if (pDecoder->ofps) { taosArrayPush(pDecoder->ofps, &ofp); } - + */ ofpCell = tdbPageGetCell(ofp, 0); int lastKeyPage = 0; @@ -1578,12 +1591,13 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * // SPgno origPgno = pgno; memcpy(&pgno, ofpCell + bytes, sizeof(pgno)); - /* + ret = tdbPagerWrite(pBt->pPager, ofp); if (ret < 0) { tdbError("failed to write page since %s", terrstr()); return -1; } + /* tdbPageDropCell(ofp, 0, pTxn, pBt); */ // SIntHdr *pIntHdr = (SIntHdr *)(ofp->pData); @@ -1591,6 +1605,11 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * // pIntHdr->pgno = 0; // ofp->pPager = NULL; + SArray *ofps = pPage->pPager->ofps; + if (ofps) { + taosArrayPush(ofps, &ofp); + } + tdbPagerReturnPage(pPage->pPager, ofp, pTxn); nLeft -= bytes; @@ -2103,8 +2122,7 @@ int tdbBtcDelete(SBTC *pBtc) { return -1; } - // btc.coder.ofps = taosArrayInit(8, sizeof(SPgno)); - pBtc->coder.ofps = taosArrayInit(8, sizeof(SPage *)); + pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *)); tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt); @@ -2161,15 +2179,15 @@ int tdbBtcDelete(SBTC *pBtc) { } } - SArray *ofps = pBtc->coder.ofps; + SArray *ofps = pBtc->pPage->pPager->ofps; if (ofps) { for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { SPage *ofp = *(SPage **)taosArrayGet(ofps, i); - // tdbPagerInsertFreePage(pBtc->pBt->pPager, ofp, pBtc->pTxn); + tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn); } taosArrayDestroy(ofps); - pBtc->coder.ofps = NULL; + pBtc->pPage->pPager->ofps = NULL; } return 0; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 5bfcdfa344..469416cd1b 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -722,6 +722,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) { // memset(pPage->pData, 0, pPage->pageSize); tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno); + // printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno); code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn); if (code < 0) { tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code); @@ -765,6 +766,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) { *pPgno = *(SPgno *)pKey; tdbTrace("tdb/remove-free-page: tbc get page: %d.", *pPgno); + // printf("tdb/remove-free-page: tbc get page: %d.\n", *pPgno); code = tdbTbcDelete(pCur); if (code < 0) { diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 879e6a3a49..8defe54868 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -407,6 +407,7 @@ struct SPager { SRBTree rbt; // u8 inTran; TXN *pActiveTxn; + SArray *ofps; SPager *pNext; // used by TDB SPager *pHashNext; // used by TDB #ifdef USE_MAINDB diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp index b4391c4a8c..40208f5070 100644 --- a/source/libs/tdb/test/tdbPageRecycleTest.cpp +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -730,15 +730,7 @@ TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) { char key[64]; char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) int64_t poolLimit = 4096; // 1M pool limit - /* - SPoolMem *pPool; - // open the pool - pPool = openPool(); - - // start a transaction - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - */ for (int iData = 0; iData < nData; ++iData) { sprintf(key, "key%03d", iData); sprintf(val, "value%03d", iData); @@ -759,19 +751,6 @@ TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) { } } - /* - // generate value payload - // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) - char val[32605]; - int valLen = sizeof(val) / sizeof(val[0]); - generateBigVal(val, valLen); - - // insert the generated big data - // char const *key = "key1"; - char const *key = "key123456789"; - ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); - GTEST_ASSERT_EQ(ret, 0); - */ // commit current transaction tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); From 317a7c83c743f312b2d3bf51f7bc87461adffa7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Jul 2023 10:55:45 +0800 Subject: [PATCH 16/34] fix(stream): fix message lost bug during pause stream. --- source/libs/stream/src/streamExec.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 46290c306f..bf6fcaced7 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -381,12 +381,12 @@ int32_t streamExecForAll(SStreamTask* pTask) { } } - if (streamTaskShouldStop(&pTask->status)) { - if (pInput) { - streamFreeQitem(pInput); - } - return 0; - } +// if (streamTaskShouldStop(&pTask->status)) { +// if (pInput) { +// streamFreeQitem(pInput); +// } +// return 0; +// } if (pInput == NULL) { break; From 5d4efe11980f6a954b9752144803c94ff2aa5ad0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Jul 2023 11:46:05 +0800 Subject: [PATCH 17/34] fix(stream): fix error in pause. --- source/libs/stream/src/streamExec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index bf6fcaced7..9fddcf9155 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -335,6 +335,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { while (1) { if (streamTaskShouldPause(&pTask->status)) { + qDebug("s-task:%s task should pause, input blocks:%s", pTask->id.idStr, batchSize); if (batchSize > 1) { break; } else { From f84bfc96fb5f1ffa30de4d17a98d04b09e246849 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Jul 2023 14:30:11 +0800 Subject: [PATCH 18/34] fix(stream): fix error in pause in stream. --- source/libs/stream/src/streamExec.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 9fddcf9155..9a53ffa088 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -325,7 +325,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { const char* id = pTask->id.idStr; while (1) { - int32_t batchSize = 1; + int32_t batchSize = 0; int16_t times = 0; SStreamQueueItem* pInput = NULL; @@ -335,8 +335,8 @@ int32_t streamExecForAll(SStreamTask* pTask) { while (1) { if (streamTaskShouldPause(&pTask->status)) { - qDebug("s-task:%s task should pause, input blocks:%s", pTask->id.idStr, batchSize); - if (batchSize > 1) { + qDebug("s-task:%s task should pause, input blocks:%d", pTask->id.idStr, batchSize); + if (batchSize > 0) { break; } else { return 0; @@ -357,6 +357,8 @@ int32_t streamExecForAll(SStreamTask* pTask) { } if (pInput == NULL) { + batchSize += 1; + pInput = qItem; streamQueueProcessSuccess(pTask->inputQueue); if (pTask->taskLevel == TASK_LEVEL__SINK) { @@ -364,18 +366,20 @@ int32_t streamExecForAll(SStreamTask* pTask) { } } else { // todo we need to sort the data block, instead of just appending into the array list. - void* newRet = NULL; - if ((newRet = streamMergeQueueItem(pInput, qItem)) == NULL) { + ASSERT(batchSize >= 1); + + void* newRet = streamMergeQueueItem(pInput, qItem); + if (newRet == NULL) { streamQueueProcessFail(pTask->inputQueue); break; } else { - batchSize++; + batchSize += 1; + pInput = newRet; streamQueueProcessSuccess(pTask->inputQueue); - if (batchSize > MAX_STREAM_EXEC_BATCH_NUM) { - qDebug("s-task:%s batch size limit:%d reached, start to process blocks", id, - MAX_STREAM_EXEC_BATCH_NUM); + if (batchSize >= MAX_STREAM_EXEC_BATCH_NUM) { + qDebug("s-task:%s batch size limit:%d reached, start to process blocks", id, MAX_STREAM_EXEC_BATCH_NUM); break; } } @@ -390,6 +394,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { // } if (pInput == NULL) { + ASSERT(batchSize == 0); break; } From ceb06635fe37557dac47600e56c31777ae109b10 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Jul 2023 15:12:35 +0800 Subject: [PATCH 19/34] fix(stream): fix error in pause stream. --- source/libs/stream/src/streamExec.c | 121 +++++++++++++--------------- 1 file changed, 57 insertions(+), 64 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 9a53ffa088..bfbaed7bf6 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -317,6 +317,60 @@ int32_t updateCheckPointInfo(SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } +static int32_t extractMsgFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks, + const char* id) { + int32_t retryTimes = 0; + int32_t MAX_RETRY_TIMES = 5; + + while (1) { + if (streamTaskShouldPause(&pTask->status)) { + qDebug("s-task:%s task should pause, input blocks:%d", pTask->id.idStr, *numOfBlocks); + return TSDB_CODE_SUCCESS; + } + + SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); + if (qItem == NULL) { + if (pTask->taskLevel == TASK_LEVEL__SOURCE && (++retryTimes) < MAX_RETRY_TIMES) { + taosMsleep(10); + qDebug("===stream===try again batchSize:%d, retry:%d", *numOfBlocks, retryTimes); + continue; + } + + qDebug("===stream===break batchSize:%d", *numOfBlocks); + return TSDB_CODE_SUCCESS; + } + + // do not merge blocks for sink node + if (pTask->taskLevel == TASK_LEVEL__SINK) { + *numOfBlocks = 1; + *pInput = qItem; + return TSDB_CODE_SUCCESS; + } + + if (pInput == NULL) { + ASSERT((*numOfBlocks) == 0); + *pInput = qItem; + } else { + // todo we need to sort the data block, instead of just appending into the array list. + void* newRet = streamMergeQueueItem(*pInput, qItem); + if (newRet == NULL) { + qError("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d", id, *numOfBlocks); + streamQueueProcessFail(pTask->inputQueue); + return TSDB_CODE_SUCCESS; + } + pInput = newRet; + } + + *numOfBlocks += 1; + streamQueueProcessSuccess(pTask->inputQueue); + + if (*numOfBlocks >= MAX_STREAM_EXEC_BATCH_NUM) { + qDebug("s-task:%s batch size limit:%d reached, start to process blocks", id, MAX_STREAM_EXEC_BATCH_NUM); + return TSDB_CODE_SUCCESS; + } + } +} + /** * todo: the batch of blocks should be tuned dynamic, according to the total elapsed time of each batch of blocks, the * appropriate batch of blocks should be handled in 5 to 10 sec. @@ -326,73 +380,12 @@ int32_t streamExecForAll(SStreamTask* pTask) { while (1) { int32_t batchSize = 0; - int16_t times = 0; - SStreamQueueItem* pInput = NULL; // merge multiple input data if possible in the input queue. qDebug("s-task:%s start to extract data block from inputQ", id); - while (1) { - if (streamTaskShouldPause(&pTask->status)) { - qDebug("s-task:%s task should pause, input blocks:%d", pTask->id.idStr, batchSize); - if (batchSize > 0) { - break; - } else { - return 0; - } - } - - SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); - if (qItem == NULL) { - if (pTask->taskLevel == TASK_LEVEL__SOURCE && batchSize < MIN_STREAM_EXEC_BATCH_NUM && times < 5) { - times++; - taosMsleep(10); - qDebug("===stream===try again batchSize:%d", batchSize); - continue; - } - - qDebug("===stream===break batchSize:%d", batchSize); - break; - } - - if (pInput == NULL) { - batchSize += 1; - - pInput = qItem; - streamQueueProcessSuccess(pTask->inputQueue); - if (pTask->taskLevel == TASK_LEVEL__SINK) { - break; - } - } else { - // todo we need to sort the data block, instead of just appending into the array list. - ASSERT(batchSize >= 1); - - void* newRet = streamMergeQueueItem(pInput, qItem); - if (newRet == NULL) { - streamQueueProcessFail(pTask->inputQueue); - break; - } else { - batchSize += 1; - - pInput = newRet; - streamQueueProcessSuccess(pTask->inputQueue); - - if (batchSize >= MAX_STREAM_EXEC_BATCH_NUM) { - qDebug("s-task:%s batch size limit:%d reached, start to process blocks", id, MAX_STREAM_EXEC_BATCH_NUM); - break; - } - } - } - } - -// if (streamTaskShouldStop(&pTask->status)) { -// if (pInput) { -// streamFreeQitem(pInput); -// } -// return 0; -// } - + /*int32_t code = */extractMsgFromInputQ(pTask, &pInput, &batchSize, id); if (pInput == NULL) { ASSERT(batchSize == 0); break; @@ -409,8 +402,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { while (pTask->taskLevel == TASK_LEVEL__SOURCE) { int8_t status = atomic_load_8(&pTask->status.taskStatus); if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__PAUSE) { - qError("stream task wait for the end of fill history, s-task:%s, status:%d", id, - atomic_load_8(&pTask->status.taskStatus)); + qError("stream task wait for the end of fill history, s-task:%s, status:%d", id, status); taosMsleep(100); } else { break; @@ -463,6 +455,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { double el = (taosGetTimestampMs() - st) / 1000.0; qDebug("s-task:%s batch of input blocks exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", id, el, resSize / 1048576.0, totalBlocks); + streamFreeQitem(pInput); } From 9e62b9d0d21a45e9d284302187867af7560266af Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Jul 2023 15:33:08 +0800 Subject: [PATCH 20/34] fix(stream): fix error in extract data from inputQ. --- source/libs/stream/src/streamExec.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index bfbaed7bf6..847ec3f159 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -347,7 +347,7 @@ static int32_t extractMsgFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu return TSDB_CODE_SUCCESS; } - if (pInput == NULL) { + if (*pInput == NULL) { ASSERT((*numOfBlocks) == 0); *pInput = qItem; } else { @@ -358,7 +358,8 @@ static int32_t extractMsgFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu streamQueueProcessFail(pTask->inputQueue); return TSDB_CODE_SUCCESS; } - pInput = newRet; + + *pInput = newRet; } *numOfBlocks += 1; From dbf47f5fc176d5c46a13fda13210019accc78baa Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Mon, 3 Jul 2023 06:39:47 -0400 Subject: [PATCH 21/34] fix: ttl fill cache only in initialization --- source/dnode/vnode/src/inc/metaTtl.h | 2 +- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/meta/metaCommit.c | 4 -- source/dnode/vnode/src/meta/metaOpen.c | 85 ++++++++++++------------ source/dnode/vnode/src/meta/metaTtl.c | 6 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 9 ++- 6 files changed, 56 insertions(+), 51 deletions(-) diff --git a/source/dnode/vnode/src/inc/metaTtl.h b/source/dnode/vnode/src/inc/metaTtl.h index bf3b897c6f..428f4438b6 100644 --- a/source/dnode/vnode/src/inc/metaTtl.h +++ b/source/dnode/vnode/src/inc/metaTtl.h @@ -81,7 +81,7 @@ typedef struct { int ttlMgrOpen(STtlManger** ppTtlMgr, TDB* pEnv, int8_t rollback); int ttlMgrClose(STtlManger* pTtlMgr); -int ttlMgrBegin(STtlManger* pTtlMgr, void* pMeta); +int ttlMgrPostOpen(STtlManger* pTtlMgr, void* pMeta); int ttlMgrConvert(TTB* pOldTtlIdx, TTB* pNewTtlIdx, void* pMeta); int ttlMgrFlush(STtlManger* pTtlMgr, TXN* pTxn); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index a9541d8c47..54bbeaea88 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -136,6 +136,7 @@ typedef struct STbUidStore STbUidStore; #define META_BEGIN_HEAP_NIL 2 int metaOpen(SVnode* pVnode, SMeta** ppMeta, int8_t rollback); +int metaPostOpen(SVnode* pVnode, SMeta** ppMeta); // for operations depend on "meta txn" int metaClose(SMeta** pMeta); int metaBegin(SMeta* pMeta, int8_t fromSys); TXN* metaGetTxn(SMeta* pMeta); diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index 1fa5b9c1e9..d262567953 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -40,10 +40,6 @@ int metaBegin(SMeta *pMeta, int8_t heap) { return -1; } - if (ttlMgrBegin(pMeta->pTtlMgr, pMeta) < 0) { - return -1; - } - tdbCommit(pMeta->pEnv, pMeta->txn); return 0; diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index fb17aff318..7469ddfcc3 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -29,6 +29,8 @@ static int ncolIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); } static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&pMeta->lock); } +static void metaCleanup(SMeta **ppMeta); + int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { SMeta *pMeta = NULL; int ret; @@ -180,51 +182,26 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { return 0; _err: - if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb); - if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx); - if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx); - if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); - if (pMeta->pTtlMgr) ttlMgrClose(pMeta->pTtlMgr); - if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); - if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); - if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); - if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); - if (pMeta->pEnv) tdbClose(pMeta->pEnv); - metaDestroyLock(pMeta); - taosMemoryFree(pMeta); + metaCleanup(&pMeta); + return -1; +} + +int metaPostOpen(SVnode *pVnode, SMeta **ppMeta) { + SMeta *pMeta = *ppMeta; + if (ttlMgrPostOpen(pMeta->pTtlMgr, pMeta) < 0) { + metaError("vgId:%d, failed to post open meta ttl since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + return 0; + +_err: + metaCleanup(ppMeta); return -1; } int metaClose(SMeta **ppMeta) { - SMeta *pMeta = *ppMeta; - if (pMeta) { - if (pMeta->pEnv) metaAbort(pMeta); - if (pMeta->pCache) metaCacheClose(pMeta); - if (pMeta->pIdx) metaCloseIdx(pMeta); - if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb); - if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx); - if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx); - if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); - if (pMeta->pTtlMgr) ttlMgrClose(pMeta->pTtlMgr); - if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); - if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); - if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); - if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); - if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); - if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); - if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); - if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); - if (pMeta->pEnv) tdbClose(pMeta->pEnv); - metaDestroyLock(pMeta); - - taosMemoryFreeClear(*ppMeta); - } - + metaCleanup(ppMeta); return 0; } @@ -270,6 +247,32 @@ int32_t metaULock(SMeta *pMeta) { return ret; } +static void metaCleanup(SMeta **ppMeta) { + SMeta *pMeta = *ppMeta; + if (pMeta) { + if (pMeta->pEnv) metaAbort(pMeta); + if (pMeta->pCache) metaCacheClose(pMeta); + if (pMeta->pIdx) metaCloseIdx(pMeta); + if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb); + if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx); + if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx); + if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); + if (pMeta->pTtlMgr) ttlMgrClose(pMeta->pTtlMgr); + if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); + if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); + if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); + if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); + if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); + if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); + if (pMeta->pTbDb) tdbTbClose(pMeta->pTbDb); + if (pMeta->pEnv) tdbClose(pMeta->pEnv); + metaDestroyLock(pMeta); + + taosMemoryFreeClear(*ppMeta); + } +} + static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { STbDbKey *pTbDbKey1 = (STbDbKey *)pKey1; STbDbKey *pTbDbKey2 = (STbDbKey *)pKey2; diff --git a/source/dnode/vnode/src/meta/metaTtl.c b/source/dnode/vnode/src/meta/metaTtl.c index af4827a9c7..7aecf1f203 100644 --- a/source/dnode/vnode/src/meta/metaTtl.c +++ b/source/dnode/vnode/src/meta/metaTtl.c @@ -79,8 +79,8 @@ int ttlMgrClose(STtlManger *pTtlMgr) { return 0; } -int ttlMgrBegin(STtlManger *pTtlMgr, void *pMeta) { - metaInfo("ttl mgr start open"); +int ttlMgrPostOpen(STtlManger *pTtlMgr, void *pMeta) { + metaInfo("ttl mgr start post open"); int ret; int64_t startNs = taosGetTimestampNs(); @@ -112,7 +112,7 @@ int ttlMgrBegin(STtlManger *pTtlMgr, void *pMeta) { int64_t endNs = taosGetTimestampNs(); - metaInfo("ttl mgr open end, hash size: %d, time consumed: %" PRId64 " ns", taosHashGetSize(pTtlMgr->pTtlCache), + metaInfo("ttl mgr post open end, hash size: %d, time consumed: %" PRId64 " ns", taosHashGetSize(pTtlMgr->pTtlCache), endNs - startNs); _out: return ret; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 583df15533..22750af1c7 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -76,7 +76,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *p } SSyncCfg *pCfg = &info.config.syncCfg; - + pCfg->replicaNum = 0; pCfg->totalReplicaNum = 0; memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo)); @@ -109,7 +109,7 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *p pCfg->myIndex = pReq->replica + pReq->learnerSelfIndex; } - vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d", + vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->totalReplicaNum, pCfg->myIndex); info.config.syncCfg = *pCfg; @@ -416,6 +416,11 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { goto _err; } + if (metaPostOpen(pVnode, &pVnode->pMeta) < 0) { + vError("vgId:%d, failed to post open vnode meta since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + // open sync if (vnodeSyncOpen(pVnode, dir)) { vError("vgId:%d, failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno)); From c66524d87bc7f6f4bc7bee29ae4b5fe7ef42ecbc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 5 Jul 2023 15:33:37 +0800 Subject: [PATCH 22/34] tdb/ofp: recycle ofp cell on parent page --- source/libs/tdb/src/db/tdbBtree.c | 75 +++++++++++++++++---- source/libs/tdb/test/tdbPageRecycleTest.cpp | 67 ++++++++++++++++++ 2 files changed, 129 insertions(+), 13 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8ffb5cd43e..08e61c2272 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -608,7 +608,30 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx for (int i = 0; i < nOlds; i++) { nCells = TDB_PAGE_TOTAL_CELLS(pParent); if (sIdx < nCells) { + bool destroyOfps = false; + if (!childNotLeaf) { + if (!pParent->pPager->ofps) { + pParent->pPager->ofps = taosArrayInit(8, sizeof(SPage *)); + destroyOfps = true; + } + } + tdbPageDropCell(pParent, sIdx, pTxn, pBt); + + if (!childNotLeaf) { + SArray *ofps = pParent->pPager->ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(pParent->pPager, ofp, pTxn); + } + + if (destroyOfps) { + taosArrayDestroy(ofps); + pParent->pPager->ofps = NULL; + } + } + } } else { ((SIntHdr *)pParent->pData)->pgno = 0; } @@ -1372,6 +1395,7 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, // load left key & val to ovpages while (pgno != 0) { tdbTrace("tdb decode-ofp, pTxn: %p, pgno:%u by cell:%p", pTxn, pgno, pCell); + // printf("tdb decode-ofp, pTxn: %p, pgno:%u by cell:%p\n", pTxn, pgno, pCell); ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { return -1; @@ -2122,10 +2146,27 @@ int tdbBtcDelete(SBTC *pBtc) { return -1; } - pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *)); + bool destroyOfps = false; + if (!pBtc->pPage->pPager->ofps) { + pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *)); + destroyOfps = true; + } tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt); + SArray *ofps = pBtc->pPage->pPager->ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn); + } + + if (destroyOfps) { + taosArrayDestroy(ofps); + pBtc->pPage->pPager->ofps = NULL; + } + } + // update interior page or do balance if (idx == nCells - 1) { if (idx) { @@ -2179,17 +2220,6 @@ int tdbBtcDelete(SBTC *pBtc) { } } - SArray *ofps = pBtc->pPage->pPager->ofps; - if (ofps) { - for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { - SPage *ofp = *(SPage **)taosArrayGet(ofps, i); - tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn); - } - - taosArrayDestroy(ofps); - pBtc->pPage->pPager->ofps = NULL; - } - return 0; } @@ -2250,7 +2280,13 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int tdbError("tdb/btc-upsert: page insert/update cell failed with ret: %d.", ret); return -1; } - + /* + bool destroyOfps = false; + if (!pBtc->pPage->pPager->ofps) { + pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *)); + destroyOfps = true; + } + */ // check balance if (pBtc->pPage->nOverflow > 0) { ret = tdbBtreeBalance(pBtc); @@ -2259,7 +2295,20 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int return -1; } } + /* + SArray *ofps = pBtc->pPage->pPager->ofps; + if (ofps) { + for (int i = 0; i < TARRAY_SIZE(ofps); ++i) { + SPage *ofp = *(SPage **)taosArrayGet(ofps, i); + tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn); + } + if (destroyOfps) { + taosArrayDestroy(ofps); + pBtc->pPage->pPager->ofps = NULL; + } + } + */ return 0; } diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp index 40208f5070..4d7b314917 100644 --- a/source/libs/tdb/test/tdbPageRecycleTest.cpp +++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp @@ -766,3 +766,70 @@ TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) { system("ls -l ./tdb"); } + +// TEST(TdbPageRecycleTest, DISABLED_recycly_delete_interior_ofp_nocommit) { +TEST(TdbPageRecycleTest, recycly_delete_interior_ofp_nocommit) { + clearDb("tdb"); + + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = NULL; // tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + char key[1024] = {0}; + int count = sizeof(key) / sizeof(key[0]); + for (int i = 0; i < count - 1; ++i) { + key[i] = 'a'; + } + + // insert n ofp keys to form 2-layer btree + { + for (int i = 0; i < 7; ++i) { + // sprintf(&key[count - 2], "%c", i); + key[count - 2] = '0' + i; + + ret = tdbTbInsert(pDb, key, count, NULL, NULL, txn); + GTEST_ASSERT_EQ(ret, 0); + } + } + /* + // delete one interior key + { + sprintf(&key[count - 2], "%c", 2); + key[count - 2] = '0' + 2; + + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + */ + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + ret = tdbClose(pEnv); + GTEST_ASSERT_EQ(ret, 0); + + system("ls -l ./tdb"); +} From 03dbcbf05587dd736d794827c5cbf2aa8bc46224 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 5 Jul 2023 15:55:55 +0800 Subject: [PATCH 23/34] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 33 ++-- source/dnode/snode/src/snode.c | 6 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tq/tq.c | 186 ++++++------------ source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/executor/src/executor.c | 29 +-- .../stream/inc/{streamInc.h => streamInt.h} | 2 +- source/libs/stream/src/stream.c | 2 +- source/libs/stream/src/streamBackendRocksdb.c | 2 +- source/libs/stream/src/streamData.c | 2 +- source/libs/stream/src/streamDispatch.c | 8 +- source/libs/stream/src/streamExec.c | 18 +- source/libs/stream/src/streamMeta.c | 2 +- source/libs/stream/src/streamQueue.c | 2 +- source/libs/stream/src/streamRecover.c | 77 +++++--- source/libs/stream/src/streamState.c | 2 +- 16 files changed, 164 insertions(+), 211 deletions(-) rename source/libs/stream/inc/{streamInc.h => streamInt.h} (97%) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index dbcc31a35e..b48992b5f2 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -459,10 +459,10 @@ typedef struct { int64_t streamId; int32_t taskId; int32_t childId; -} SStreamRecoverFinishReq, SStreamTransferReq; +} SStreamScanHistoryFinishReq, SStreamTransferReq; -int32_t tEncodeStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq); -int32_t tDecodeStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq); +int32_t tEncodeStreamScanHistoryFinishReq(SEncoder* pEncoder, const SStreamScanHistoryFinishReq* pReq); +int32_t tDecodeStreamScanHistoryFinishReq(SDecoder* pDecoder, SStreamScanHistoryFinishReq* pReq); typedef struct { int64_t streamId; @@ -537,8 +537,8 @@ int32_t tDecodeStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) int32_t tEncodeStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp); int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp); -int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq); -int32_t tDecodeSStreamTaskRecoverReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq); +int32_t tEncodeSStreamTaskScanHistoryReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq); +int32_t tDecodeSStreamTaskScanHistoryReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq); int32_t tEncodeSStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamRecoverDownstreamRsp* pRsp); int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamRecoverDownstreamRsp* pRsp); @@ -578,15 +578,17 @@ int32_t streamTaskCheckDownstreamTasks(SStreamTask* pTask); int32_t streamTaskLaunchScanHistory(SStreamTask* pTask); int32_t streamTaskCheckStatus(SStreamTask* pTask); int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp); -int32_t streamCheckHistoryTaskDownstrem(SStreamTask* pTask); +int32_t streamCheckHistoryTaskDownstream(SStreamTask* pTask); int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask); int32_t streamStartRecoverTask(SStreamTask* pTask, int8_t igUntreated); +void streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask); + bool streamTaskRecoverScanStep1Finished(SStreamTask* pTask); bool streamTaskRecoverScanStep2Finished(SStreamTask* pTask); int32_t streamTaskRecoverSetAllStepFinished(SStreamTask* pTask); // common -int32_t streamSetParamForScanHistoryData(SStreamTask* pTask); +int32_t streamSetParamForScanHistory(SStreamTask* pTask); int32_t streamRestoreParam(SStreamTask* pTask); int32_t streamSetStatusNormal(SStreamTask* pTask); const char* streamGetTaskStatusStr(int32_t status); @@ -596,32 +598,29 @@ int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange* p int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow); int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated); int32_t streamSourceScanHistoryData(SStreamTask* pTask); -// int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver); int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask); int32_t streamDispatchTransferStateMsg(SStreamTask* pTask); // agg level -int32_t streamAggRecoverPrepare(SStreamTask* pTask); -int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t taskId, int32_t childId); +int32_t streamAggScanHistoryPrepare(SStreamTask* pTask); +int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, int32_t taskId, int32_t childId); +// stream task meta void streamMetaInit(); void streamMetaCleanup(); SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId); void streamMetaClose(SStreamMeta* streamMeta); - -int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); -int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); -int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen); -int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta); - +int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); +int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); +int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen); +int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta); // todo remove it SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); int32_t streamMetaBegin(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); -int32_t streamMetaRollBack(SStreamMeta* pMeta); int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); // checkpoint diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 3d9adf8156..e4bc184be3 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -279,11 +279,11 @@ int32_t sndProcessTaskRecoverFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); // deserialize - SStreamRecoverFinishReq req; + SStreamScanHistoryFinishReq req; SDecoder decoder; tDecoderInit(&decoder, msg, msgLen); - tDecodeStreamRecoverFinishReq(&decoder, &req); + tDecodeStreamScanHistoryFinishReq(&decoder, &req); tDecoderClear(&decoder); // find task @@ -292,7 +292,7 @@ int32_t sndProcessTaskRecoverFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { return -1; } // do process request - if (streamProcessRecoverFinishReq(pTask, req.taskId, req.childId) < 0) { + if (streamProcessScanHistoryFinishReq(pTask, req.taskId, req.childId) < 0) { streamMetaReleaseTask(pSnode->pMeta, pTask); return -1; } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7c6c72e995..6b403ca498 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -248,7 +248,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskTransferStateReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessStreamTaskScanHistoryFinishReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqCheckLogInWal(STQ* pTq, int64_t version); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9702779470..bf0632958f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1071,13 +1071,6 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { return -1; } - // check param - int64_t fillVer1 = pTask->chkInfo.version; - if (fillVer1 <= 0) { - streamMetaReleaseTask(pMeta, pTask); - return -1; - } - // do recovery step 1 const char* pId = pTask->id.idStr; tqDebug("s-task:%s start history data scan stage(step 1), status:%s", pId, @@ -1091,7 +1084,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { return 0; } - if (!pReq->igUntreated && !streamTaskRecoverScanStep1Finished(pTask)) { + if (!streamTaskRecoverScanStep1Finished(pTask)) { streamSourceScanHistoryData(pTask); } @@ -1121,39 +1114,23 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { // wait for the stream task get ready for scan history data while (((pStreamTask->status.downstreamReady == 0) && (pStreamTask->status.taskStatus != TASK_STATUS__STOP)) || pStreamTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) { - tqDebug( - "s-task:%s level:%d related stream task:%s not ready for halt, wait for it continue and recheck in 100ms", - pTask->id.idStr, pTask->info.taskLevel, pStreamTask->id.idStr); + tqDebug("s-task:%s level:%d related stream task:%s not ready for halt, wait for it and recheck in 100ms", pId, + pTask->info.taskLevel, pId); taosMsleep(100); } // now we can stop the stream task execution pStreamTask->status.taskStatus = TASK_STATUS__HALT; - tqDebug("s-task:%s level:%d status is set to halt by history scan task:%s", pStreamTask->id.idStr, + tqDebug("s-task:%s level:%d status is set to halt by history scan task:%s", pId, pStreamTask->info.taskLevel, pId); // if it's an source task, extract the last version in wal. - pRange = &pTask->dataRange.range; - int64_t latestVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader); - ASSERT(latestVer >= pRange->maxVer); - - int64_t nextStartVer = pRange->maxVer + 1; - if (nextStartVer > latestVer - 1) { - // no input data yet. no need to execute the secondardy scan while stream task halt - streamTaskRecoverSetAllStepFinished(pTask); - tqDebug("s-task:%s no need to perform secondary scan-history-data(step 2), since no data ingest during secondary scan", pId); - } else { - // 2. do secondary scan of the history data, the time window remain, and the version range is updated to - // [pTask->dataRange.range.maxVer, ver1] - pRange->minVer = nextStartVer; - pRange->maxVer = latestVer - 1; - } + streamHistoryTaskSetVerRangeStep2(pTask); } if (!streamTaskRecoverScanStep1Finished(pTask)) { - tqDebug("s-task:%s level:%d verRange:%" PRId64 " - %" PRId64 - " do secondary scan-history-data after halt the related stream task:%s", - pId, pTask->info.taskLevel, pRange->minVer, pRange->maxVer, pStreamTask->id.idStr); + tqDebug("s-task:%s level:%d verRange:%" PRId64 " - %" PRId64 " do secondary scan-history-data after halt the related stream task:%s", + pId, pTask->info.taskLevel, pRange->minVer, pRange->maxVer, pId); ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__WAITING); st = taosGetTimestampMs(); @@ -1162,6 +1139,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { if (!streamTaskRecoverScanStep2Finished(pTask)) { streamSourceScanHistoryData(pTask); + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING || streamTaskShouldPause(&pTask->status)) { tqDebug("s-task:%s is dropped or paused, abort recover in step1", pId); streamMetaReleaseTask(pMeta, pTask); @@ -1174,7 +1152,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { el = (taosGetTimestampMs() - st) / 1000.0; tqDebug("s-task:%s history data scan stage(step 2) ended, elapsed time:%.2fs", pId, el); - // 3. notify the downstream tasks to transfer executor state after handle all history blocks. + // 3. notify downstream tasks to transfer executor state after handle all history blocks. if (!pTask->status.transferState) { code = streamDispatchTransferStateMsg(pTask); if (code != TSDB_CODE_SUCCESS) { @@ -1210,14 +1188,16 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { if (pTask->historyTaskId.taskId == 0) { *pWindow = (STimeWindow){INT64_MIN, INT64_MAX}; - tqDebug("s-task:%s no associated task, reset the time window:%" PRId64 " - %" PRId64, pId, pWindow->skey, - pWindow->ekey); + tqDebug("s-task:%s no related scan-history-data task, reset the time window:%" PRId64 " - %" PRId64, pId, + pWindow->skey, pWindow->ekey); } else { - tqDebug("s-task:%s history data scan completed, now start to scan data from wal, start ver:%" PRId64 - ", window:%" PRId64 " - %" PRId64, - pId, pTask->chkInfo.currentVer, pWindow->skey, pWindow->ekey); + tqDebug( + "s-task:%s history data in current time window scan completed, now start to handle data from WAL, start " + "ver:%" PRId64 ", window:%" PRId64 " - %" PRId64, + pId, pTask->chkInfo.currentVer, pWindow->skey, pWindow->ekey); } + // notify the downstream agg tasks that upstream tasks are ready to processing the WAL data, update the code = streamTaskScanHistoryDataComplete(pTask); streamMetaReleaseTask(pMeta, pTask); @@ -1238,7 +1218,7 @@ int32_t tqProcessTaskTransferStateReq(STQ* pTq, int64_t sversion, char* msg, int SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - int32_t code = tDecodeStreamRecoverFinishReq(&decoder, &req); + int32_t code = tDecodeStreamScanHistoryFinishReq(&decoder, &req); tDecoderClear(&decoder); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId); @@ -1251,61 +1231,17 @@ int32_t tqProcessTaskTransferStateReq(STQ* pTq, int64_t sversion, char* msg, int streamTaskReleaseState(pTask); tqDebug("s-task:%s receive state transfer req", pTask->id.idStr); + // related stream task load the state from the state storage backend SStreamTask* pStreamTask = streamMetaAcquireTask(pTq->pStreamMeta, pTask->streamTaskId.taskId); + if (pStreamTask == NULL) { + tqError("failed to find related stream task:0x%x, it may have been dropped already", req.taskId); + return -1; + } + streamTaskReloadState(pStreamTask); ASSERT(pTask->streamTaskId.taskId != 0); - pTask->status.transferState = true; // persistent data? - -#if 0 - // do check if current task handle all data in the input queue - int64_t st = taosGetTimestampMs(); - tqDebug("s-task:%s start step2 recover, ts:%" PRId64, pTask->id.idStr, st); - - code = streamSourceRecoverScanStep2(pTask, sversion); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; - } - - qDebug("s-task:%s set start wal scan start ver:%"PRId64, pTask->id.idStr, sversion); - - walReaderSeekVer(pTask->exec.pWalReader, sversion); - pTask->chkInfo.currentVer = sversion; - - if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return 0; - } - - // restore param - code = streamRestoreParam(pTask); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; - } - - // set status normal - tqDebug("s-task:%s blocking stage completed, set the status to be normal", pTask->id.idStr); - code = streamSetStatusNormal(pTask); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; - } - - double el = (taosGetTimestampMs() - st) / 1000.0; - tqDebug("s-task:%s step2 recover finished, el:%.2fs", pTask->id.idStr, el); - - // dispatch recover finish req to all related downstream task - code = streamDispatchScanHistoryFinishMsg(pTask); - if (code < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return -1; - } - - atomic_store_8(&pTask->info.fillHistory, 0); - streamMetaSaveTask(pTq->pStreamMeta, pTask); -#endif + pTask->status.transferState = true; streamSchedExec(pTask); streamMetaReleaseTask(pTq->pStreamMeta, pTask); @@ -1313,31 +1249,28 @@ int32_t tqProcessTaskTransferStateReq(STQ* pTq, int64_t sversion, char* msg, int return 0; } -int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg) { +int32_t tqProcessStreamTaskScanHistoryFinishReq(STQ* pTq, SRpcMsg* pMsg) { char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); // deserialize - SStreamRecoverFinishReq req; + SStreamScanHistoryFinishReq req = {0}; SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - tDecodeStreamRecoverFinishReq(&decoder, &req); + tDecodeStreamScanHistoryFinishReq(&decoder, &req); tDecoderClear(&decoder); // find task SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId); if (pTask == NULL) { - return -1; - } - // do process request - if (streamProcessRecoverFinishReq(pTask, req.taskId, req.childId) < 0) { - streamMetaReleaseTask(pTq->pStreamMeta, pTask); + tqError("failed to find task:0x%x, it may be destroyed, vgId:%d", req.taskId, pTq->pStreamMeta->vgId); return -1; } + int32_t code = streamProcessScanHistoryFinishReq(pTask, req.taskId, req.childId); streamMetaReleaseTask(pTq->pStreamMeta, pTask); - return 0; + return code; } int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg) { @@ -1423,10 +1356,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { pTask->chkInfo.version); streamProcessRunReq(pTask); } else { -// if (streamTaskShouldPause(&pTask->status)) { - atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); -// } - + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, pTask->id.idStr, streamGetTaskStatusStr(pTask->status.taskStatus), pTask->status.schedStatus); } @@ -1517,34 +1447,35 @@ int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, int8_t igUntreated) { int32_t vgId = pTq->pStreamMeta->vgId; - if (pTask) { - if (streamTaskShouldPause(&pTask->status)) { - atomic_store_8(&pTask->status.taskStatus, pTask->status.keepTaskStatus); - - // no lock needs to secure the access of the version - if (igUntreated && pTask->info.taskLevel == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) { - // discard all the data when the stream task is suspended. - walReaderSetSkipToVersion(pTask->exec.pWalReader, sversion); - tqDebug("vgId:%d s-task:%s resume to exec, prev paused version:%" PRId64 ", start from vnode ver:%" PRId64 - ", schedStatus:%d", - vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); - } else { // from the previous paused version and go on - tqDebug("vgId:%d s-task:%s resume to exec, from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", - vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); - } - - if (pTask->info.fillHistory && pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - streamStartRecoverTask(pTask, igUntreated); - } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE && taosQueueItemSize(pTask->inputQueue->queue) == 0) { - tqStartStreamTasks(pTq); - } else { - streamSchedExec(pTask); - } - } - streamMetaReleaseTask(pTq->pStreamMeta, pTask); - } else { + if (pTask == NULL) { return -1; } + + if (streamTaskShouldPause(&pTask->status)) { + atomic_store_8(&pTask->status.taskStatus, pTask->status.keepTaskStatus); + + // no lock needs to secure the access of the version + if (igUntreated && pTask->info.taskLevel == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) { + // discard all the data when the stream task is suspended. + walReaderSetSkipToVersion(pTask->exec.pWalReader, sversion); + tqDebug("vgId:%d s-task:%s resume to exec, prev paused version:%" PRId64 ", start from vnode ver:%" PRId64 + ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); + } else { // from the previous paused version and go on + tqDebug("vgId:%d s-task:%s resume to exec, from paused ver:%" PRId64 ", vnode ver:%" PRId64 ", schedStatus:%d", + vgId, pTask->id.idStr, pTask->chkInfo.currentVer, sversion, pTask->status.schedStatus); + } + + if (pTask->info.fillHistory && pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + streamStartRecoverTask(pTask, igUntreated); + } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE && taosQueueItemSize(pTask->inputQueue->queue) == 0) { + tqStartStreamTasks(pTq); + } else { + streamSchedExec(pTask); + } + } + + streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } @@ -1560,6 +1491,7 @@ int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms if (pHistoryTask) { code = tqProcessTaskResumeImpl(pTq, pHistoryTask, sversion, pReq->igUntreated); } + return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 40bca57827..88a260b3a3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -665,7 +665,7 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) return tqProcessTaskTransferStateReq(pVnode->pTq, 0, pReq, len); } case TDMT_STREAM_SCAN_HISTORY_FINISH: - return tqProcessTaskRecoverFinishReq(pVnode->pTq, pMsg); + return tqProcessStreamTaskScanHistoryFinishReq(pVnode->pTq, pMsg); case TDMT_STREAM_SCAN_HISTORY_FINISH_RSP: return tqProcessTaskRecoverFinishRsp(pVnode->pTq, pMsg); default: diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a3d94a0891..900505acb3 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -911,8 +911,8 @@ int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRan pStreamInfo->recoverStep1Finished = true; pStreamInfo->recoverStep2Finished = false; - qDebug("%s step 2. set param for stream scanner for scan history data, verRange:%" PRId64 " - %" PRId64 ", window:%" PRId64 - " - %" PRId64, + qDebug("%s step 2. set param for stream scanner for scan history data, verRange:%" PRId64 " - %" PRId64 + ", window:%" PRId64 " - %" PRId64, GET_TASKID(pTaskInfo), pStreamInfo->fillHistoryVer.minVer, pStreamInfo->fillHistoryVer.maxVer, pWindow->skey, pWindow->ekey); return 0; @@ -999,31 +999,35 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) { int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + const char* id = GET_TASKID(pTaskInfo); SOperatorInfo* pOperator = pTaskInfo->pRoot; while (1) { - if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { + uint16_t type = pOperator->operatorType; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; - qInfo("restore stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); - } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || - pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { + qInfo("%s restore stream agg executors param for interval: %d, %" PRId64, id, pInfo->twAggSup.calTrigger, + pInfo->twAggSup.deleteMark); + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || + type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; - qInfo("restore stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); - } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { + qInfo("%s restore stream agg executor param for session: %d, %" PRId64, id, pInfo->twAggSup.calTrigger, + pInfo->twAggSup.deleteMark); + } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; - qInfo("restore stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + qInfo("%s restore stream agg executor param for state: %d, %" PRId64, id, pInfo->twAggSup.calTrigger, + pInfo->twAggSup.deleteMark); } // iterate operator tree @@ -1037,7 +1041,6 @@ int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo) { pOperator = pOperator->pDownstream[0]; } } - return 0; } bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) { diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInt.h similarity index 97% rename from source/libs/stream/inc/streamInc.h rename to source/libs/stream/inc/streamInt.h index eec37d7dbb..2164b63caf 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInt.h @@ -49,7 +49,7 @@ int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* p int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pData); int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); -int32_t streamDoDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, +int32_t streamDoDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamScanHistoryFinishReq* pReq, int32_t vgId, SEpSet* pEpSet); SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 691d31e64c..ddbc8da3ec 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" #include "ttimer.h" #define STREAM_TASK_INPUT_QUEUE_CAPACITY 20480 diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 4646af641f..18ec80e87a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -16,7 +16,7 @@ #include "streamBackendRocksdb.h" #include "executor.h" #include "query.h" -#include "streamInc.h" +#include "streamInt.h" #include "tcommon.h" #include "tref.h" diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 37923ca807..bad104bc8e 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" SStreamDataBlock* createStreamDataFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg) { SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, pReq->totalLen); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index d93de7b1e5..9241df2e70 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ +#include "streamInt.h" #include "ttimer.h" -#include "streamInc.h" #define MAX_BLOCK_NAME_NUM 1024 #define DISPATCH_RETRY_INTERVAL_MS 300 @@ -276,14 +276,14 @@ int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pR return 0; } -int32_t streamDoDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, +int32_t streamDoDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamScanHistoryFinishReq* pReq, int32_t vgId, SEpSet* pEpSet) { void* buf = NULL; int32_t code = -1; SRpcMsg msg = {0}; int32_t tlen; - tEncodeSize(tEncodeStreamRecoverFinishReq, pReq, tlen, code); + tEncodeSize(tEncodeStreamScanHistoryFinishReq, pReq, tlen, code); if (code < 0) { return -1; } @@ -299,7 +299,7 @@ int32_t streamDoDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamRe SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); - if ((code = tEncodeStreamRecoverFinishReq(&encoder, pReq)) < 0) { + if ((code = tEncodeStreamScanHistoryFinishReq(&encoder, pReq)) < 0) { if (buf) { rpcFreeCont(buf); } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 6e1804b08e..76f0d41a28 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" // maximum allowed processed block batches. One block may include several submit blocks #define MAX_STREAM_EXEC_BATCH_NUM 32 @@ -358,7 +358,8 @@ static int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { ASSERT(pStreamTask != NULL && pStreamTask->historyTaskId.taskId == pTask->id.taskId); STimeWindow* pTimeWindow = &pStreamTask->dataRange.window; - // here we need to wait for the stream task handle all data in the input queue. + // It must be halted for a source stream task, since when the related scan-history-data task start scan the history + // for the step 2. For a agg task if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { ASSERT(pStreamTask->status.taskStatus == TASK_STATUS__HALT); } else { @@ -369,21 +370,18 @@ static int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { // wait for the stream task to be idle waitForTaskIdle(pTask, pStreamTask); + // In case of sink tasks, no need to be halted for them. + // In case of source tasks and agg tasks, we should HALT them, and wait for them to be idle. And then, it's safe to + // start the task state transfer procedure. + // When a task is idle with halt status, all data in inputQ are consumed. if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { // update the scan data range for source task. qDebug("s-task:%s level:%d stream task window %" PRId64 " - %" PRId64 " update to %" PRId64 " - %" PRId64 ", status:%s, sched-status:%d", pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN, pTimeWindow->ekey, streamGetTaskStatusStr(TASK_STATUS__NORMAL), pStreamTask->status.schedStatus); - - // todo transfer state } else { - // for sink tasks, they are continue to execute, no need to be halt. - // the process should be stopped for a while, during the term of transfer task state. - // OR wait for the inputQ && outputQ of agg tasks are all consumed, and then start the state transfer - qDebug("s-task:%s no need to update time window, for non-source task", pStreamTask->id.idStr); - - // todo transfer state + qDebug("s-task:%s no need to update time window for non-source task", pStreamTask->id.idStr); } // expand the query time window for stream scanner diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 8242f84312..e1f625dd52 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -15,7 +15,7 @@ #include "executor.h" #include "streamBackendRocksdb.h" -#include "streamInc.h" +#include "streamInt.h" #include "tref.h" #include "ttimer.h" diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 4cfeedab57..aaf9fdec72 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" SStreamQueue* streamQueueOpen(int64_t cap) { SStreamQueue* pQueue = taosMemoryCalloc(1, sizeof(SStreamQueue)); diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 9ded58597f..a3fc3418aa 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "streamInc.h" +#include "streamInt.h" #include "ttimer.h" #include "wal.h" @@ -53,7 +53,7 @@ static int32_t doLaunchScanHistoryTask(SStreamTask* pTask) { qDebug("s-task:%s vgId:%d status:%s, start scan-history-data task, verRange:%" PRId64 " - %" PRId64, pTask->id.idStr, pTask->info.nodeId, streamGetTaskStatusStr(pTask->status.taskStatus), pRange->minVer, pRange->maxVer); - streamSetParamForScanHistoryData(pTask); + streamSetParamForScanHistory(pTask); streamSetParamForStreamScannerStep1(pTask, pRange, &pTask->dataRange.window); int32_t code = streamStartRecoverTask(pTask, 0); @@ -72,8 +72,8 @@ int32_t streamTaskLaunchScanHistory(SStreamTask* pTask) { } } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) { streamSetStatusNormal(pTask); - streamSetParamForScanHistoryData(pTask); - streamAggRecoverPrepare(pTask); + streamSetParamForScanHistory(pTask); + streamAggScanHistoryPrepare(pTask); } else if (pTask->info.taskLevel == TASK_LEVEL__SINK) { streamSetStatusNormal(pTask); qDebug("s-task:%s sink task convert to normal immediately", pTask->id.idStr); @@ -202,10 +202,10 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs pTask->checkReqIds = NULL; if (pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) { - qDebug("s-task:%s all %d downstream tasks are ready, now enter into scan-history-data stage, status:%s", id, numOfReqs, - streamGetTaskStatusStr(pTask->status.taskStatus)); + qDebug("s-task:%s all %d downstream tasks are ready, now enter into scan-history-data stage, status:%s", id, + numOfReqs, streamGetTaskStatusStr(pTask->status.taskStatus)); streamTaskLaunchScanHistory(pTask); - } else { + } else { // todo add assert, agg tasks? ASSERT(pTask->status.taskStatus == TASK_STATUS__NORMAL); qDebug("s-task:%s fixed downstream task is ready, now ready for data from wal, status:%s", id, streamGetTaskStatusStr(pTask->status.taskStatus)); @@ -215,7 +215,8 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs qDebug("s-task:%s (vgId:%d) recv check rsp from task:0x%x (vgId:%d) status:%d, total:%d not ready:%d", id, pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status, total, left); } - } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + } else { + ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH); if (pRsp->reqId != pTask->checkReqId) { return -1; } @@ -233,8 +234,6 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs qDebug("s-task:%s fixed downstream task is ready, ready for data from inputQ, status:%s", id, streamGetTaskStatusStr(pTask->status.taskStatus)); } - } else { - ASSERT(0); } } else { // not ready, wait for 100ms and retry qDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, wait for 100ms and retry", id, pRsp->downstreamTaskId, @@ -248,7 +247,7 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs } // common -int32_t streamSetParamForScanHistoryData(SStreamTask* pTask) { +int32_t streamSetParamForScanHistory(SStreamTask* pTask) { qDebug("s-task:%s set operator option for scan-history-data", pTask->id.idStr); return qSetStreamOperatorOptionForScanHistory(pTask->exec.pExecutor); } @@ -286,7 +285,7 @@ int32_t streamSourceScanHistoryData(SStreamTask* pTask) { } int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask) { - SStreamRecoverFinishReq req = { .streamId = pTask->id.streamId, .childId = pTask->info.selfChildId }; + SStreamScanHistoryFinishReq req = { .streamId = pTask->id.streamId, .childId = pTask->info.selfChildId }; // serialize if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { @@ -314,7 +313,7 @@ static int32_t doDispatchTransferMsg(SStreamTask* pTask, const SStreamTransferRe SRpcMsg msg = {0}; int32_t tlen; - tEncodeSize(tEncodeStreamRecoverFinishReq, pReq, tlen, code); + tEncodeSize(tEncodeStreamScanHistoryFinishReq, pReq, tlen, code); if (code < 0) { return -1; } @@ -330,7 +329,7 @@ static int32_t doDispatchTransferMsg(SStreamTask* pTask, const SStreamTransferRe SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); - if ((code = tEncodeStreamRecoverFinishReq(&encoder, pReq)) < 0) { + if ((code = tEncodeStreamScanHistoryFinishReq(&encoder, pReq)) < 0) { if (buf) { rpcFreeCont(buf); } @@ -375,7 +374,7 @@ int32_t streamDispatchTransferStateMsg(SStreamTask* pTask) { } // agg -int32_t streamAggRecoverPrepare(SStreamTask* pTask) { +int32_t streamAggScanHistoryPrepare(SStreamTask* pTask) { pTask->numOfWaitingUpstream = taosArrayGetSize(pTask->pUpstreamEpInfoList); qDebug("s-task:%s agg task is ready and wait for %d upstream tasks complete scan-history procedure", pTask->id.idStr, pTask->numOfWaitingUpstream); @@ -391,19 +390,19 @@ int32_t streamAggUpstreamScanHistoryFinish(SStreamTask* pTask) { if (qStreamRecoverFinish(exec) < 0) { return -1; } - -// streamSetStatusNormal(pTask); return 0; } -int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t taskId, int32_t childId) { +int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, int32_t taskId, int32_t childId) { if (pTask->info.taskLevel == TASK_LEVEL__AGG) { int32_t left = atomic_sub_fetch_32(&pTask->numOfWaitingUpstream, 1); ASSERT(left >= 0); if (left == 0) { int32_t numOfTasks = taosArrayGetSize(pTask->pUpstreamEpInfoList); - qDebug("s-task:%s all %d upstream tasks finish scan-history data", pTask->id.idStr, numOfTasks); + qDebug("s-task:%s all %d upstream tasks finish scan-history data, set param for agg task for stream data", + pTask->id.idStr, numOfTasks); + streamAggUpstreamScanHistoryFinish(pTask); } else { qDebug("s-task:%s receive scan-history data finish msg from upstream:0x%x(index:%d), unfinished:%d", @@ -411,6 +410,7 @@ int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t taskId, int32_ } } + return 0; } @@ -467,8 +467,8 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) { if (pHTask == NULL && (!streamTaskShouldStop(&pTask->status))) { const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); qWarn( - "s-task:%s vgId:%d status:%s failed to launch history task:0x%x, since it may not be built, or have been " - "destroyed, or should stop exec", + "s-task:%s vgId:%d status:%s failed to launch history task:0x%x, since it may not be built, or may have been " + "destroyed, or should stop", pTask->id.idStr, pMeta->vgId, pStatus, pTask->historyTaskId.taskId); taosTmrReset(tryLaunchHistoryTask, 100, pInfo, streamEnv.timer, &pTask->launchTaskTimer); @@ -493,7 +493,7 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) { // todo fix the bug: 2. race condition // an fill history task needs to be started. -int32_t streamCheckHistoryTaskDownstrem(SStreamTask* pTask) { +int32_t streamCheckHistoryTaskDownstream(SStreamTask* pTask) { SStreamMeta* pMeta = pTask->pMeta; int32_t hTaskId = pTask->historyTaskId.taskId; @@ -573,6 +573,27 @@ int32_t streamTaskRecoverSetAllStepFinished(SStreamTask* pTask) { return qStreamRecoverSetAllStepFinished(exec); } +void streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask) { + SVersionRange* pRange = &pTask->dataRange.range; + int64_t latestVer = walReaderGetCurrentVer(pTask->exec.pWalReader); + ASSERT(latestVer >= pRange->maxVer); + + int64_t nextStartVer = pRange->maxVer + 1; + if (nextStartVer > latestVer - 1) { + // no input data yet. no need to execute the secondardy scan while stream task halt + streamTaskRecoverSetAllStepFinished(pTask); + qDebug( + "s-task:%s no need to perform secondary scan-history-data(step 2), since no data ingest during secondary scan", + pTask->id.idStr); + } else { + // 2. do secondary scan of the history data, the time window remain, and the version range is updated to + // [pTask->dataRange.range.maxVer, ver1] + pRange->minVer = nextStartVer; + pRange->maxVer = latestVer - 1; + } +} + + int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->reqId) < 0) return -1; @@ -627,7 +648,7 @@ int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) return 0; } -int32_t tEncodeStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq) { +int32_t tEncodeStreamScanHistoryFinishReq(SEncoder* pEncoder, const SStreamScanHistoryFinishReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; @@ -635,7 +656,7 @@ int32_t tEncodeStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFi tEndEncode(pEncoder); return pEncoder->pos; } -int32_t tDecodeStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq) { +int32_t tDecodeStreamScanHistoryFinishReq(SDecoder* pDecoder, SStreamScanHistoryFinishReq* pReq) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; @@ -652,12 +673,12 @@ void streamPrepareNdoCheckDownstream(SStreamTask* pTask) { // calculate the correct start time window, and start the handle the history data for the main task. if (pTask->historyTaskId.taskId != 0) { // check downstream tasks for associated scan-history-data tasks - streamCheckHistoryTaskDownstrem(pTask); + streamCheckHistoryTaskDownstream(pTask); // launch current task SHistDataRange* pRange = &pTask->dataRange; - int64_t ekey = pRange->window.ekey + 1; - int64_t ver = pRange->range.minVer; + int64_t ekey = pRange->window.ekey + 1; + int64_t ver = pRange->range.minVer; pRange->window.skey = ekey; pRange->window.ekey = INT64_MAX; @@ -665,7 +686,7 @@ void streamPrepareNdoCheckDownstream(SStreamTask* pTask) { pRange->range.maxVer = ver; qDebug("s-task:%s level:%d fill-history task exists, update stream time window:%" PRId64 " - %" PRId64 - ", ver range:%" PRId64 " - %" PRId64, + ", ver range:%" PRId64 " - %" PRId64, pTask->id.idStr, pTask->info.taskLevel, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer); } else { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 9873e7b4c8..0a4f73a67c 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -18,7 +18,7 @@ #include "osMemory.h" #include "rocksdb/c.h" #include "streamBackendRocksdb.h" -#include "streamInc.h" +#include "streamInt.h" #include "tcoding.h" #include "tcommon.h" #include "tcompare.h" From e6e52c3641252a69142474b9bc75992dfb2fbaa0 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 5 Jul 2023 15:58:09 +0800 Subject: [PATCH 24/34] fix: ttlmgr convert in metaUpgrade --- source/dnode/vnode/src/inc/metaTtl.h | 12 +-- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/meta/metaOpen.c | 29 +++++-- source/dnode/vnode/src/meta/metaTtl.c | 103 +++++++++++++++---------- source/dnode/vnode/src/vnd/vnodeOpen.c | 9 +-- 5 files changed, 96 insertions(+), 59 deletions(-) diff --git a/source/dnode/vnode/src/inc/metaTtl.h b/source/dnode/vnode/src/inc/metaTtl.h index 428f4438b6..a3d3ceab24 100644 --- a/source/dnode/vnode/src/inc/metaTtl.h +++ b/source/dnode/vnode/src/inc/metaTtl.h @@ -79,16 +79,18 @@ typedef struct { TXN* pTxn; } STtlDelTtlCtx; -int ttlMgrOpen(STtlManger** ppTtlMgr, TDB* pEnv, int8_t rollback); -int ttlMgrClose(STtlManger* pTtlMgr); -int ttlMgrPostOpen(STtlManger* pTtlMgr, void* pMeta); +int ttlMgrOpen(STtlManger** ppTtlMgr, TDB* pEnv, int8_t rollback); +void ttlMgrClose(STtlManger* pTtlMgr); +int ttlMgrPostOpen(STtlManger* pTtlMgr, void* pMeta); -int ttlMgrConvert(TTB* pOldTtlIdx, TTB* pNewTtlIdx, void* pMeta); -int ttlMgrFlush(STtlManger* pTtlMgr, TXN* pTxn); +bool ttlMgrNeedUpgrade(TDB* pEnv); +int ttlMgrUpgrade(STtlManger* pTtlMgr, void* pMeta); int ttlMgrInsertTtl(STtlManger* pTtlMgr, const STtlUpdTtlCtx* pUpdCtx); int ttlMgrDeleteTtl(STtlManger* pTtlMgr, const STtlDelTtlCtx* pDelCtx); int ttlMgrUpdateChangeTime(STtlManger* pTtlMgr, const STtlUpdCtimeCtx* pUpdCtimeCtx); + +int ttlMgrFlush(STtlManger* pTtlMgr, TXN* pTxn); int ttlMgrFindExpired(STtlManger* pTtlMgr, int64_t timePointMs, SArray* pTbUids); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 54bbeaea88..cbf0933358 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -136,7 +136,7 @@ typedef struct STbUidStore STbUidStore; #define META_BEGIN_HEAP_NIL 2 int metaOpen(SVnode* pVnode, SMeta** ppMeta, int8_t rollback); -int metaPostOpen(SVnode* pVnode, SMeta** ppMeta); // for operations depend on "meta txn" +int metaUpgrade(SVnode* pVnode, SMeta** ppMeta); int metaClose(SMeta** pMeta); int metaBegin(SMeta* pMeta, int8_t fromSys); TXN* metaGetTxn(SMeta* pMeta); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 7469ddfcc3..511cc8d6ec 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -186,18 +186,35 @@ _err: return -1; } -int metaPostOpen(SVnode *pVnode, SMeta **ppMeta) { +int metaUpgrade(SVnode *pVnode, SMeta **ppMeta) { + int code = TSDB_CODE_SUCCESS; SMeta *pMeta = *ppMeta; - if (ttlMgrPostOpen(pMeta->pTtlMgr, pMeta) < 0) { - metaError("vgId:%d, failed to post open meta ttl since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; + + if (ttlMgrNeedUpgrade(pMeta->pEnv)) { + code = metaBegin(pMeta, META_BEGIN_HEAP_OS); + if (code < 0) { + metaError("vgId:%d, failed to upgrade meta, meta begin failed since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + code = ttlMgrUpgrade(pMeta->pTtlMgr, pMeta); + if (code < 0) { + metaError("vgId:%d, failed to upgrade meta ttl since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + + code = metaCommit(pMeta, pMeta->txn); + if (code < 0) { + metaError("vgId:%d, failed to upgrade meta ttl, meta commit failed since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } } - return 0; + return TSDB_CODE_SUCCESS; _err: metaCleanup(ppMeta); - return -1; + return code; } int metaClose(SMeta **ppMeta) { diff --git a/source/dnode/vnode/src/meta/metaTtl.c b/source/dnode/vnode/src/meta/metaTtl.c index 7aecf1f203..c6cb826149 100644 --- a/source/dnode/vnode/src/meta/metaTtl.c +++ b/source/dnode/vnode/src/meta/metaTtl.c @@ -21,6 +21,10 @@ typedef struct { SMeta *pMeta; } SConvertData; +static void ttlMgrCleanup(STtlManger *pTtlMgr); + +static int ttlMgrConvert(TTB *pOldTtlIdx, TTB *pNewTtlIdx, void *pMeta); + static void ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid); static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int ttlIdxKeyV1Cmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); @@ -36,27 +40,17 @@ const char *ttlTbname = "ttl.idx"; const char *ttlV1Tbname = "ttlv1.idx"; int ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback) { - int ret; + int ret = TSDB_CODE_SUCCESS; + int64_t startNs = taosGetTimestampNs(); *ppTtlMgr = NULL; STtlManger *pTtlMgr = (STtlManger *)tdbOsCalloc(1, sizeof(*pTtlMgr)); - if (pTtlMgr == NULL) { - return -1; - } - - if (tdbTbExist(ttlTbname, pEnv)) { - ret = tdbTbOpen(ttlTbname, sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pEnv, &pTtlMgr->pOldTtlIdx, rollback); - if (ret < 0) { - metaError("failed to open %s index since %s", ttlTbname, tstrerror(terrno)); - return ret; - } - } + if (pTtlMgr == NULL) return TSDB_CODE_OUT_OF_MEMORY; ret = tdbTbOpen(ttlV1Tbname, TDB_VARIANT_LEN, TDB_VARIANT_LEN, ttlIdxKeyV1Cmpr, pEnv, &pTtlMgr->pTtlIdx, rollback); if (ret < 0) { metaError("failed to open %s since %s", ttlV1Tbname, tstrerror(terrno)); - tdbOsFree(pTtlMgr); return ret; } @@ -66,42 +60,57 @@ int ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback) { taosThreadRwlockInit(&pTtlMgr->lock, NULL); + ret = ttlMgrFillCache(pTtlMgr); + if (ret < 0) { + metaError("failed to fill hash since %s", tstrerror(terrno)); + ttlMgrCleanup(pTtlMgr); + return ret; + } + + int64_t endNs = taosGetTimestampNs(); + metaInfo("ttl mgr open end, hash size: %d, time consumed: %" PRId64 " ns", taosHashGetSize(pTtlMgr->pTtlCache), + endNs - startNs); + *ppTtlMgr = pTtlMgr; - return 0; + return TSDB_CODE_SUCCESS; } -int ttlMgrClose(STtlManger *pTtlMgr) { - taosHashCleanup(pTtlMgr->pTtlCache); - taosHashCleanup(pTtlMgr->pDirtyUids); - tdbTbClose(pTtlMgr->pTtlIdx); - taosThreadRwlockDestroy(&pTtlMgr->lock); - tdbOsFree(pTtlMgr); - return 0; +void ttlMgrClose(STtlManger *pTtlMgr) { ttlMgrCleanup(pTtlMgr); } + +bool ttlMgrNeedUpgrade(TDB *pEnv) { + bool needUpgrade = tdbTbExist(ttlTbname, pEnv); + if (needUpgrade) { + metaInfo("find ttl idx in old version , will convert"); + } + return needUpgrade; } -int ttlMgrPostOpen(STtlManger *pTtlMgr, void *pMeta) { - metaInfo("ttl mgr start post open"); - int ret; +int ttlMgrUpgrade(STtlManger *pTtlMgr, void *pMeta) { + SMeta *meta = (SMeta *)pMeta; + int ret = TSDB_CODE_SUCCESS; + + if (!tdbTbExist(ttlTbname, meta->pEnv)) return TSDB_CODE_SUCCESS; + + metaInfo("ttl mgr start upgrade"); int64_t startNs = taosGetTimestampNs(); - SMeta *meta = (SMeta *)pMeta; + ret = tdbTbOpen(ttlTbname, sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, meta->pEnv, &pTtlMgr->pOldTtlIdx, 0); + if (ret < 0) { + metaError("failed to open %s index since %s", ttlTbname, tstrerror(terrno)); + goto _out; + } - if (pTtlMgr->pOldTtlIdx) { - ret = ttlMgrConvert(pTtlMgr->pOldTtlIdx, pTtlMgr->pTtlIdx, pMeta); - if (ret < 0) { - metaError("failed to convert ttl index since %s", tstrerror(terrno)); - goto _out; - } + ret = ttlMgrConvert(pTtlMgr->pOldTtlIdx, pTtlMgr->pTtlIdx, pMeta); + if (ret < 0) { + metaError("failed to convert ttl index since %s", tstrerror(terrno)); + goto _out; + } - ret = tdbTbDropByName(ttlTbname, meta->pEnv, meta->txn); - if (ret < 0) { - metaError("failed to drop old ttl index since %s", tstrerror(terrno)); - goto _out; - } - - tdbTbClose(pTtlMgr->pOldTtlIdx); - pTtlMgr->pOldTtlIdx = NULL; + ret = tdbTbDropByName(ttlTbname, meta->pEnv, meta->txn); + if (ret < 0) { + metaError("failed to drop old ttl index since %s", tstrerror(terrno)); + goto _out; } ret = ttlMgrFillCache(pTtlMgr); @@ -111,13 +120,23 @@ int ttlMgrPostOpen(STtlManger *pTtlMgr, void *pMeta) { } int64_t endNs = taosGetTimestampNs(); - - metaInfo("ttl mgr post open end, hash size: %d, time consumed: %" PRId64 " ns", taosHashGetSize(pTtlMgr->pTtlCache), + metaInfo("ttl mgr upgrade end, hash size: %d, time consumed: %" PRId64 " ns", taosHashGetSize(pTtlMgr->pTtlCache), endNs - startNs); _out: + tdbTbClose(pTtlMgr->pOldTtlIdx); + pTtlMgr->pOldTtlIdx = NULL; + return ret; } +static void ttlMgrCleanup(STtlManger *pTtlMgr) { + taosHashCleanup(pTtlMgr->pTtlCache); + taosHashCleanup(pTtlMgr->pDirtyUids); + tdbTbClose(pTtlMgr->pTtlIdx); + taosThreadRwlockDestroy(&pTtlMgr->lock); + tdbOsFree(pTtlMgr); +} + static void ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid) { if (ttlDays <= 0) return; @@ -205,7 +224,7 @@ _out: return ret; } -int ttlMgrConvert(TTB *pOldTtlIdx, TTB *pNewTtlIdx, void *pMeta) { +static int ttlMgrConvert(TTB *pOldTtlIdx, TTB *pNewTtlIdx, void *pMeta) { SMeta *meta = pMeta; metaInfo("ttlMgr convert ttl start."); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 22750af1c7..c794c7ebd6 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -371,6 +371,10 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { goto _err; } + if (metaUpgrade(pVnode, &pVnode->pMeta) < 0) { + vError("vgId:%d, failed to upgrade meta since %s", TD_VID(pVnode), tstrerror(terrno)); + } + // open tsdb if (!VND_IS_RSMA(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL, rollback) < 0) { vError("vgId:%d, failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); @@ -416,11 +420,6 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { goto _err; } - if (metaPostOpen(pVnode, &pVnode->pMeta) < 0) { - vError("vgId:%d, failed to post open vnode meta since %s", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } - // open sync if (vnodeSyncOpen(pVnode, dir)) { vError("vgId:%d, failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno)); From 7a333dafff93144ac589f895f68a690518ef726f Mon Sep 17 00:00:00 2001 From: sunpeng Date: Thu, 6 Jul 2023 09:22:10 +0800 Subject: [PATCH 25/34] docs: fix example in python document --- docs/examples/python/tmq_assignment_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/python/tmq_assignment_example.py b/docs/examples/python/tmq_assignment_example.py index a07347a9b9..9316e14a17 100644 --- a/docs/examples/python/tmq_assignment_example.py +++ b/docs/examples/python/tmq_assignment_example.py @@ -55,4 +55,4 @@ def taos_get_assignment_and_seek_demo(): if __name__ == '__main__': - taosws_get_assignment_and_seek_demo() + taos_get_assignment_and_seek_demo From d425cf33d6d67a2d387c8f934a38c05f4af99546 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Thu, 6 Jul 2023 09:41:33 +0800 Subject: [PATCH 26/34] docs: fix-example-in-python-document --- docs/examples/python/tmq_assignment_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/python/tmq_assignment_example.py b/docs/examples/python/tmq_assignment_example.py index 9316e14a17..41737e3fc4 100644 --- a/docs/examples/python/tmq_assignment_example.py +++ b/docs/examples/python/tmq_assignment_example.py @@ -55,4 +55,4 @@ def taos_get_assignment_and_seek_demo(): if __name__ == '__main__': - taos_get_assignment_and_seek_demo + taos_get_assignment_and_seek_demo() From 4f29e14963e4b6cc55e0a05d2d38a0ddb7e5ddc5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 6 Jul 2023 08:11:46 +0800 Subject: [PATCH 27/34] tsim/sma/drop_sma: sleep 1s before creating db to avoid in dropping --- tests/script/tsim/sma/drop_sma.sim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/tsim/sma/drop_sma.sim b/tests/script/tsim/sma/drop_sma.sim index 0d2712f8db..8fd8ebdcfd 100644 --- a/tests/script/tsim/sma/drop_sma.sim +++ b/tests/script/tsim/sma/drop_sma.sim @@ -129,6 +129,7 @@ sql DROP INDEX sma_index_3 ; print ========== step8 sql drop database if exists db; +sleep 2000 sql create database db duration 300; sql use db; sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); From 1f71ce9409910ff0a5d92c40347ffe851bc434a8 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 6 Jul 2023 10:33:01 +0800 Subject: [PATCH 28/34] feature: get last timestamp before create sma index --- include/common/tmsg.h | 1 + include/libs/nodes/cmdnodes.h | 21 ++++--- source/common/src/tmsg.c | 2 + source/libs/nodes/src/nodesUtilFuncs.c | 5 ++ source/libs/parser/inc/parInt.h | 1 + source/libs/parser/src/parTranslater.c | 64 +++++++++++++++++++-- source/libs/parser/src/parser.c | 2 + source/libs/parser/test/parInitialCTest.cpp | 12 ++++ source/libs/planner/test/planTestUtil.cpp | 10 ++++ 9 files changed, 104 insertions(+), 14 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6e182c1c35..126da5b4e8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3025,6 +3025,7 @@ typedef struct { char* sql; char* ast; int64_t deleteMark; + int64_t lastTs; } SMCreateSmaReq; int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 3ac971344b..bd0b70c310 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -319,19 +319,22 @@ typedef struct SIndexOptions { SNode* pInterval; SNode* pOffset; SNode* pSliding; + int8_t tsPrecision; SNode* pStreamOptions; } SIndexOptions; typedef struct SCreateIndexStmt { - ENodeType type; - EIndexType indexType; - bool ignoreExists; - char indexDbName[TSDB_DB_NAME_LEN]; - char indexName[TSDB_INDEX_NAME_LEN]; - char dbName[TSDB_DB_NAME_LEN]; - char tableName[TSDB_TABLE_NAME_LEN]; - SNodeList* pCols; - SIndexOptions* pOptions; + ENodeType type; + EIndexType indexType; + bool ignoreExists; + char indexDbName[TSDB_DB_NAME_LEN]; + char indexName[TSDB_INDEX_NAME_LEN]; + char dbName[TSDB_DB_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; + SNodeList* pCols; + SIndexOptions* pOptions; + SNode* pPrevQuery; + SMCreateSmaReq* pReq; } SCreateIndexStmt; typedef struct SDropIndexStmt { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index debb93e8ba..adb3dd48c6 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -835,6 +835,7 @@ int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq if (tEncodeBinary(&encoder, pReq->ast, pReq->astLen) < 0) return -1; } if (tEncodeI64(&encoder, pReq->deleteMark) < 0) return -1; + if (tEncodeI64(&encoder, pReq->lastTs) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -884,6 +885,7 @@ int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pR if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } if (tDecodeI64(&decoder, &pReq->deleteMark) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->lastTs) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); return 0; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 15232b95b6..c8197721fb 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -907,6 +907,10 @@ void nodesDestroyNode(SNode* pNode) { SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pNode; nodesDestroyNode((SNode*)pStmt->pOptions); nodesDestroyList(pStmt->pCols); + if (pStmt->pReq) { + tFreeSMCreateSmaReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); + } break; } case QUERY_NODE_DROP_INDEX_STMT: // no pointer field @@ -1053,6 +1057,7 @@ void nodesDestroyNode(SNode* pNode) { } case QUERY_NODE_QUERY: { SQuery* pQuery = (SQuery*)pNode; + nodesDestroyNode(pQuery->pPrevRoot); nodesDestroyNode(pQuery->pRoot); nodesDestroyNode(pQuery->pPostRoot); taosMemoryFreeClear(pQuery->pResSchema); diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index d79aa84bb8..69253e62e2 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -35,6 +35,7 @@ int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMe int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow); +int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8fc4be5f95..942c36d721 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5803,6 +5803,15 @@ static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStm if (TSDB_CODE_SUCCESS == code) { code = getSmaIndexAst(pCxt, pStmt, &pReq->ast, &pReq->astLen, &pReq->expr, &pReq->exprLen); } + if (TSDB_CODE_SUCCESS == code) { + STableMeta* pMetaCache = NULL; + code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + pStmt->pOptions->tsPrecision = pMetaCache->tableInfo.precision; + code = createLastTsSelectStmt(pStmt->dbName, pStmt->tableName, pMetaCache, &pStmt->pPrevQuery); + } + taosMemoryFreeClear(pMetaCache); + } return code; } @@ -5828,15 +5837,60 @@ static int32_t checkCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pS } static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { - SMCreateSmaReq createSmaReq = {0}; int32_t code = checkCreateSmaIndex(pCxt, pStmt); + pStmt->pReq = taosMemoryCalloc(1, sizeof(SMCreateSmaReq)); + if (pStmt->pReq == NULL) code = TSDB_CODE_OUT_OF_MEMORY; if (TSDB_CODE_SUCCESS == code) { - code = buildCreateSmaReq(pCxt, pStmt, &createSmaReq); + code = buildCreateSmaReq(pCxt, pStmt, pStmt->pReq); + } + TSWAP(pCxt->pPrevRoot, pStmt->pPrevQuery); + return code; +} + +int32_t createIntervalFromCreateSmaIndexStmt(SCreateIndexStmt* pStmt, SInterval* pInterval) { + pInterval->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; + pInterval->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; + pInterval->offset = NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0; + pInterval->sliding = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pInterval->interval; + pInterval->slidingUnit = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pInterval->intervalUnit; + pInterval->precision = pStmt->pOptions->tsPrecision; + return TSDB_CODE_SUCCESS; +} + +int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void ** pResRow) { + int32_t code = TSDB_CODE_SUCCESS; + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot; + int64_t lastTs = 0; + SInterval interval = {0}; + STranslateContext pCxt = {0}; + code = initTranslateContext(pParseCxt, NULL, &pCxt); + if (TSDB_CODE_SUCCESS == code) { + code = createIntervalFromCreateSmaIndexStmt(pStmt, &interval); } if (TSDB_CODE_SUCCESS == code) { - code = buildCmdMsg(pCxt, TDMT_MND_CREATE_SMA, (FSerializeFunc)tSerializeSMCreateSmaReq, &createSmaReq); + if (pResRow && pResRow[0]) { + lastTs = *(int64_t*)pResRow[0]; + } else if (interval.interval > 0) { + lastTs = convertTimePrecision(taosGetTimestampMs(), TSDB_TIME_PRECISION_MILLI, interval.precision); + } else { + lastTs = taosGetTimestampMs(); + } } - tFreeSMCreateSmaReq(&createSmaReq); + if (TSDB_CODE_SUCCESS == code) { + if (interval.interval > 0) { + pStmt->pReq->lastTs = taosTimeTruncate(lastTs, &interval); + } else { + pStmt->pReq->lastTs = lastTs; + } + code = buildCmdMsg(&pCxt, TDMT_MND_CREATE_SMA, (FSerializeFunc)tSerializeSMCreateSmaReq, pStmt->pReq); + } + if (TSDB_CODE_SUCCESS == code) { + code = setQuery(&pCxt, pQuery); + } + setRefreshMate(&pCxt, pQuery); + destroyTranslateContext(&pCxt); + tFreeSMCreateSmaReq(pStmt->pReq); + taosMemoryFreeClear(pStmt->pReq); return code; } @@ -6989,7 +7043,7 @@ static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt* return code; } -int32_t buildIntervalForCreateStream(SCreateStreamStmt* pStmt, SInterval* pInterval) { +static int32_t buildIntervalForCreateStream(SCreateStreamStmt* pStmt, SInterval* pInterval) { int32_t code = TSDB_CODE_SUCCESS; if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery)) { return code; diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index cbddaf8115..10fda8741b 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -227,6 +227,8 @@ int32_t qContinueParsePostQuery(SParseContext* pCxt, SQuery* pQuery, void** pRes case QUERY_NODE_CREATE_STREAM_STMT: code = translatePostCreateStream(pCxt, pQuery, pResRow); break; + case QUERY_NODE_CREATE_INDEX_STMT: + code = translatePostCreateSmaIndex(pCxt, pQuery, pResRow); default: break; } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 6d27bb0d29..856fdb4804 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -542,6 +542,18 @@ TEST_F(ParserInitialCTest, createSmaIndex) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_INDEX_STMT); SMCreateSmaReq req = {0}; + ASSERT_TRUE(pQuery->pPrevRoot); + ASSERT_EQ(QUERY_NODE_SELECT_STMT, nodeType(pQuery->pPrevRoot)); + + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot; + SCmdMsgInfo* pCmdMsg = (SCmdMsgInfo*)taosMemoryMalloc(sizeof(SCmdMsgInfo)); + if (NULL == pCmdMsg) FAIL(); + pCmdMsg->msgType = TDMT_MND_CREATE_SMA; + pCmdMsg->msgLen = tSerializeSMCreateSmaReq(NULL, 0, pStmt->pReq); + pCmdMsg->pMsg = taosMemoryMalloc(pCmdMsg->msgLen); + if (!pCmdMsg->pMsg) FAIL(); + tSerializeSMCreateSmaReq(pCmdMsg->pMsg, pCmdMsg->msgLen, pStmt->pReq); + ((SQuery*)pQuery)->pCmdMsg = pCmdMsg; ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); ASSERT_EQ(std::string(req.name), std::string(expect.name)); diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index d89e669a90..3b432b9890 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -441,6 +441,16 @@ class PlannerTestBaseImpl { pCxt->topicQuery = true; } else if (QUERY_NODE_CREATE_INDEX_STMT == nodeType(pQuery->pRoot)) { SMCreateSmaReq req = {0}; + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot; + SCmdMsgInfo* pCmdMsg = (SCmdMsgInfo*)taosMemoryMalloc(sizeof(SCmdMsgInfo)); + if (NULL == pCmdMsg) FAIL(); + pCmdMsg->msgType = TDMT_MND_CREATE_SMA; + pCmdMsg->msgLen = tSerializeSMCreateSmaReq(NULL, 0, pStmt->pReq); + pCmdMsg->pMsg = taosMemoryMalloc(pCmdMsg->msgLen); + if (!pCmdMsg->pMsg) FAIL(); + tSerializeSMCreateSmaReq(pCmdMsg->pMsg, pCmdMsg->msgLen, pStmt->pReq); + ((SQuery*)pQuery)->pCmdMsg = pCmdMsg; + tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); g_mockCatalogService->createSmaIndex(&req); nodesStringToNode(req.ast, &pCxt->pAstRoot); From 2de37b9426a947a2afd0c15690d060cb86f82923 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Thu, 6 Jul 2023 11:23:19 +0800 Subject: [PATCH 29/34] refine select interval from sys table error msg --- include/util/taoserror.h | 1 + source/libs/parser/src/parTranslater.c | 4 ++++ source/libs/parser/src/parUtil.c | 2 ++ source/libs/parser/test/parInitialDTest.cpp | 9 +++++++++ source/util/src/terror.c | 1 + 5 files changed, 17 insertions(+) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 772a668f0f..0cd73f2d9a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -706,6 +706,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_TIMELINE_QUERY TAOS_DEF_ERROR_CODE(0, 0x2666) #define TSDB_CODE_PAR_INVALID_OPTR_USAGE TAOS_DEF_ERROR_CODE(0, 0x2667) #define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2668) +#define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2669) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8fc4be5f95..7914105ac1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3520,6 +3520,10 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } + if (pSelect->pFromTable->type == QUERY_NODE_REAL_TABLE && + ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType == TSDB_SYSTEM_TABLE) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, "WINDOW"); + } pCxt->currClause = SQL_CLAUSE_WINDOW; int32_t code = translateExpr(pCxt, &pSelect->pWindow); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index f82d56ac56..263318b92f 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -172,6 +172,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "%s function is not supported in group query"; case TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC: return "%s function is not supported in system table query"; + case TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED: + return "%s is not supported in system table query"; case TSDB_CODE_PAR_INVALID_INTERP_CLAUSE: return "Invalid usage of RANGE clause, EVERY clause or FILL clause"; case TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN: diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp index cddd2aa8f7..937f76176e 100644 --- a/source/libs/parser/test/parInitialDTest.cpp +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -291,4 +291,13 @@ TEST_F(ParserInitialDTest, dropUser) { run("DROP USER wxy"); } +TEST_F(ParserInitialDTest, IntervalOnSysTable) { + login("root"); + run("SELECT count('reboot_time') FROM information_schema.ins_dnodes interval(14m) sliding(9m)", + TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, PARSER_STAGE_TRANSLATE); + + run("SELECT count('create_time') FROM information_schema.ins_qnodes interval(14m) sliding(9m)", + TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, PARSER_STAGE_TRANSLATE); +} + } // namespace ParserTest diff --git a/source/util/src/terror.c b/source/util/src/terror.c index d2b9edf753..7d3859e04a 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -568,6 +568,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_SELECTED_EXPR, "Invalid SELECTed ex TAOS_DEFINE_ERROR(TSDB_CODE_PAR_GET_META_ERROR, "Fail to get table info") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_UNIQUE_TABLE_ALIAS, "Not unique table/alias") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC, "System table not allowed") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED, "System table not allowed") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error") //planner From 4f814db5d5ded7a0a207dcc9ef1b00cc15a91ab6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 6 Jul 2023 18:34:01 +0800 Subject: [PATCH 30/34] fix(stream): fix error during transferring executor state, while a task is not in normal status. --- source/libs/stream/src/streamExec.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index bcb479e71e..d0d63215e6 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -351,9 +351,13 @@ static void waitForTaskIdle(SStreamTask* pTask, SStreamTask* pStreamTask) { static int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { SStreamTask* pStreamTask = streamMetaAcquireTask(pTask->pMeta, pTask->streamTaskId.taskId); - qDebug("s-task:%s scan history task end, update stream task:%s info, transfer exec state", pTask->id.idStr, pStreamTask->id.idStr); - - // todo handle stream task is dropped here + if (pStreamTask == NULL) { + qError("s-task:%s failed to find related stream task:0x%x, it may have been destoryed or closed", + pTask->id.idStr, pTask->streamTaskId.taskId); + return TSDB_CODE_STREAM_TASK_NOT_EXIST; + } else { + qDebug("s-task:%s scan history task end, update stream task:%s info, transfer exec state", pTask->id.idStr, pStreamTask->id.idStr); + } ASSERT(pStreamTask != NULL && pStreamTask->historyTaskId.taskId == pTask->id.taskId); STimeWindow* pTimeWindow = &pStreamTask->dataRange.window; @@ -377,7 +381,7 @@ static int32_t streamTransferStateToStreamTask(SStreamTask* pTask) { if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) { // update the scan data range for source task. qDebug("s-task:%s level:%d stream task window %" PRId64 " - %" PRId64 " update to %" PRId64 " - %" PRId64 - ", status:%s, sched-status:%d", + ", status:%s, sched-status:%d", pStreamTask->id.idStr, TASK_LEVEL__SOURCE, pTimeWindow->skey, pTimeWindow->ekey, INT64_MIN, pTimeWindow->ekey, streamGetTaskStatusStr(TASK_STATUS__NORMAL), pStreamTask->status.schedStatus); } else { @@ -473,6 +477,9 @@ int32_t streamExecForAll(SStreamTask* pTask) { ASSERT(batchSize == 0); if (pTask->info.fillHistory && pTask->status.transferState) { int32_t code = streamTransferStateToStreamTask(pTask); + if (code != TSDB_CODE_SUCCESS) { // todo handle this + return 0; + } } break; @@ -564,7 +571,7 @@ int32_t streamTryExec(SStreamTask* pTask) { if (schedStatus == TASK_SCHED_STATUS__WAITING) { int32_t code = streamExecForAll(pTask); - if (code < 0) { + if (code < 0) { // todo this status shoudl be removed atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__FAILED); return -1; } From 4db5cb7c1a604ce3676d9fcec1a25eb96fd45035 Mon Sep 17 00:00:00 2001 From: huolibo Date: Tue, 20 Jun 2023 17:35:01 +0800 Subject: [PATCH 31/34] docs: add request Id description --- docs/en/14-reference/03-connector/04-java.mdx | 50 +++++++++++++++++++ docs/zh/08-connector/14-java.mdx | 46 +++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index 9c5a852c70..9376850d3e 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -288,6 +288,7 @@ The configuration parameters in the URL are as follows: - httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when batchfetch is false. - messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when batchfetch is true. - useSSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection. +- httpPoolSize: size of REST concurrent requests. The default value is 20. **Note**: Some configuration items (e.g., locale, timezone) do not work in the REST connection. @@ -355,6 +356,7 @@ The configuration parameters in properties are as follows. - TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection and batchfetch is false. - TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when using JDBC REST connection and batchfetch is true. - TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection. It only takes effect when using JDBC REST connection. +- TSDBDriver.HTTP_POOL_SIZE: size of REST concurrent requests. The default value is 20. For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only). ### Priority of configuration parameters @@ -419,6 +421,19 @@ while(resultSet.next()){ > The query is consistent with operating a relational database. When using subscripts to get the contents of the returned fields, you have to start from 1. However, we recommend using the field names to get the values of the fields in the result set. +### execute SQL with reqId + +This reqId can be used to request link tracing. + +``` +AbstractStatement aStmt = (AbstractStatement) connection.createStatement(); +aStmt.execute("create database if not exists db", 1L); +aStmt.executeUpdate("use db", 2L); +try (ResultSet rs = aStmt.executeQuery("select * from tb", 3L)) { + Timestamp ts = rs.getTimestamp(1); +} +``` + ### Writing data via parameter binding TDengine has significantly improved the bind APIs to support data writing (INSERT) scenarios. Writing data in this way avoids the resource consumption of SQL syntax parsing, resulting in significant write performance improvements in many cases. @@ -936,6 +951,14 @@ public class SchemalessWsTest { +### Schemaless with reqId + +This reqId can be used to request link tracing. + +``` +writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS, 1L); +``` + ### Data Subscription The TDengine Java Connector supports subscription functionality with the following application API. @@ -1002,6 +1025,29 @@ Map endOffsets(String topic) throws SQLException; void seek(TopicPartition partition, long offset) throws SQLException; ``` +Example usage is as follows. + +``` +String topic = "offset_seek_test"; +Map offset = null; +try (TaosConsumer consumer = new TaosConsumer<>(properties)) { + consumer.subscribe(Collections.singletonList(topic)); + for (int i = 0; i < 10; i++) { + if (i == 3) { + // Saving consumption position + offset = consumer.position(topic); + } + if (i == 5) { + // reset consumption to the previously saved position + for (Map.Entry entry : offset.entrySet()) { + consumer.seek(entry.getKey(), entry.getValue()); + } + } + ConsumerRecords records = consumer.poll(Duration.ofMillis(500)); + } +} +``` + #### Close subscriptions ```java @@ -1308,3 +1354,7 @@ For additional troubleshooting, see [FAQ](../../../train-faq/faq). ## API Reference [taos-jdbcdriver doc](https://docs.taosdata.com/api/taos-jdbcdriver) + +``` + +``` diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 1588159b57..7b22055c62 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -291,6 +291,7 @@ url 中的配置参数如下： - httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。 - messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 - useSSL: 连接中是否使用 SSL。 +- httpPoolSize: REST 并发请求大小，默认 20。 **注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。 @@ -358,6 +359,7 @@ properties 中的配置参数如下： - TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 5000。仅在 REST 连接且 batchfetch 设置为 false 时生效。 - TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。 - TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。 +- TSDBDriver.HTTP_POOL_SIZE: REST 并发请求大小，默认 20。 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。 ### 配置参数的优先级 @@ -422,6 +424,19 @@ while(resultSet.next()){ > 查询和操作关系型数据库一致，使用下标获取返回字段内容时从 1 开始，建议使用字段名称获取。 +### 执行带有 reqId 的 SQL + +此 reqId 可用于请求链路追踪。 + +``` +AbstractStatement aStmt = (AbstractStatement) connection.createStatement(); +aStmt.execute("create database if not exists db", 1L); +aStmt.executeUpdate("use db", 2L); +try (ResultSet rs = aStmt.executeQuery("select * from tb", 3L)) { + Timestamp ts = rs.getTimestamp(1); +} +``` + ### 通过参数绑定写入数据 TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据写入（INSERT）场景的支持。采用这种方式写入数据时，能避免 SQL 语法解析的资源消耗，从而在很多情况下显著提升写入性能。 @@ -939,6 +954,14 @@ public class SchemalessWsTest { +### 执行带有 reqId 的无模式写入 + +此 reqId 可用于请求链路追踪。 + +``` +writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS, 1L); +``` + ### 数据订阅 TDengine Java 连接器支持订阅功能，应用 API 如下： @@ -1005,6 +1028,29 @@ Map endOffsets(String topic) throws SQLException; void seek(TopicPartition partition, long offset) throws SQLException; ``` +示例代码： + +``` +String topic = "offset_seek_test"; +Map offset = null; +try (TaosConsumer consumer = new TaosConsumer<>(properties)) { + consumer.subscribe(Collections.singletonList(topic)); + for (int i = 0; i < 10; i++) { + if (i == 3) { + // Saving consumption position + offset = consumer.position(topic); + } + if (i == 5) { + // reset consumption to the previously saved position + for (Map.Entry entry : offset.entrySet()) { + consumer.seek(entry.getKey(), entry.getValue()); + } + } + ConsumerRecords records = consumer.poll(Duration.ofMillis(500)); + } +} +``` + #### 关闭订阅 ```java From 6328b4310400b5e963913e95970b2d25a88b4a3a Mon Sep 17 00:00:00 2001 From: huolibo Date: Tue, 20 Jun 2023 18:03:27 +0800 Subject: [PATCH 32/34] docs: markdown format --- docs/en/14-reference/03-connector/04-java.mdx | 8 ++++---- docs/zh/08-connector/14-java.mdx | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index 9376850d3e..ebd2891a9e 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -425,7 +425,7 @@ while(resultSet.next()){ This reqId can be used to request link tracing. -``` +```java AbstractStatement aStmt = (AbstractStatement) connection.createStatement(); aStmt.execute("create database if not exists db", 1L); aStmt.executeUpdate("use db", 2L); @@ -955,7 +955,7 @@ public class SchemalessWsTest { This reqId can be used to request link tracing. -``` +```java writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS, 1L); ``` @@ -1016,7 +1016,7 @@ while(true) { #### Assignment subscription Offset -``` +```java long position(TopicPartition partition) throws SQLException; Map position(String topic) throws SQLException; Map beginningOffsets(String topic) throws SQLException; @@ -1027,7 +1027,7 @@ void seek(TopicPartition partition, long offset) throws SQLException; Example usage is as follows. -``` +```java String topic = "offset_seek_test"; Map offset = null; try (TaosConsumer consumer = new TaosConsumer<>(properties)) { diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 7b22055c62..27b732b883 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -428,7 +428,7 @@ while(resultSet.next()){ 此 reqId 可用于请求链路追踪。 -``` +```java AbstractStatement aStmt = (AbstractStatement) connection.createStatement(); aStmt.execute("create database if not exists db", 1L); aStmt.executeUpdate("use db", 2L); @@ -958,7 +958,7 @@ public class SchemalessWsTest { 此 reqId 可用于请求链路追踪。 -``` +```java writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS, 1L); ``` @@ -1019,7 +1019,7 @@ while(true) { #### 指定订阅 Offset -``` +```java long position(TopicPartition partition) throws SQLException; Map position(String topic) throws SQLException; Map beginningOffsets(String topic) throws SQLException; @@ -1030,7 +1030,7 @@ void seek(TopicPartition partition, long offset) throws SQLException; 示例代码： -``` +```java String topic = "offset_seek_test"; Map offset = null; try (TaosConsumer consumer = new TaosConsumer<>(properties)) { From 0b58fb1fb8d83fe2dc94c96b1682590b445b9d65 Mon Sep 17 00:00:00 2001 From: huolibo Date: Tue, 27 Jun 2023 17:47:30 +0800 Subject: [PATCH 33/34] docs(driver): jdbc 3.2.3 --- docs/en/14-reference/03-connector/04-java.mdx | 15 ++++++++------- docs/zh/08-connector/14-java.mdx | 17 +++++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index ebd2891a9e..e8c407b125 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -36,7 +36,8 @@ REST connection supports all platforms that can run Java. | taos-jdbcdriver version | major changes | TDengine version | | :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | :--------------: | -| 3.2.1 | subscription add seek function | 3.0.5.0 or later | +| 3.2.3 | Fixed resultSet data parsing failure in some cases | 3.0.5.0 or later | +| 3.2.2 | subscription add seek function | 3.0.5.0 or later | | 3.2.1 | JDBC REST connection supports schemaless/prepareStatement over WebSocket | 3.0.3.0 or later | | 3.2.0 | This version has been deprecated | - | | 3.1.0 | JDBC REST connection supports subscription over WebSocket | - | @@ -284,9 +285,9 @@ The configuration parameters in the URL are as follows: - batchfetch: true: pulls result sets in batches when executing queries; false: pulls result sets row by row. The default value is: false. batchfetch uses HTTP for data transfer. JDBC REST supports batch pulls. taos-jdbcdriver and TDengine transfer data via WebSocket connection. Compared with HTTP, WebSocket enables JDBC REST connection to support large data volume querying and improve query performance. - charset: specify the charset to parse the string, this parameter is valid only when set batchfetch to true. - batchErrorIgnore: true: when executing executeBatch of Statement, if one SQL execution fails in the middle, continue to execute the following SQL. false: no longer execute any statement after the failed SQL. The default value is: false. -- httpConnectTimeout: REST connection timeout in milliseconds, the default value is 5000 ms. -- httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when batchfetch is false. -- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when batchfetch is true. +- httpConnectTimeout: REST connection timeout in milliseconds, the default value is 60000 ms. +- httpSocketTimeout: socket timeout in milliseconds, the default value is 60000 ms. It only takes effect when batchfetch is false. +- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 60000 ms. It only takes effect when batchfetch is true. - useSSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection. - httpPoolSize: size of REST concurrent requests. The default value is 20. @@ -352,9 +353,9 @@ The configuration parameters in properties are as follows. - TSDBDriver.PROPERTY_KEY_CHARSET: In the character set used by the client, the default value is the system character set. - TSDBDriver.PROPERTY_KEY_LOCALE: this only takes effect when using JDBC native connection. Client language environment, the default value is system current locale. - TSDBDriver.PROPERTY_KEY_TIME_ZONE: only takes effect when using JDBC native connection. In the time zone used by the client, the default value is the system's current time zone. -- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection. -- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection and batchfetch is false. -- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when using JDBC REST connection and batchfetch is true. +- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection. +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection and batchfetch is false. +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection and batchfetch is true. - TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection. It only takes effect when using JDBC REST connection. - TSDBDriver.HTTP_POOL_SIZE: size of REST concurrent requests. The default value is 20. For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only). diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 27b732b883..c7da2bd4f5 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -36,14 +36,15 @@ REST 连接支持所有能运行 Java 的平台。 | taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 | | :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------: | +| 3.2.3 | 修复 ResultSet 在一些情况数据解析失败 | - | | 3.2.2 | 新增功能：数据订阅支持 seek 功能。 | 3.0.5.0 及更高版本 | | 3.2.1 | 新增功能：WebSocket 连接支持 schemaless 与 prepareStatement 写入。变更：consumer poll 返回结果集为 ConsumerRecord，可通过 value() 获取指定结果集数据。 | 3.0.3.0 及更高版本 | | 3.2.0 | 存在连接问题，不推荐使用 | - | | 3.1.0 | WebSocket 连接支持订阅功能 | - | | 3.0.1 - 3.0.4 | 修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用其他版本 | - | | 3.0.0 | 支持 TDengine 3.0 | 3.0.0.0 及更高版本 | -| 2.0.42 | 修在 WebSocket 连接中 wasNull 接口返回值 | - | -| 2.0.41 | 修正 REST 连接中用户名和密码转码方式 | - | +| 2.0.42 | 修复 WebSocket 连接中 wasNull 接口返回值 | - | +| 2.0.41 | 修复 REST 连接中用户名和密码转码方式 | - | | 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | - | | 2.0.38 | JDBC REST 连接增加批量拉取功能 | - | | 2.0.37 | 增加对 json tag 支持 | - | @@ -287,9 +288,9 @@ url 中的配置参数如下： - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 -- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 -- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。 -- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 +- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 60000。 +- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 60000。仅在 batchfetch 设置为 false 时生效。 +- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 60000。 仅在 batchfetch 设置为 true 时生效。 - useSSL: 连接中是否使用 SSL。 - httpPoolSize: REST 并发请求大小，默认 20。 @@ -355,9 +356,9 @@ properties 中的配置参数如下： - TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。 - TSDBDriver.PROPERTY_KEY_LOCALE：仅在使用 JDBC 原生连接时生效。 客户端语言环境，默认值系统当前 locale。 - TSDBDriver.PROPERTY_KEY_TIME_ZONE：仅在使用 JDBC 原生连接时生效。 客户端使用的时区，默认值为系统当前时区。 -- TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间，单位 ms， 默认值为 5000。仅在 REST 连接时生效。 -- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 5000。仅在 REST 连接且 batchfetch 设置为 false 时生效。 -- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。 +- TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间，单位 ms， 默认值为 60000。仅在 REST 连接时生效。 +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 60000。仅在 REST 连接且 batchfetch 设置为 false 时生效。 +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 60000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。 - TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。 - TSDBDriver.HTTP_POOL_SIZE: REST 并发请求大小，默认 20。 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。 From 9e6d5fff58b4137d3ea132fe9a2dc9bafe300b16 Mon Sep 17 00:00:00 2001 From: huolibo Date: Fri, 7 Jul 2023 09:43:36 +0800 Subject: [PATCH 34/34] docs(driver): jdbc 3.2.4 description --- docs/en/07-develop/07-tmq.mdx | 4 ---- docs/en/14-reference/03-connector/04-java.mdx | 7 ++++--- docs/zh/07-develop/07-tmq.mdx | 4 ---- docs/zh/08-connector/14-java.mdx | 1 + 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 578f38e73d..65d789b2d3 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -81,10 +81,6 @@ Set subscription() throws SQLException; ConsumerRecords poll(Duration timeout) throws SQLException; -void commitAsync(); - -void commitAsync(OffsetCommitCallback callback); - void commitSync() throws SQLException; void close() throws SQLException; diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index e8c407b125..b68aeda94c 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -36,15 +36,16 @@ REST connection supports all platforms that can run Java. | taos-jdbcdriver version | major changes | TDengine version | | :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | :--------------: | +| 3.2.4 | Subscription add the enable.auto.commit parameter and the unsubscribe() method in the WebSocket connection | 3.0.5.0 or later | | 3.2.3 | Fixed resultSet data parsing failure in some cases | 3.0.5.0 or later | -| 3.2.2 | subscription add seek function | 3.0.5.0 or later | +| 3.2.2 | Subscription add seek function | 3.0.5.0 or later | | 3.2.1 | JDBC REST connection supports schemaless/prepareStatement over WebSocket | 3.0.3.0 or later | | 3.2.0 | This version has been deprecated | - | | 3.1.0 | JDBC REST connection supports subscription over WebSocket | - | | 3.0.1 - 3.0.4 | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use other version in the JDK 8 environment | - | | 3.0.0 | Support for TDengine 3.0 | 3.0.0.0 or later | -| 2.0.42 | fix wasNull interface return value in WebSocket connection | - | -| 2.0.41 | fix decode method of username and password in REST connection | - | +| 2.0.42 | Fix wasNull interface return value in WebSocket connection | - | +| 2.0.41 | Fix decode method of username and password in REST connection | - | | 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | - | | 2.0.38 | JDBC REST connections add bulk pull function | - | | 2.0.37 | Support json tags | - | diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index a87a1f64f8..7a82761191 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -81,10 +81,6 @@ Set subscription() throws SQLException; ConsumerRecords poll(Duration timeout) throws SQLException; -void commitAsync(); - -void commitAsync(OffsetCommitCallback callback); - void commitSync() throws SQLException; void close() throws SQLException; diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index c7da2bd4f5..96f8991eea 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -36,6 +36,7 @@ REST 连接支持所有能运行 Java 的平台。 | taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 | | :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------: | +| 3.2.4 | 数据订阅在 WebSocket 连接下增加 enable.auto.commit 参数，以及 unsubscribe() 方法。 | - | | 3.2.3 | 修复 ResultSet 在一些情况数据解析失败 | - | | 3.2.2 | 新增功能：数据订阅支持 seek 功能。 | 3.0.5.0 及更高版本 | | 3.2.1 | 新增功能：WebSocket 连接支持 schemaless 与 prepareStatement 写入。变更：consumer poll 返回结果集为 ConsumerRecord，可通过 value() 获取指定结果集数据。 | 3.0.3.0 及更高版本 |