From d3446a5c7aa1daeabba0bf3959d7ad9bc8da46c9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 09:32:21 +0000 Subject: [PATCH 01/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 2 +- source/libs/tdb/src/db/tdbPage.c | 253 ---------------------------- source/libs/tdb/src/inc/tdbPage.h | 26 +-- source/libs/tdb/src/page/tdbPage.c | 244 +++++++++++++++++---------- source/libs/tdb/src/page/tdbPageL.c | 45 +++-- 5 files changed, 198 insertions(+), 372 deletions(-) delete mode 100644 source/libs/tdb/src/db/tdbPage.c diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c6e8c9dca9..423de0b53c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -414,7 +414,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { } // TODO: need to update the SPage.nFree - pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; + // pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; pPage->nOverflow = 0; return 0; diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c deleted file mode 100644 index df158de756..0000000000 --- a/source/libs/tdb/src/db/tdbPage.c +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -typedef struct __attribute__((__packed__)) { - u8 szCell[2]; - u8 nxOffset[2]; -} SFreeCell; - -typedef struct __attribute__((__packed__)) { - u8 szCell[3]; - u8 nxOffset[3]; -} SFreeCellL; - -/* For small page */ -#define TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCell *)(PCELL))->szCell) -#define TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCell *)(PCELL))->nxOffset) - -#define TDB_SPAGE_FREE_CELL_SIZE(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL))[0] -#define TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))[0] - -#define TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) (TDB_SPAGE_FREE_CELL_SIZE(PCELL) = (SIZE)) -#define TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) (TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) = (OFFSET)) - -/* For large page */ -#define TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCellL *)(PCELL))->szCell) -#define TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCellL *)(PCELL))->nxOffset) - -#define TDB_LPAGE_FREE_CELL_SIZE(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL)) -#define TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) - -#define TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL), SIZE) -#define TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL), OFFSET) - -/* For page */ -#define TDB_PAGE_FREE_CELL_SIZE_PTR(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) : TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL)) -#define TDB_PAGE_FREE_CELL_NXOFFSET_PTR(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) - -#define TDB_PAGE_FREE_CELL_SIZE(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE(PCELL) : TDB_SPAGE_FREE_CELL_SIZE(PCELL)) -#define TDB_PAGE_FREE_CELL_NXOFFSET(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL)) - -#define TDB_PAGE_FREE_CELL_SIZE_SET(PPAGE, PCELL, SIZE) \ - do { \ - if (TDB_IS_LARGE_PAGE(PPAGE)) { \ - TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ - } else { \ - TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ - } \ - } while (0) -#define TDB_PAGE_FREE_CELL_NXOFFSET_SET(PPAGE, PCELL, OFFSET) \ - do { \ - if (TDB_IS_LARGE_PAGE(PPAGE)) { \ - TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ - } else { \ - TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ - } \ - } while (0) - -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); -static int tdbPageDefragment(SPage *pPage); - -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { - SPage *pPage; - u8 *ptr; - int size; - - ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); - - *ppPage = NULL; - size = pageSize + sizeof(*pPage); - - ptr = (u8 *)((*xMalloc)(arg, size)); - if (pPage == NULL) { - return -1; - } - - memset(ptr, 0, size); - pPage = (SPage *)(ptr + pageSize); - - pPage->pData = ptr; - pPage->pageSize = pageSize; - if (pageSize < 65536) { - pPage->szOffset = 2; - pPage->szPageHdr = sizeof(SPageHdr); - pPage->szFreeCell = sizeof(SFreeCell); - } else { - pPage->szOffset = 3; - pPage->szPageHdr = sizeof(SPageHdrL); - pPage->szFreeCell = sizeof(SFreeCellL); - } - TDB_INIT_PAGE_LOCK(pPage); - - /* TODO */ - - *ppPage = pPage; - return 0; -} - -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { - u8 *ptr; - - ptr = pPage->pData; - (*xFree)(arg, ptr); - - return 0; -} - -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { - int ret; - SCell *pTarget; - u8 *pTmp; - int j; - - if (pPage->nOverflow || szCell + pPage->szOffset > pPage->nFree) { - // TODO: need to figure out if pCell may be used by outside of this function - j = pPage->nOverflow++; - - pPage->apOvfl[j] = pCell; - pPage->aiOvfl[j] = idx; - } else { - ret = tdbPageAllocate(pPage, szCell, &pTarget); - if (ret < 0) { - return -1; - } - - memcpy(pTarget, pCell, szCell); - pTmp = pPage->pCellIdx + idx * pPage->szOffset; - memmove(pTmp + pPage->szOffset, pTmp, pPage->pFreeStart - pTmp - pPage->szOffset); - TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); - TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); - } - - return 0; -} - -int tdbPageDropCell(SPage *pPage, int idx) { - // TODO - return 0; -} - -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { - SCell *pCell; - SFreeCell *pFreeCell; - u8 *pOffset; - int ret; - - ASSERT(pPage->nFree > size + pPage->szOffset); - - pCell = NULL; - *ppCell = NULL; - - // 1. Try to allocate from the free space area - if (pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset) { - pPage->pFreeEnd -= size; - pPage->pFreeStart += pPage->szOffset; - pCell = pPage->pFreeEnd; - } - - // 2. Try to allocate from the page free list - if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= pPage->szOffset) && TDB_PAGE_FCELL(pPage)) { - int szCell; - int nxOffset; - - pCell = pPage->pData + TDB_PAGE_FCELL(pPage); - pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell - : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - - for (;;) { - // Find a cell - if (szCell >= size) { - if (szCell - size >= pPage->szFreeCell) { - SCell *pTmpCell = pCell + size; - - TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); - TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); - // TODO: *pOffset = pTmpCell - pPage->pData; - } else { - TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); - // TODO: *pOffset = nxOffset; - } - break; - } - - // Not find a cell yet - if (nxOffset > 0) { - pCell = pPage->pData + nxOffset; - pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - continue; - } else { - pCell = NULL; - break; - } - } - - if (pCell) { - pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; - } - } - - // 3. Try to dfragment and allocate again - if (pCell == NULL) { - ret = tdbPageDefragment(pPage); - if (ret < 0) { - return -1; - } - - ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset); - ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); - - // Allocate from the free space area again - pPage->pFreeEnd -= size; - pPage->pFreeStart += pPage->szOffset; - pCell = pPage->pFreeEnd; - } - - ASSERT(pCell != NULL); - - pPage->nFree = pPage->nFree - size - pPage->szOffset; - *ppCell = pCell; - return 0; -} - -static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { - // TODO - return 0; -} - -static int tdbPageDefragment(SPage *pPage) { - // TODO - ASSERT(0); - return 0; -} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 6479574abe..418a1b6626 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -45,6 +45,9 @@ typedef struct { // cell offset at idx int (*getCellOffset)(SPage *, int); void (*setCellOffset)(SPage *, int, int); + // free cell info + void (*getFreeCellInfo)(SCell *pCell, int *szCell, int *nxOffset); + void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset); } SPageMethods; // Page footer @@ -59,20 +62,19 @@ struct SPage { SPageMethods *pPageMethods; // Fields below used by pager and am u8 szAmHdr; - u8 *pPageHdr; u8 *pAmHdr; + u8 *pPageHdr; u8 *pCellIdx; u8 *pFreeStart; u8 *pFreeEnd; SPageFtr *pPageFtr; - int kLen; // key length of the page, -1 for unknown - int vLen; // value length of the page, -1 for unknown - int nFree; - int maxLocal; - int minLocal; int nOverflow; SCell *apOvfl[4]; int aiOvfl[4]; + int kLen; // key length of the page, -1 for unknown + int vLen; // value length of the page, -1 for unknown + int maxLocal; + int minLocal; // Fields used by SPCache TDB_PCACHE_PAGE }; @@ -92,8 +94,6 @@ struct SPage { #define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) #define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) - #define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) // For page lock @@ -119,10 +119,12 @@ struct SPage { }) // APIs -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); -int tdbPageDropCell(SPage *pPage, int idx); +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); +void tdbPageZero(SPage *pPage); +void tdbPageInit(SPage *pPage); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); +int tdbPageDropCell(SPage *pPage, int idx); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 4ec3a895e7..e198891338 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -18,10 +18,11 @@ extern SPageMethods pageMethods; extern SPageMethods pageLargeMethods; -typedef struct __attribute__((__packed__)) { - u16 szCell; - u16 nxOffset; -} SFreeCell; +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) +#define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) +#define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) +#define TDB_PAGE_MAX_FREE_BLOCK(pPage) \ + ((pPage)->pageSize - (pPage)->szAmHdr - TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); static int tdbPageDefragment(SPage *pPage); @@ -44,6 +45,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) memset(ptr, 0, size); pPage = (SPage *)(ptr + pageSize); + TDB_INIT_PAGE_LOCK(pPage); pPage->pData = ptr; pPage->pageSize = pageSize; if (pageSize < 65536) { @@ -51,9 +53,6 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) } else { pPage->pPageMethods = &pageLargeMethods; } - TDB_INIT_PAGE_LOCK(pPage); - - /* TODO */ *ppPage = pPage; return 0; @@ -68,29 +67,64 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) return 0; } +void tdbPageZero(SPage *pPage) { + TDB_PAGE_NCELLS_SET(pPage, 0); + TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); + TDB_PAGE_FCELL_SET(pPage, 0); + TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_MAX_FREE_BLOCK(pPage)); + tdbPageInit(pPage); +} + +void tdbPageInit(SPage *pPage) { + pPage->pAmHdr = pPage->pData; + pPage->pPageHdr = pPage->pAmHdr + pPage->szAmHdr; + pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); + pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); + pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); + pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + pPage->nOverflow = 0; + + ASSERT(pPage->pFreeEnd >= pPage->pFreeStart); + ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); +} + int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { + int nFree; int ret; - SCell *pTarget; - u8 *pTmp; - int j; + int nCells; + int lidx; // local idx + SCell *pNewCell; - if (pPage->nOverflow || szCell + TDB_PAGE_OFFSET_SIZE(pPage) > pPage->nFree) { - // TODO: need to figure out if pCell may be used by outside of this function - j = pPage->nOverflow++; + ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage)); - pPage->apOvfl[j] = pCell; - pPage->aiOvfl[j] = idx; - } else { - ret = tdbPageAllocate(pPage, szCell, &pTarget); - if (ret < 0) { - return -1; + nFree = TDB_PAGE_NFREE(pPage); + nCells = TDB_PAGE_NCELLS(pPage); + + if (nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + // page must has enough space to hold the cell locally + ret = tdbPageAllocate(pPage, szCell, &pNewCell); + ASSERT(ret == 0); + + memcpy(pNewCell, pCell, szCell); + + if (pPage->nOverflow == 0) { + lidx = idx; + } else { + // TODO + // lidx = ; } - memcpy(pTarget, pCell, szCell); - pTmp = pPage->pCellIdx + idx * TDB_PAGE_OFFSET_SIZE(pPage); - memmove(pTmp + TDB_PAGE_OFFSET_SIZE(pPage), pTmp, pPage->pFreeStart - pTmp - TDB_PAGE_OFFSET_SIZE(pPage)); - TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); - TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); + // no overflow cell exists in this page + u8 *src = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * lidx; + u8 *dest = src + TDB_PAGE_OFFSET_SIZE(pPage); + memmove(dest, src, pPage->pFreeStart - dest); + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, TDB_PAGE_OFFSET_SIZE(pPage) * lidx); + TDB_PAGE_NCELLS_SET(pPage, nCells + 1); + } else { + // TODO: page not has enough space + pPage->apOvfl[pPage->nOverflow] = pCell; + pPage->aiOvfl[pPage->nOverflow] = idx; + pPage->nOverflow++; } return 0; @@ -101,73 +135,80 @@ int tdbPageDropCell(SPage *pPage, int idx) { return 0; } -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { - SCell *pCell; - SFreeCell *pFreeCell; - u8 *pOffset; - int ret; +static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { + SCell *pFreeCell; + u8 *pOffset; + int nFree; + int ret; + int cellFree; - ASSERT(pPage->nFree > size + TDB_PAGE_OFFSET_SIZE(pPage)); - - pCell = NULL; *ppCell = NULL; + nFree = TDB_PAGE_NFREE(pPage); - // 1. Try to allocate from the free space area - if (pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)) { - pPage->pFreeEnd -= size; + ASSERT(nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)); + ASSERT(TDB_PAGE_CCELLS(pPage) == pPage->pFreeEnd - pPage->pData); + + // 1. Try to allocate from the free space block area + if (pPage->pFreeEnd - pPage->pFreeStart >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); - pCell = pPage->pFreeEnd; + pPage->pFreeEnd -= szCell; + + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); + + *ppCell = pPage->pFreeEnd; + return 0; } // 2. Try to allocate from the page free list - if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) && - TDB_PAGE_FCELL(pPage)) { -#if 0 - int szCell; - int nxOffset; - - pCell = pPage->pData + TDB_PAGE_FCELL(pPage); - pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell - : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + cellFree = TDB_PAGE_FCELL(pPage); + ASSERT(cellFree == 0 || cellFree > pPage->pFreeEnd - pPage->pData); + if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { + SCell *pPrevFreeCell = NULL; + int szPrevFreeCell; + int szFreeCell; + int nxFreeCell; + int newSize; for (;;) { - // Find a cell - if (szCell >= size) { - if (szCell - size >= pPage->szFreeCell) { - SCell *pTmpCell = pCell + size; + if (cellFree == 0) break; - TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); - TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); - // TODO: *pOffset = pTmpCell - pPage->pData; + pFreeCell = pPage->pData + cellFree; + pPage->pPageMethods->getFreeCellInfo(pFreeCell, &szFreeCell, &nxFreeCell); + + if (szFreeCell >= szCell) { + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + TDB_PAGE_NFREE_SET(pPage, nFree - TDB_PAGE_OFFSET_SIZE(pPage) - szCell); + *ppCell = pFreeCell; + + newSize = szFreeCell - szCell; + pFreeCell += szCell; + if (newSize >= TDB_PAGE_FREE_CELL_SIZE(pPage)) { + pPage->pPageMethods->setFreeCellInfo(pFreeCell, newSize, nxFreeCell); + if (pPrevFreeCell) { + pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, pFreeCell - pPage->pData); + } else { + TDB_PAGE_FCELL_SET(pPage, pFreeCell - pPage->pData); + } + + return 0; } else { - TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); - // TODO: *pOffset = nxOffset; + if (pPrevFreeCell) { + pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, nxFreeCell); + } else { + TDB_PAGE_FCELL_SET(pPage, nxFreeCell); + } } - break; - } - - // Not find a cell yet - if (nxOffset > 0) { - pCell = pPage->pData + nxOffset; - pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - continue; } else { - pCell = NULL; - break; + pPrevFreeCell = pFreeCell; + szPrevFreeCell = szFreeCell; + cellFree = nxFreeCell; } } - - if (pCell) { - pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; - } -#endif } - // 3. Try to dfragment and allocate again +// 3. Try to dfragment and allocate again +#if 0 if (pCell == NULL) { ret = tdbPageDefragment(pPage); if (ret < 0) { @@ -175,7 +216,7 @@ static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { } ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)); - ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); + // ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); // Allocate from the free space area again pPage->pFreeEnd -= size; @@ -185,8 +226,9 @@ static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { ASSERT(pCell != NULL); - pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); + // pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); *ppCell = pCell; +#endif return 0; } @@ -210,6 +252,11 @@ typedef struct __attribute__((__packed__)) { u16 nFree; } SPageHdr; +typedef struct __attribute__((__packed__)) { + u16 szCell; + u16 nxOffset; +} SFreeCell; + // flags static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } @@ -253,20 +300,35 @@ static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { ((u16 *)pPage->pCellIdx)[idx] = (u16)offset; } +// free cell info +static inline void getPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { + SFreeCell *pFreeCell = (SFreeCell *)pCell; + *szCell = pFreeCell->szCell; + *nxOffset = pFreeCell->nxOffset; +} + +static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { + SFreeCell *pFreeCell = (SFreeCell *)pCell; + pFreeCell->szCell = szCell; + pFreeCell->nxOffset = nxOffset; +} + SPageMethods pageMethods = { - 2, // szOffset - sizeof(SPageHdr), // szPageHdr - sizeof(SFreeCell), // szFreeCell - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp - getPageCellNum, // getCellNum - setPageCellNum, // setCellNum - getPageCellBody, // getCellBody - setPageCellBody, // setCellBody - getPageCellFree, // getCellFree - setPageCellFree, // setCellFree - getPageNFree, // getFreeBytes - setPageNFree, // setFreeBytes - getPageCellOffset, // getCellOffset - setPageCellOffset // setCellOffset + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell + getPageFlags, // getPageFlags + setPageFlags, // setFlagsp + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset, // setCellOffset + getPageFreeCellInfo, // getFreeCellInfo + setPageFreeCellInfo // setFreeCellInfo }; \ No newline at end of file diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index e7c60118d2..e3c1bb8d37 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -66,20 +66,35 @@ static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset); } +// free cell info +static inline void getPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + *szCell = TDB_GET_U24(pFreeCell->szCell); + *nxOffset = TDB_GET_U24(pFreeCell->nxOffset); +} + +static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + TDB_PUT_U24(pFreeCell->szCell, szCell); + TDB_PUT_U24(pFreeCell->nxOffset, nxOffset); +} + SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp - getPageCellNum, // getCellNum - setPageCellNum, // setCellNum - getPageCellBody, // getCellBody - setPageCellBody, // setCellBody - getPageCellFree, // getCellFree - setPageCellFree, // setCellFree - getPageNFree, // getFreeBytes - setPageNFree, // setFreeBytes - getPageCellOffset, // getCellOffset - setPageCellOffset // setCellOffset + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell + getPageFlags, // getPageFlags + setPageFlags, // setFlagsp + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset, // setCellOffset + getPageFreeCellInfo, // getFreeCellInfo + setPageFreeCellInfo // setFreeCellInfo }; \ No newline at end of file From 57a92869ae28c7ce6f2776e521edcb879f9a4c77 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 09:45:49 +0000 Subject: [PATCH 02/92] more TDB --- source/libs/tdb/src/page/tdbPage.c | 46 +++++++++++------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index e198891338..892d7ecb31 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -141,6 +141,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { int nFree; int ret; int cellFree; + SCell *pCell = NULL; *ppCell = NULL; nFree = TDB_PAGE_NFREE(pPage); @@ -150,14 +151,10 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { // 1. Try to allocate from the free space block area if (pPage->pFreeEnd - pPage->pFreeStart >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { - pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); pPage->pFreeEnd -= szCell; - + pCell = pPage->pFreeEnd; TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); - TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); - - *ppCell = pPage->pFreeEnd; - return 0; + goto _alloc_finish; } // 2. Try to allocate from the page free list @@ -177,9 +174,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { pPage->pPageMethods->getFreeCellInfo(pFreeCell, &szFreeCell, &nxFreeCell); if (szFreeCell >= szCell) { - pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); - TDB_PAGE_NFREE_SET(pPage, nFree - TDB_PAGE_OFFSET_SIZE(pPage) - szCell); - *ppCell = pFreeCell; + pCell = pFreeCell; newSize = szFreeCell - szCell; pFreeCell += szCell; @@ -191,7 +186,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { TDB_PAGE_FCELL_SET(pPage, pFreeCell - pPage->pData); } - return 0; + goto _alloc_finish; } else { if (pPrevFreeCell) { pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, nxFreeCell); @@ -207,28 +202,21 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { } } -// 3. Try to dfragment and allocate again -#if 0 - if (pCell == NULL) { - ret = tdbPageDefragment(pPage); - if (ret < 0) { - return -1; - } + // 3. Try to dfragment and allocate again + tdbPageDefragment(pPage); + ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + ASSERT(nFree == TDB_PAGE_NFREE(pPage)); + ASSERT(pPage->pFreeEnd - pPage->pData == TDB_PAGE_CCELLS(pPage)); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)); - // ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); + pPage->pFreeEnd -= szCell; + pCell = pPage->pFreeEnd; + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); - // Allocate from the free space area again - pPage->pFreeEnd -= size; - pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); - pCell = pPage->pFreeEnd; - } - - ASSERT(pCell != NULL); - - // pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); +_alloc_finish: + ASSERT(pCell); + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); *ppCell = pCell; -#endif return 0; } From 0353e590814cfec20350627f07e5e11e3bcf6c67 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 12:02:26 +0000 Subject: [PATCH 03/92] more TDB --- source/libs/tdb/src/page/tdbPage.c | 34 ++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 892d7ecb31..1c9984cd01 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -220,8 +220,38 @@ _alloc_finish: return 0; } -static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { - // TODO +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { + int nFree; + int cellFree; + u8 *dest; + u8 *src; + + ASSERT(pCell >= pPage->pFreeEnd); + ASSERT(pCell + szCell <= (u8 *)(pPage->pPageFtr)); + ASSERT(pCell == TDB_PAGE_CELL_AT(pPage, idx)); + + nFree = TDB_PAGE_NFREE(pPage); + + if (pCell == pPage->pFreeEnd) { + pPage->pFreeEnd += szCell; + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + } else { + if (szCell >= TDB_PAGE_FREE_CELL_SIZE(pPage)) { + cellFree = TDB_PAGE_FCELL(pPage); + pPage->pPageMethods->setFreeCellInfo(pCell, szCell, cellFree); + TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); + } else { + ASSERT(0); + } + } + + dest = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * idx; + src = dest + TDB_PAGE_OFFSET_SIZE(pPage); + memmove(dest, src, pPage->pFreeStart - src); + + pPage->pFreeStart -= TDB_PAGE_OFFSET_SIZE(pPage); + nFree = nFree + szCell + TDB_PAGE_OFFSET_SIZE(pPage); + TDB_PAGE_NFREE_SET(pPage, nFree); return 0; } From dd7344c7800d88c6f444031319965130824cc7b7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 12:41:15 +0000 Subject: [PATCH 04/92] more TDB --- source/libs/tdb/src/inc/tdbPage.h | 1 + source/libs/tdb/src/page/tdbPage.c | 58 ++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 418a1b6626..9c40e9d5c1 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -75,6 +75,7 @@ struct SPage { int vLen; // value length of the page, -1 for unknown int maxLocal; int minLocal; + int (*xCellSize)(SCell *pCell); // Fields used by SPCache TDB_PCACHE_PAGE }; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 1c9984cd01..b10cd7b0ec 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -256,8 +256,62 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { } static int tdbPageDefragment(SPage *pPage) { - // TODO - ASSERT(0); + int nFree; + int nCells; + SCell *pCell; + SCell *pNextCell; + SCell *pTCell; + int szCell; + int idx; + int iCell; + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart < nFree); + + nFree = TDB_PAGE_NFREE(pPage); + nCells = TDB_PAGE_NCELLS(pPage); + + // Loop to compact the page content + // Here we use an O(n^2) algorithm to do the job since + // this is a low frequency job. + pNextCell = (u8 *)pPage->pPageFtr; + pCell = NULL; + for (iCell = 0;; iCell++) { + // compact over + if (iCell == nCells) { + pPage->pFreeEnd = pNextCell; + break; + } + + for (int i = 0; i < nCells; i++) { + if (TDB_PAGE_CELL_OFFSET_AT(pPage, i) < pNextCell - pPage->pData) { + pTCell = TDB_PAGE_CELL_AT(pPage, i); + if (pCell == NULL || pCell < pTCell) { + pCell = pTCell; + idx = i; + } + } else { + continue; + } + } + + ASSERT(pCell != NULL); + + szCell = (*pPage->xCellSize)(pCell); + + ASSERT(pCell + szCell <= pNextCell); + if (pCell + szCell < pNextCell) { + memmove(pNextCell - szCell, pCell, szCell); + } + + pCell = NULL; + pNextCell = pNextCell - szCell; + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pNextCell - pPage->pData); + } + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + TDB_PAGE_FCELL_SET(pPage, 0); + return 0; } From f91124ef8edc8f2ae300270b46798a9885f7b81b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 13:00:51 +0000 Subject: [PATCH 05/92] more TDB --- source/libs/tdb/src/page/tdbPage.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index b10cd7b0ec..9d679118e7 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -26,6 +26,7 @@ extern SPageMethods pageLargeMethods; static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); static int tdbPageDefragment(SPage *pPage); +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { SPage *pPage; @@ -131,7 +132,25 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { } int tdbPageDropCell(SPage *pPage, int idx) { - // TODO + int lidx; + SCell *pCell; + int szCell; + int nCells; + + nCells = TDB_PAGE_NCELLS(pPage); + + if (pPage->nOverflow == 0) { + lidx = idx; + } else { + // TODO + } + + pCell = TDB_PAGE_CELL_AT(pPage, lidx); + szCell = (*pPage->xCellSize)(pCell); + tdbPageFree(pPage, lidx, pCell, szCell); + + TDB_PAGE_NCELLS_SET(pPage, nCells - 1); + return 0; } From 6eb2e1b288d1363abf998b1604978a64340ad20a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 13:13:08 +0000 Subject: [PATCH 06/92] more TDB --- source/libs/tdb/src/page/tdbPage.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 9d679118e7..d91a32cb7c 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -136,19 +136,26 @@ int tdbPageDropCell(SPage *pPage, int idx) { SCell *pCell; int szCell; int nCells; + int iOvfl; nCells = TDB_PAGE_NCELLS(pPage); - if (pPage->nOverflow == 0) { - lidx = idx; - } else { - // TODO + ASSERT(idx >= 0 && idx < nCells + pPage->nOverflow); + + iOvfl = 0; + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] == idx) { + // TODO: remove the over flow cell + return 0; + } else if (pPage->aiOvfl[iOvfl] > idx) { + break; + } } + lidx = idx - iOvfl; pCell = TDB_PAGE_CELL_AT(pPage, lidx); szCell = (*pPage->xCellSize)(pCell); tdbPageFree(pPage, lidx, pCell, szCell); - TDB_PAGE_NCELLS_SET(pPage, nCells - 1); return 0; From 7d5030bc7fc2576411698f718f50fbaf7ff697ec Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 15:04:29 +0000 Subject: [PATCH 07/92] more TDB --- source/libs/tdb/src/page/tdbPage.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index d91a32cb7c..2c9563d36d 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -145,7 +145,13 @@ int tdbPageDropCell(SPage *pPage, int idx) { iOvfl = 0; for (; iOvfl < pPage->nOverflow; iOvfl++) { if (pPage->aiOvfl[iOvfl] == idx) { - // TODO: remove the over flow cell + // remove the over flow cell + for (; (++iOvfl) < pPage->nOverflow;) { + pPage->aiOvfl[iOvfl - 1] = pPage->aiOvfl[iOvfl] - 1; + pPage->apOvfl[iOvfl - 1] = pPage->apOvfl[iOvfl]; + } + + pPage->nOverflow--; return 0; } else if (pPage->aiOvfl[iOvfl] > idx) { break; @@ -158,6 +164,11 @@ int tdbPageDropCell(SPage *pPage, int idx) { tdbPageFree(pPage, lidx, pCell, szCell); TDB_PAGE_NCELLS_SET(pPage, nCells - 1); + for (; iOvfl < pPage->nOverflow; iOvfl++) { + pPage->aiOvfl[iOvfl]--; + ASSERT(pPage->aiOvfl[iOvfl] > 0); + } + return 0; } From 7a5985ae46de83a3fbe359aa5d79e21e54b4a8de Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 15:39:51 +0000 Subject: [PATCH 08/92] more TDB --- source/libs/tdb/src/page/tdbPage.c | 39 ++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 2c9563d36d..4673a561e6 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -91,8 +91,8 @@ void tdbPageInit(SPage *pPage) { int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { int nFree; - int ret; int nCells; + int iOvfl; int lidx; // local idx SCell *pNewCell; @@ -100,32 +100,45 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); + iOvfl = 0; + + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] >= idx) { + break; + } + } + + lidx = idx - iOvfl; if (nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { // page must has enough space to hold the cell locally - ret = tdbPageAllocate(pPage, szCell, &pNewCell); - ASSERT(ret == 0); + tdbPageAllocate(pPage, szCell, &pNewCell); memcpy(pNewCell, pCell, szCell); - if (pPage->nOverflow == 0) { - lidx = idx; - } else { - // TODO - // lidx = ; - } - // no overflow cell exists in this page u8 *src = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * lidx; u8 *dest = src + TDB_PAGE_OFFSET_SIZE(pPage); memmove(dest, src, pPage->pFreeStart - dest); TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, TDB_PAGE_OFFSET_SIZE(pPage) * lidx); TDB_PAGE_NCELLS_SET(pPage, nCells + 1); + + ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); } else { - // TODO: page not has enough space - pPage->apOvfl[pPage->nOverflow] = pCell; - pPage->aiOvfl[pPage->nOverflow] = idx; + // add the cell as an overflow cell + for (int i = pPage->nOverflow; i > iOvfl; i--) { + pPage->apOvfl[i] = pPage->apOvfl[i - 1]; + pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; + } + + pPage->apOvfl[iOvfl] = pCell; + pPage->aiOvfl[iOvfl] = idx; pPage->nOverflow++; + iOvfl++; + } + + for (; iOvfl < pPage->nOverflow; iOvfl++) { + pPage->aiOvfl[iOvfl]++; } return 0; From 13b7ba3edb940b226754215ecd17af97025eb727 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 02:27:46 +0000 Subject: [PATCH 09/92] refact TDB --- source/libs/tdb/src/db/tdbBtree.c | 63 +++++++++++++++++------------- source/libs/tdb/src/inc/tdbPage.h | 44 +++++++++++++-------- source/libs/tdb/src/page/tdbPage.c | 20 ++++++++-- 3 files changed, 80 insertions(+), 47 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 423de0b53c..621afc9dfb 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -145,7 +145,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } if (pCur->idx == -1) { - ASSERT(TDB_PAGE_NCELLS(pCur->pPage) == 0); + ASSERT(TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0); idx = 0; } else { if (cret > 0) { @@ -218,7 +218,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p pCur->iPage = 0; - if (TDB_PAGE_NCELLS(pCur->pPage) == 0) { + if (TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0) { // Current page is empty ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); return 0; @@ -231,7 +231,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p SCellDecoder cd = {0}; pPage = pCur->pPage; - nCells = TDB_PAGE_NCELLS(pPage); + nCells = TDB_PAGE_TOTAL_CELLS(pPage); lidx = 0; ridx = nCells - 1; @@ -242,7 +242,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p midx = (lidx + ridx) >> 1; - pCell = TDB_PAGE_CELL_AT(pPage, midx); + pCell = tdbPageGetCell(pPage, midx); ret = tdbBtreeDecodeCell(pPage, pCell, &cd); if (ret < 0) { // TODO: handle error @@ -283,7 +283,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p } else { // TODO: reset cd as uninitialized pCur->idx = midx + 1; - pCell = TDB_PAGE_CELL_AT(pPage, midx + 1); + pCell = tdbPageGetCell(pPage, midx + 1); tdbBtreeDecodeCell(pPage, pCell, &cd); tdbBtCursorMoveToChild(pCur, cd.pgno); } @@ -391,16 +391,12 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { } else { pPage->szAmHdr = sizeof(SBtPageHdr); } - pPage->pPageHdr = pPage->pData; - pPage->pAmHdr = pPage->pPageHdr + pPage->pPageMethods->szPageHdr; - pPage->pCellIdx = pPage->pAmHdr + pPage->szAmHdr; - pPage->pFreeStart = pPage->pCellIdx + pPage->pPageMethods->szOffset * TDB_PAGE_NCELLS(pPage); - pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); - pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + pPage->xCellSize = NULL; // TODO + + tdbPageInit(pPage); TDB_BTREE_ASSERT_FLAG(flags); - // Init other fields if (isLeaf) { pPage->kLen = pBt->keyLen; pPage->vLen = pBt->valLen; @@ -413,30 +409,41 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { pPage->minLocal = pBt->minLocal; } - // TODO: need to update the SPage.nFree - // pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; - pPage->nOverflow = 0; - return 0; } static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u16 flags; SBTree *pBt; + u8 isLeaf; flags = ((SBtreeZeroPageArg *)arg)->flags; pBt = ((SBtreeZeroPageArg *)arg)->pBt; - - pPage->pPageHdr = pPage->pData; + isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Init the page header TDB_PAGE_FLAGS_SET(pPage, flags); - TDB_PAGE_NCELLS_SET(pPage, 0); - TDB_PAGE_CCELLS_SET(pPage, pBt->pageSize - sizeof(SPageFtr)); - TDB_PAGE_FCELL_SET(pPage, 0); - TDB_PAGE_NFREE_SET(pPage, 0); + // Set szAmHdr + if (isLeaf) { + pPage->szAmHdr = 0; + } else { + pPage->szAmHdr = sizeof(SBtPageHdr); + } + pPage->xCellSize = NULL; // TODO - tdbBtreeInitPage(pPage, (void *)pBt); + tdbPageZero(pPage); + + if (isLeaf) { + pPage->kLen = pBt->keyLen; + pPage->vLen = pBt->valLen; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + } else { + pPage->kLen = pBt->keyLen; + pPage->vLen = sizeof(SPgno); + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + } return 0; } @@ -453,7 +460,8 @@ typedef struct { } SBtreeBalanceHelper; static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { - int nCells = TDB_PAGE_NCELLS(pFrom); +#if 0 + int nCells = TDB_PAGE_TOTAL_CELLS(pFrom); int cCells = TDB_PAGE_CCELLS(pFrom); int fCell = TDB_PAGE_FCELL(pFrom); int nFree = TDB_PAGE_NFREE(pFrom); @@ -469,6 +477,7 @@ static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { TDB_PAGE_NFREE_SET(pTo, nFree); // TODO: update other fields +#endif return 0; } @@ -529,7 +538,7 @@ static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { SBTree *pBt; pParent = pBlh->pParent; - nCells = TDB_PAGE_NCELLS(pParent); + nCells = TDB_PAGE_TOTAL_CELLS(pParent); nChild = nCells + 1; pBt = pBlh->pBt; @@ -556,7 +565,7 @@ static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { if (idxStart + i == nCells) { pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; } else { - pCell = TDB_PAGE_CELL_AT(pParent, idxStart + i); + pCell = tdbPageGetCell(pParent, idxStart + i); // TODO: no need to decode the payload part, and even the kLen, vLen part // we only need the pgno part ret = tdbBtreeDecodeCell(pParent, pCell, &cd); @@ -610,7 +619,7 @@ static int tdbBtreeBalanceStep2(SBtreeBalanceHelper *pBlh) { if (cidx < limit) { // Get local cells - pCell = TDB_PAGE_CELL_AT(pPage, cidx); + pCell = tdbPageGetCell(pPage, cidx); } else if (cidx == limit) { // Get overflow cells pCell = pPage->apOvfl[oidx++]; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 9c40e9d5c1..82287b0921 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -57,8 +57,8 @@ typedef struct __attribute__((__packed__)) { struct SPage { pthread_spinlock_t lock; - u8 *pData; int pageSize; + u8 *pData; SPageMethods *pPageMethods; // Fields below used by pager and am u8 szAmHdr; @@ -81,21 +81,8 @@ struct SPage { }; /* For page */ -#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) -#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) -#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage) -#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage) -#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) -#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx) - -#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) -#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS) -#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS) -#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL) -#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) -#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) - -#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) +#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) +#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) // For page lock #define P_LOCK_SUCC 0 @@ -120,6 +107,8 @@ struct SPage { }) // APIs +#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) + int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); void tdbPageZero(SPage *pPage); @@ -127,6 +116,29 @@ void tdbPageInit(SPage *pPage); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); +static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { + SCell *pCell; + int iOvfl; + int lidx; + + ASSERT(idx >= 0 && idx < pPage->nOverflow + pPage->pPageMethods->getCellNum(pPage)); + + iOvfl = 0; + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] == idx) { + pCell = pPage->apOvfl[iOvfl]; + return pCell; + } else if (pPage->aiOvfl[iOvfl] > idx) { + break; + } + } + + lidx = idx - iOvfl; + pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); + + return pCell; +} + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 4673a561e6..2b0e573aae 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -18,9 +18,20 @@ extern SPageMethods pageMethods; extern SPageMethods pageLargeMethods; -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) -#define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) -#define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) +#define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) +#define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) +#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) +#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage) +#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage) +#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) +#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx) +#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS) +#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS) +#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL) +#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) +#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) +#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) #define TDB_PAGE_MAX_FREE_BLOCK(pPage) \ ((pPage)->pageSize - (pPage)->szAmHdr - TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) @@ -47,8 +58,8 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) pPage = (SPage *)(ptr + pageSize); TDB_INIT_PAGE_LOCK(pPage); - pPage->pData = ptr; pPage->pageSize = pageSize; + pPage->pData = ptr; if (pageSize < 65536) { pPage->pPageMethods = &pageMethods; } else { @@ -125,6 +136,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); } else { + // TODO: make it extensible // add the cell as an overflow cell for (int i = pPage->nOverflow; i > iOvfl; i--) { pPage->apOvfl[i] = pPage->apOvfl[i - 1]; From cde7a7c6a1558a8bb76ad3d7bd83fef44fc89172 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 02:54:00 +0000 Subject: [PATCH 10/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 14 ++++++++------ source/libs/tdb/src/inc/tdbPage.h | 6 +++--- source/libs/tdb/src/page/tdbPage.c | 24 ++++++++++++++++-------- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 621afc9dfb..0c83dd609c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -381,19 +381,20 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { SBTree *pBt; u16 flags; u8 isLeaf; + u8 szAmHdr; pBt = (SBTree *)arg; flags = TDB_PAGE_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { - pPage->szAmHdr = 0; + szAmHdr = 0; } else { - pPage->szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SBtPageHdr); } pPage->xCellSize = NULL; // TODO - tdbPageInit(pPage); + tdbPageInit(pPage, szAmHdr); TDB_BTREE_ASSERT_FLAG(flags); @@ -416,6 +417,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u16 flags; SBTree *pBt; u8 isLeaf; + u8 szAmHdr; flags = ((SBtreeZeroPageArg *)arg)->flags; pBt = ((SBtreeZeroPageArg *)arg)->pBt; @@ -425,13 +427,13 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { TDB_PAGE_FLAGS_SET(pPage, flags); // Set szAmHdr if (isLeaf) { - pPage->szAmHdr = 0; + szAmHdr = 0; } else { - pPage->szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SBtPageHdr); } pPage->xCellSize = NULL; // TODO - tdbPageZero(pPage); + tdbPageZero(pPage, szAmHdr); if (isLeaf) { pPage->kLen = pBt->keyLen; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 82287b0921..58ae1f3963 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -61,7 +61,6 @@ struct SPage { u8 *pData; SPageMethods *pPageMethods; // Fields below used by pager and am - u8 szAmHdr; u8 *pAmHdr; u8 *pPageHdr; u8 *pCellIdx; @@ -111,8 +110,8 @@ struct SPage { int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -void tdbPageZero(SPage *pPage); -void tdbPageInit(SPage *pPage); +void tdbPageZero(SPage *pPage, u8 szAmHdr); +void tdbPageInit(SPage *pPage, u8 szAmHdr); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); @@ -134,6 +133,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { } lidx = idx - iOvfl; + ASSERT(lidx >= 0 && lidx < pPage->pPageMethods->getCellNum(pPage)); pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); return pCell; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 2b0e573aae..11c9a9247e 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -32,8 +32,8 @@ extern SPageMethods pageLargeMethods; #define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) #define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) #define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) -#define TDB_PAGE_MAX_FREE_BLOCK(pPage) \ - ((pPage)->pageSize - (pPage)->szAmHdr - TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) +#define TDB_PAGE_MAX_FREE_BLOCK(pPage, szAmHdr) \ + ((pPage)->pageSize - (szAmHdr)-TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); static int tdbPageDefragment(SPage *pPage); @@ -79,17 +79,25 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) return 0; } -void tdbPageZero(SPage *pPage) { +void tdbPageZero(SPage *pPage, u8 szAmHdr) { + pPage->pAmHdr = pPage->pData; + pPage->pPageHdr = pPage->pAmHdr + szAmHdr; TDB_PAGE_NCELLS_SET(pPage, 0); TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); TDB_PAGE_FCELL_SET(pPage, 0); - TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_MAX_FREE_BLOCK(pPage)); - tdbPageInit(pPage); + TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_MAX_FREE_BLOCK(pPage, szAmHdr)); + pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); + pPage->pFreeStart = pPage->pCellIdx; + pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); + pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + pPage->nOverflow = 0; + + ASSERT((u8 *)pPage->pPageFtr == pPage->pFreeEnd); } -void tdbPageInit(SPage *pPage) { +void tdbPageInit(SPage *pPage, u8 szAmHdr) { pPage->pAmHdr = pPage->pData; - pPage->pPageHdr = pPage->pAmHdr + pPage->szAmHdr; + pPage->pPageHdr = pPage->pAmHdr + szAmHdr; pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); @@ -107,7 +115,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { int lidx; // local idx SCell *pNewCell; - ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage)); + ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pAmHdr)); nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); From 50aa291ff60089a8270f3b85f3e4bd84d0a2aae9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 03:29:39 +0000 Subject: [PATCH 11/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0c83dd609c..59c132ecb2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -423,9 +423,6 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { pBt = ((SBtreeZeroPageArg *)arg)->pBt; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); - // Init the page header - TDB_PAGE_FLAGS_SET(pPage, flags); - // Set szAmHdr if (isLeaf) { szAmHdr = 0; } else { @@ -433,6 +430,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { } pPage->xCellSize = NULL; // TODO + TDB_PAGE_FLAGS_SET(pPage, flags); tdbPageZero(pPage, szAmHdr); if (isLeaf) { From 03eaa648d68c2b9f11a54f780d88e13630809d99 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 03:42:18 +0000 Subject: [PATCH 12/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 59c132ecb2..52f0c7d466 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -385,6 +385,9 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { pBt = (SBTree *)arg; + ASSERT(0); + + // TODO: here has problem flags = TDB_PAGE_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { @@ -430,8 +433,8 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { } pPage->xCellSize = NULL; // TODO - TDB_PAGE_FLAGS_SET(pPage, flags); tdbPageZero(pPage, szAmHdr); + TDB_PAGE_FLAGS_SET(pPage, flags); if (isLeaf) { pPage->kLen = pBt->keyLen; From 4e8f8445de7f719dbef9a75321253df3d4b1fff5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 05:47:06 +0000 Subject: [PATCH 13/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 41 +++++++++++++++++------------ source/libs/tdb/src/inc/tdbPage.h | 4 +++ source/libs/tdb/src/page/tdbPage.c | 10 ++++--- source/libs/tdb/src/page/tdbPageL.c | 10 ++++--- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 52f0c7d466..4b2394f861 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -39,9 +39,16 @@ struct SBTree { u8 *pTmp; }; +#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; + typedef struct __attribute__((__packed__)) { - SPgno rChild; -} SBtPageHdr; + TDB_BTREE_PAGE_COMMON_HDR +} SLeafHdr; + +typedef struct __attribute__((__packed__)) { + TDB_BTREE_PAGE_COMMON_HDR; + SPgno pgno; // right-most child +} SIntHdr; typedef struct { u16 flags; @@ -220,7 +227,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p if (TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0) { // Current page is empty - ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); + // ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); return 0; } @@ -265,7 +272,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p } // Move downward or break - u16 flags = TDB_PAGE_FLAGS(pPage); + u16 flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (leaf) { pCur->idx = midx; @@ -279,7 +286,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p if (midx == nCells - 1) { /* Move to right-most child */ pCur->idx = midx + 1; - tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); + // tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); } else { // TODO: reset cd as uninitialized pCur->idx = midx + 1; @@ -388,12 +395,12 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { ASSERT(0); // TODO: here has problem - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { - szAmHdr = 0; + szAmHdr = sizeof(SLeafHdr); } else { - szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SIntHdr); } pPage->xCellSize = NULL; // TODO @@ -427,14 +434,14 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { - szAmHdr = 0; + szAmHdr = sizeof(SLeafHdr); } else { - szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SIntHdr); } pPage->xCellSize = NULL; // TODO tdbPageZero(pPage, szAmHdr); - TDB_PAGE_FLAGS_SET(pPage, flags); + // TDB_PAGE_FLAGS_SET(pPage, flags); if (isLeaf) { pPage->kLen = pBt->keyLen; @@ -522,7 +529,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } - ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; + // TODO: ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; *ppChild = pChild; return 0; @@ -566,7 +573,7 @@ static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { i = pBlh->nOld - 1; if (idxStart + i == nCells) { - pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; + // pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; } else { pCell = tdbPageGetCell(pParent, idxStart + i); // TODO: no need to decode the payload part, and even the kLen, vLen part @@ -676,7 +683,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; SBtreeBalanceHelper blh; - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); + // ASSERT(!TDB_BTREE_PGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); blh.pBt = pBt; blh.pParent = pParent; @@ -748,7 +755,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { for (;;) { iPage = pCur->iPage; pPage = pCur->pPage; - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); root = TDB_BTREE_PAGE_IS_ROOT(flags); @@ -842,7 +849,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo nPayload = 0; nHeader = 0; - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // 1. Encode Header part @@ -911,7 +918,7 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD int ret; nHeader = 0; - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Clear the state of decoder diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 58ae1f3963..155e7b2e37 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -27,9 +27,11 @@ typedef struct { int szOffset; int szPageHdr; int szFreeCell; +#if 0 // flags u16 (*getFlags)(SPage *); void (*setFlags)(SPage *, u16); +#endif // cell number int (*getCellNum)(SPage *); void (*setCellNum)(SPage *, int); @@ -79,9 +81,11 @@ struct SPage { TDB_PCACHE_PAGE }; +#if 0 /* For page */ #define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) #define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) +#endif // For page lock #define P_LOCK_SUCC 0 diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 11c9a9247e..61793b6025 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -399,9 +399,11 @@ typedef struct __attribute__((__packed__)) { u16 nxOffset; } SFreeCell; +#if 0 // flags static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } +#endif // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } @@ -456,11 +458,13 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageMethods = { - 2, // szOffset - sizeof(SPageHdr), // szPageHdr - sizeof(SFreeCell), // szFreeCell + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell +#if 0 getPageFlags, // getPageFlags setPageFlags, // setFlagsp +#endif getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index e3c1bb8d37..f4bcc62494 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -28,9 +28,11 @@ typedef struct __attribute__((__packed__)) { u8 nxOffset[3]; } SFreeCellL; +#if 0 // flags static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdrL *)(pPage->pPageHdr))[0].flags; } static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdrL *)(pPage->pPageHdr))[0].flags = flags; } +#endif // cellNum static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } @@ -80,11 +82,13 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell +#if 0 getPageFlags, // getPageFlags setPageFlags, // setFlagsp +#endif getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody From df306127a644ebec5d8dc6ff37264ed8c4d9fed3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 05:56:33 +0000 Subject: [PATCH 14/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 3 ++- source/libs/tdb/src/inc/tdbPage.h | 11 ----------- source/libs/tdb/src/page/tdbPage.c | 16 +++------------- source/libs/tdb/src/page/tdbPageL.c | 16 +++------------- 4 files changed, 8 insertions(+), 38 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4b2394f861..68319da3b2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -39,7 +39,8 @@ struct SBTree { u8 *pTmp; }; -#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; +#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; +#define TDB_BTREE_PAGE_FLAGS(PAGE) (PAGE)->pAmHdr[0] typedef struct __attribute__((__packed__)) { TDB_BTREE_PAGE_COMMON_HDR diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 155e7b2e37..dc3b803708 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -27,11 +27,6 @@ typedef struct { int szOffset; int szPageHdr; int szFreeCell; -#if 0 - // flags - u16 (*getFlags)(SPage *); - void (*setFlags)(SPage *, u16); -#endif // cell number int (*getCellNum)(SPage *); void (*setCellNum)(SPage *, int); @@ -81,12 +76,6 @@ struct SPage { TDB_PCACHE_PAGE }; -#if 0 -/* For page */ -#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) -#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) -#endif - // For page lock #define P_LOCK_SUCC 0 #define P_LOCK_BUSY 1 diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 61793b6025..b3a3bdbdcc 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -399,12 +399,6 @@ typedef struct __attribute__((__packed__)) { u16 nxOffset; } SFreeCell; -#if 0 -// flags -static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } -static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } -#endif - // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } static inline void setPageCellNum(SPage *pPage, int cellNum) { @@ -458,13 +452,9 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageMethods = { - 2, // szOffset - sizeof(SPageHdr), // szPageHdr - sizeof(SFreeCell), // szFreeCell -#if 0 - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp -#endif + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index f4bcc62494..cfd8c919df 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -28,12 +28,6 @@ typedef struct __attribute__((__packed__)) { u8 nxOffset[3]; } SFreeCellL; -#if 0 -// flags -static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdrL *)(pPage->pPageHdr))[0].flags; } -static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdrL *)(pPage->pPageHdr))[0].flags = flags; } -#endif - // cellNum static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } static inline void setPageCellNum(SPage *pPage, int cellNum) { @@ -82,13 +76,9 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell -#if 0 - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp -#endif + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody From 2b7da2a79b1972c35853419cb564e9e429f40609 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:05:32 +0000 Subject: [PATCH 15/92] more TDB --- source/libs/tdb/src/btree/tdbBtreeBalance.c | 14 +++++++++++ source/libs/tdb/src/btree/tdbBtreeCommon.c | 14 +++++++++++ source/libs/tdb/src/btree/tdbBtreeInt.h | 27 +++++++++++++++++++++ source/libs/tdb/src/btree/tdbBtreeOpen.c | 14 +++++++++++ 4 files changed, 69 insertions(+) create mode 100644 source/libs/tdb/src/btree/tdbBtreeBalance.c create mode 100644 source/libs/tdb/src/btree/tdbBtreeCommon.c create mode 100644 source/libs/tdb/src/btree/tdbBtreeInt.h create mode 100644 source/libs/tdb/src/btree/tdbBtreeOpen.c diff --git a/source/libs/tdb/src/btree/tdbBtreeBalance.c b/source/libs/tdb/src/btree/tdbBtreeBalance.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeBalance.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeCommon.c b/source/libs/tdb/src/btree/tdbBtreeCommon.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeCommon.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeInt.h b/source/libs/tdb/src/btree/tdbBtreeInt.h new file mode 100644 index 0000000000..b8a935a614 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeInt.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TDB_BTREE_INT_H_ +#define _TDB_BTREE_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TDB_BTREE_INT_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeOpen.c b/source/libs/tdb/src/btree/tdbBtreeOpen.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeOpen.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file From d9c4bcab6ccb19c9494587472c7ee6598e7cdd94 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:14:47 +0000 Subject: [PATCH 16/92] more TDB --- source/libs/tdb/CMakeLists.txt | 1 - source/libs/tdb/src/btree/tdbBtreeDelete.c | 14 ++++++++++++++ source/libs/tdb/src/btree/tdbBtreeInsert.c | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 source/libs/tdb/src/btree/tdbBtreeDelete.c create mode 100644 source/libs/tdb/src/btree/tdbBtreeInsert.c diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 978649499a..a9b56d42b8 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -8,7 +8,6 @@ target_sources(tdb "src/db/tdbBtree.c" "src/db/tdbDb.c" "src/db/tdbEnv.c" - # "src/db/tdbPage.c" "src/page/tdbPage.c" "src/page/tdbPageL.c" ) diff --git a/source/libs/tdb/src/btree/tdbBtreeDelete.c b/source/libs/tdb/src/btree/tdbBtreeDelete.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeDelete.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeInsert.c b/source/libs/tdb/src/btree/tdbBtreeInsert.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeInsert.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file From a84155b951b24d87b204a2a4009c9c19cbe6f2cc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:23:32 +0000 Subject: [PATCH 17/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 21 ++++++++++++++------- source/libs/tdb/src/inc/tdbPage.h | 2 +- source/libs/tdb/src/page/tdbPage.c | 4 ++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 68319da3b2..2a2d630cd8 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -52,9 +52,9 @@ typedef struct __attribute__((__packed__)) { } SIntHdr; typedef struct { - u16 flags; + u8 flags; SBTree *pBt; -} SBtreeZeroPageArg; +} SBtreeInitPageArg; typedef struct { int kLen; @@ -74,6 +74,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo int *szCell); static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); static int tdbBtreeBalance(SBtCursor *pCur); +static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; @@ -371,7 +372,7 @@ static int tdbBtreeOpenImpl(SBTree *pBt) { } // Try to create a new database - SBtreeZeroPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; + SBtreeInitPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg); if (ret < 0) { return -1; @@ -430,8 +431,8 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u8 isLeaf; u8 szAmHdr; - flags = ((SBtreeZeroPageArg *)arg)->flags; - pBt = ((SBtreeZeroPageArg *)arg)->pBt; + flags = ((SBtreeInitPageArg *)arg)->flags; + pBt = ((SBtreeInitPageArg *)arg)->pBt; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { @@ -498,7 +499,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPage *pChild; SPgno pgnoChild; int ret; - SBtreeZeroPageArg zArg; + SBtreeInitPageArg zArg; pPager = pRoot->pPager; @@ -959,4 +960,10 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD return 0; } -#endif \ No newline at end of file +#endif + +static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { + // TODO + ASSERT(0); + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index dc3b803708..0960d88c33 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -71,7 +71,7 @@ struct SPage { int vLen; // value length of the page, -1 for unknown int maxLocal; int minLocal; - int (*xCellSize)(SCell *pCell); + int (*xCellSize)(const SPage *, SCell *); // Fields used by SPCache TDB_PCACHE_PAGE }; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index b3a3bdbdcc..7088904285 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -193,7 +193,7 @@ int tdbPageDropCell(SPage *pPage, int idx) { lidx = idx - iOvfl; pCell = TDB_PAGE_CELL_AT(pPage, lidx); - szCell = (*pPage->xCellSize)(pCell); + szCell = (*pPage->xCellSize)(pPage, pCell); tdbPageFree(pPage, lidx, pCell, szCell); TDB_PAGE_NCELLS_SET(pPage, nCells - 1); @@ -366,7 +366,7 @@ static int tdbPageDefragment(SPage *pPage) { ASSERT(pCell != NULL); - szCell = (*pPage->xCellSize)(pCell); + szCell = (*pPage->xCellSize)(pPage, pCell); ASSERT(pCell + szCell <= pNextCell); if (pCell + szCell < pNextCell) { From 0a6be10ff1d1f5b35c0ffceb095485347f774194 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:34:34 +0000 Subject: [PATCH 18/92] more tdb --- source/libs/tdb/src/db/tdbBtree.c | 44 ++++++++++++++---------------- source/libs/tdb/src/inc/tdbPage.h | 4 +-- source/libs/tdb/src/page/tdbPage.c | 6 ++-- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 2a2d630cd8..06f96e5498 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -39,8 +39,10 @@ struct SBTree { u8 *pTmp; }; -#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; -#define TDB_BTREE_PAGE_FLAGS(PAGE) (PAGE)->pAmHdr[0] +#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; + +#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pAmHdr[0] +#define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pAmHdr[0] = (flags)) typedef struct __attribute__((__packed__)) { TDB_BTREE_PAGE_COMMON_HDR @@ -274,8 +276,8 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p } // Move downward or break - u16 flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); - u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + u8 flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (leaf) { pCur->idx = midx; *pCRst = c; @@ -388,25 +390,23 @@ static int tdbBtreeOpenImpl(SBTree *pBt) { static int tdbBtreeInitPage(SPage *pPage, void *arg) { SBTree *pBt; - u16 flags; + u8 flags; u8 isLeaf; u8 szAmHdr; - pBt = (SBTree *)arg; - - ASSERT(0); - - // TODO: here has problem - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + pBt = ((SBtreeInitPageArg *)arg)->pBt; + flags = ((SBtreeInitPageArg *)arg)->flags; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + + ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); + if (isLeaf) { szAmHdr = sizeof(SLeafHdr); } else { szAmHdr = sizeof(SIntHdr); } - pPage->xCellSize = NULL; // TODO - tdbPageInit(pPage, szAmHdr); + tdbPageInit(pPage, szAmHdr, tdbBtreeCellSize); TDB_BTREE_ASSERT_FLAG(flags); @@ -426,7 +426,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { } static int tdbBtreeZeroPage(SPage *pPage, void *arg) { - u16 flags; + u8 flags; SBTree *pBt; u8 isLeaf; u8 szAmHdr; @@ -440,10 +440,8 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { } else { szAmHdr = sizeof(SIntHdr); } - pPage->xCellSize = NULL; // TODO - tdbPageZero(pPage, szAmHdr); - // TDB_PAGE_FLAGS_SET(pPage, flags); + tdbPageZero(pPage, szAmHdr, tdbBtreeCellSize); if (isLeaf) { pPage->kLen = pBt->keyLen; @@ -749,7 +747,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { SPage *pParent; SPage *pPage; int ret; - u16 flags; + u8 flags; u8 leaf; u8 root; @@ -757,7 +755,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { for (;;) { iPage = pCur->iPage; pPage = pCur->pPage; - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); root = TDB_BTREE_PAGE_IS_ROOT(flags); @@ -840,7 +838,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, u8 *pPayload, const void *pKey, i static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell) { - u16 flags; + u8 flags; u8 leaf; int nHeader; int nPayload; @@ -851,7 +849,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo nPayload = 0; nHeader = 0; - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // 1. Encode Header part @@ -914,13 +912,13 @@ static int tdbBtreeDecodePayload(SPage *pPage, const u8 *pPayload, SCellDecoder } static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder) { - u16 flags; + u8 flags; u8 leaf; int nHeader; int ret; nHeader = 0; - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Clear the state of decoder diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 0960d88c33..b9007b99e0 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -103,8 +103,8 @@ struct SPage { int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -void tdbPageZero(SPage *pPage, u8 szAmHdr); -void tdbPageInit(SPage *pPage, u8 szAmHdr); +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 7088904285..97ed9bb703 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -79,7 +79,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) return 0; } -void tdbPageZero(SPage *pPage, u8 szAmHdr) { +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { pPage->pAmHdr = pPage->pData; pPage->pPageHdr = pPage->pAmHdr + szAmHdr; TDB_PAGE_NCELLS_SET(pPage, 0); @@ -91,11 +91,12 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr) { pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); pPage->nOverflow = 0; + pPage->xCellSize = xCellSize; ASSERT((u8 *)pPage->pPageFtr == pPage->pFreeEnd); } -void tdbPageInit(SPage *pPage, u8 szAmHdr) { +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { pPage->pAmHdr = pPage->pData; pPage->pPageHdr = pPage->pAmHdr + szAmHdr; pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); @@ -103,6 +104,7 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr) { pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); pPage->nOverflow = 0; + pPage->xCellSize = xCellSize; ASSERT(pPage->pFreeEnd >= pPage->pFreeStart); ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); From ad0e3a007bba08799cf7b193db4b2f97323edf27 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:45:08 +0000 Subject: [PATCH 19/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 06f96e5498..8b4bef50a5 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -392,7 +392,6 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { SBTree *pBt; u8 flags; u8 isLeaf; - u8 szAmHdr; pBt = ((SBtreeInitPageArg *)arg)->pBt; flags = ((SBtreeInitPageArg *)arg)->flags; @@ -400,13 +399,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); - if (isLeaf) { - szAmHdr = sizeof(SLeafHdr); - } else { - szAmHdr = sizeof(SIntHdr); - } - - tdbPageInit(pPage, szAmHdr, tdbBtreeCellSize); + tdbPageInit(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); TDB_BTREE_ASSERT_FLAG(flags); @@ -429,26 +422,26 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u8 flags; SBTree *pBt; u8 isLeaf; - u8 szAmHdr; flags = ((SBtreeInitPageArg *)arg)->flags; pBt = ((SBtreeInitPageArg *)arg)->pBt; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); - if (isLeaf) { - szAmHdr = sizeof(SLeafHdr); - } else { - szAmHdr = sizeof(SIntHdr); - } - - tdbPageZero(pPage, szAmHdr, tdbBtreeCellSize); + tdbPageZero(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); if (isLeaf) { + SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pPageHdr); + pLeafHdr->flags = flags; + pPage->kLen = pBt->keyLen; pPage->vLen = pBt->valLen; pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; } else { + SIntHdr *pIntHdr = (SIntHdr *)(pPage->pPageHdr); + pIntHdr->flags = flags; + pIntHdr->pgno = 0; + pPage->kLen = pBt->keyLen; pPage->vLen = sizeof(SPgno); pPage->maxLocal = pBt->maxLocal; From 9c8102027fc921e0c9d35f50b16e4579a5df6569 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:46:37 +0000 Subject: [PATCH 20/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8b4bef50a5..f72ce41355 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -430,7 +430,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { tdbPageZero(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); if (isLeaf) { - SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pPageHdr); + SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pAmHdr); pLeafHdr->flags = flags; pPage->kLen = pBt->keyLen; @@ -438,7 +438,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; } else { - SIntHdr *pIntHdr = (SIntHdr *)(pPage->pPageHdr); + SIntHdr *pIntHdr = (SIntHdr *)(pPage->pAmHdr); pIntHdr->flags = flags; pIntHdr->pgno = 0; From 0cd8cca66dd1aa03b16079de53514c4545ec9d31 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:55:17 +0000 Subject: [PATCH 21/92] more TDB --- source/libs/tdb/src/page/tdbPage.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 97ed9bb703..4a067bb978 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -141,7 +141,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { u8 *src = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * lidx; u8 *dest = src + TDB_PAGE_OFFSET_SIZE(pPage); memmove(dest, src, pPage->pFreeStart - dest); - TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, TDB_PAGE_OFFSET_SIZE(pPage) * lidx); + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, pNewCell - pPage->pData); TDB_PAGE_NCELLS_SET(pPage, nCells + 1); ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); From 8becb5421110425ce073ce2e8fb70176087862b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 07:30:34 +0000 Subject: [PATCH 22/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 7 ------- source/libs/tdb/src/inc/tdbPage.h | 1 + source/libs/tdb/src/page/tdbPage.c | 10 ++++++++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f72ce41355..4c7cddf311 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -752,13 +752,6 @@ static int tdbBtreeBalance(SBtCursor *pCur) { leaf = TDB_BTREE_PAGE_IS_LEAF(flags); root = TDB_BTREE_PAGE_IS_ROOT(flags); - // TODO: Get the page free space if not get yet - // if (pPage->nFree < 0) { - // if (tdbBtreeComputeFreeSpace(pPage) < 0) { - // return -1; - // } - // } - // when the page is not overflow and not too empty, the balance work // is finished. Just break out the balance loop. if (pPage->nOverflow == 0 /* TODO: && pPage->nFree <= */) { diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index b9007b99e0..00322067df 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -107,6 +107,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); +void tdbPageCopy(SPage *pFromPage, SPage *pToPage); static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { SCell *pCell; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 4a067bb978..c3fec25d81 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -207,6 +207,16 @@ int tdbPageDropCell(SPage *pPage, int idx) { return 0; } +void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { + pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); + pToPage->pFreeEnd = (u8 *)(pToPage->pPageFtr) - ((u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); + + ASSERT(pToPage->pFreeEnd >= pToPage->pFreeStart); + + memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); + memcpy(pToPage->pFreeEnd, pFromPage->pFreeEnd, (u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); +} + static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { SCell *pFreeCell; u8 *pOffset; From b8954ae1c5036a91293b385c80fc69501a8e5e82 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 07:55:56 +0000 Subject: [PATCH 23/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 28 +--------------------------- source/libs/tdb/src/page/tdbPage.c | 12 ++++++++++++ 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4c7cddf311..9e6ea91a4c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -462,29 +462,6 @@ typedef struct { SPage *pNewPages[5]; } SBtreeBalanceHelper; -static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { -#if 0 - int nCells = TDB_PAGE_TOTAL_CELLS(pFrom); - int cCells = TDB_PAGE_CCELLS(pFrom); - int fCell = TDB_PAGE_FCELL(pFrom); - int nFree = TDB_PAGE_NFREE(pFrom); - - pTo->pFreeStart = pTo->pCellIdx + nCells * pFrom->pPageMethods->szOffset; - memcpy(pTo->pCellIdx, pFrom->pCellIdx, nCells * pFrom->pPageMethods->szOffset); - pTo->pFreeEnd = (u8 *)pTo->pPageFtr - (u8 *)(pFrom->pPageFtr) + pFrom->pFreeEnd; - memcpy(pTo->pFreeEnd, pFrom->pFreeEnd, (u8 *)pFrom->pPageFtr - pFrom->pFreeEnd); - - TDB_PAGE_NCELLS_SET(pTo, nCells); - TDB_PAGE_CCELLS_SET(pTo, cCells); - TDB_PAGE_FCELL_SET(pTo, fCell); - TDB_PAGE_NFREE_SET(pTo, nFree); - - // TODO: update other fields -#endif - - return 0; -} - static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPager *pPager; SPage *pChild; @@ -503,10 +480,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { } // Copy the root page content to the child page - ret = tdbBtreeCopyPageContent(pRoot, pChild); - if (ret < 0) { - return -1; - } + tdbPageCopy(pRoot, pChild); pChild->nOverflow = pRoot->nOverflow; for (int i = 0; i < pChild->nOverflow; i++) { diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index c3fec25d81..803bf168cf 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -208,6 +208,8 @@ int tdbPageDropCell(SPage *pPage, int idx) { } void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { + int delta, nFree; + pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); pToPage->pFreeEnd = (u8 *)(pToPage->pPageFtr) - ((u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); @@ -215,6 +217,16 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); memcpy(pToPage->pFreeEnd, pFromPage->pFreeEnd, (u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); + + ASSERT(TDB_PAGE_CCELLS(pToPage) == pToPage->pFreeEnd - pToPage->pData); + + delta = (pToPage->pPageHdr - pToPage->pAmHdr) - (pFromPage->pPageHdr - pFromPage->pAmHdr); + if (delta != 0) { + nFree = TDB_PAGE_NFREE(pFromPage); + TDB_PAGE_NFREE_SET(pToPage, nFree - delta); + } + + // TODO: do we need to copy the overflow part ??? } static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { From 2e6f7f15d5934ab67f6ee2cb1f2e7bb435bccb9e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 07:59:37 +0000 Subject: [PATCH 24/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 6 ------ source/libs/tdb/src/page/tdbPage.c | 7 ++++++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 9e6ea91a4c..5f56b2f1e7 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -482,12 +482,6 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { // Copy the root page content to the child page tdbPageCopy(pRoot, pChild); - pChild->nOverflow = pRoot->nOverflow; - for (int i = 0; i < pChild->nOverflow; i++) { - pChild->apOvfl[i] = pRoot->apOvfl[i]; - pChild->aiOvfl[i] = pRoot->aiOvfl[i]; - } - // Reinitialize the root page zArg.flags = TDB_BTREE_ROOT; zArg.pBt = pBt; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 803bf168cf..2f6d4101a5 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -226,7 +226,12 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { TDB_PAGE_NFREE_SET(pToPage, nFree - delta); } - // TODO: do we need to copy the overflow part ??? + // Copy the overflow cells + for (int iOvfl = 0; iOvfl < pFromPage->nOverflow; iOvfl++) { + pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl]; + pToPage->apOvfl[iOvfl] = pFromPage->apOvfl[iOvfl]; + } + pToPage->nOverflow = pFromPage->nOverflow; } static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { From 5ca3e29cf19ed454ec9a0a4e7c00c3d474d01c99 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 08:17:58 +0000 Subject: [PATCH 25/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 6 ++++-- source/libs/tdb/src/page/tdbPage.c | 1 - source/libs/tdb/src/page/tdbPageL.c | 9 ++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 5f56b2f1e7..4e4b2275f8 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -467,6 +467,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPage *pChild; SPgno pgnoChild; int ret; + SIntHdr *pIntHdr; SBtreeInitPageArg zArg; pPager = pRoot->pPager; @@ -490,7 +491,8 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } - // TODO: ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; + pIntHdr = (SIntHdr *)(pRoot->pAmHdr); + pIntHdr->pgno = pgnoChild; *ppChild = pChild; return 0; @@ -731,7 +733,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { // ignore the case of empty if (pPage->nOverflow == 0) break; - ret = tdbBtreeBalanceDeeper(pCur->pBt, pCur->pPage, &(pCur->pgStack[1])); + ret = tdbBtreeBalanceDeeper(pCur->pBt, pPage, &(pCur->pgStack[1])); if (ret < 0) { return -1; } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 2f6d4101a5..bc00da9381 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -416,7 +416,6 @@ static int tdbPageDefragment(SPage *pPage) { /* ---------------------------------------------------------------------------------------------------------- */ typedef struct __attribute__((__packed__)) { - u16 flags; u16 cellNum; u16 cellBody; u16 cellFree; diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index cfd8c919df..c5d4a6047f 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -16,11 +16,10 @@ #include "tdbInt.h" typedef struct __attribute__((__packed__)) { - u16 flags; - u8 cellNum[3]; - u8 cellBody[3]; - u8 cellFree[3]; - u8 nFree[3]; + u8 cellNum[3]; + u8 cellBody[3]; + u8 cellFree[3]; + u8 nFree[3]; } SPageHdrL; typedef struct __attribute__((__packed__)) { From db870af8b7afb105417d4d3d7e6acaaf1bc90235 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 09:33:10 +0000 Subject: [PATCH 26/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 65 +++++++++++++++++++++++++----- source/libs/tdb/src/inc/tdbPage.h | 3 +- source/libs/tdb/src/page/tdbPage.c | 10 ++--- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4e4b2275f8..b801002822 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -41,8 +41,8 @@ struct SBTree { #define TDB_BTREE_PAGE_COMMON_HDR u8 flags; -#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pAmHdr[0] -#define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pAmHdr[0] = (flags)) +#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pData[0] +#define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pData[0] = (flags)) typedef struct __attribute__((__packed__)) { TDB_BTREE_PAGE_COMMON_HDR @@ -393,8 +393,8 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { u8 flags; u8 isLeaf; - pBt = ((SBtreeInitPageArg *)arg)->pBt; - flags = ((SBtreeInitPageArg *)arg)->flags; + pBt = (SBTree *)arg; + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); @@ -430,7 +430,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { tdbPageZero(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); if (isLeaf) { - SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pAmHdr); + SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pData); pLeafHdr->flags = flags; pPage->kLen = pBt->keyLen; @@ -438,7 +438,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; } else { - SIntHdr *pIntHdr = (SIntHdr *)(pPage->pAmHdr); + SIntHdr *pIntHdr = (SIntHdr *)(pPage->pData); pIntHdr->flags = flags; pIntHdr->pgno = 0; @@ -491,7 +491,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } - pIntHdr = (SIntHdr *)(pRoot->pAmHdr); + pIntHdr = (SIntHdr *)(pRoot->pData); pIntHdr->pgno = pgnoChild; *ppChild = pChild; @@ -643,15 +643,59 @@ static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { } static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { - int ret; + int ret; +#if 0 SBtreeBalanceHelper blh; - // ASSERT(!TDB_BTREE_PGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); - blh.pBt = pBt; blh.pParent = pParent; blh.idx = idx; +#endif + int nOlds; + SPage *pOlds[3]; + + { + // Find 3 child pages at most to do balance + int nCells = TDB_PAGE_TOTAL_CELLS(pParent); + int sIdx; + if (nCells <= 2) { + sIdx = 0; + nOlds = nCells + 1; + } else { + // has more than three child pages + if (idx == 0) { + sIdx = 0; + } else if (idx == nCells) { + sIdx = idx - 2; + } else { + sIdx = idx - 1; + } + nOlds = 3; + } + for (int i = 0; i < nOlds; i++, sIdx++) { + ASSERT(sIdx <= nCells); + + SPgno pgno; + if (sIdx == nCells) { + ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pParent))); + pgno = ((SIntHdr *)(pParent->pData))->pgno; + } else { + SCell *pCell; + + pCell = tdbPageGetCell(pParent, sIdx); + pgno = *(SPgno *)pCell; + } + + ret = tdbPagerFetchPage(pBt->pPager, pgno, pOlds + i, tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + } + +#if 0 // Step 1: find two sibling pages and get engough info about the old pages ret = tdbBtreeBalanceStep1(&blh); if (ret < 0) { @@ -693,6 +737,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { ASSERT(0); return -1; } +#endif { // TODO: Reset states diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 00322067df..e513a63fe4 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -58,7 +58,6 @@ struct SPage { u8 *pData; SPageMethods *pPageMethods; // Fields below used by pager and am - u8 *pAmHdr; u8 *pPageHdr; u8 *pCellIdx; u8 *pFreeStart; @@ -114,7 +113,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { int iOvfl; int lidx; - ASSERT(idx >= 0 && idx < pPage->nOverflow + pPage->pPageMethods->getCellNum(pPage)); + ASSERT(idx >= 0 && idx < TDB_PAGE_TOTAL_CELLS(pPage)); iOvfl = 0; for (; iOvfl < pPage->nOverflow; iOvfl++) { diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index bc00da9381..0dc7853940 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -80,8 +80,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) } void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { - pPage->pAmHdr = pPage->pData; - pPage->pPageHdr = pPage->pAmHdr + szAmHdr; + pPage->pPageHdr = pPage->pData + szAmHdr; TDB_PAGE_NCELLS_SET(pPage, 0); TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); TDB_PAGE_FCELL_SET(pPage, 0); @@ -97,8 +96,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell } void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { - pPage->pAmHdr = pPage->pData; - pPage->pPageHdr = pPage->pAmHdr + szAmHdr; + pPage->pPageHdr = pPage->pData + szAmHdr; pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); @@ -117,7 +115,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { int lidx; // local idx SCell *pNewCell; - ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pAmHdr)); + ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); @@ -220,7 +218,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { ASSERT(TDB_PAGE_CCELLS(pToPage) == pToPage->pFreeEnd - pToPage->pData); - delta = (pToPage->pPageHdr - pToPage->pAmHdr) - (pFromPage->pPageHdr - pFromPage->pAmHdr); + delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); if (delta != 0) { nFree = TDB_PAGE_NFREE(pFromPage); TDB_PAGE_NFREE_SET(pToPage, nFree - delta); From b3b5298510d28f1a75ec917dcf8cec1978538697 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 09:47:46 +0000 Subject: [PATCH 27/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 34 ++++++++++++++++--------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index b801002822..52ea45578b 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -695,6 +695,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + // >>>>>>>>>>>>>>>>>>> + #if 0 // Step 1: find two sibling pages and get engough info about the old pages ret = tdbBtreeBalanceStep1(&blh); @@ -854,6 +856,14 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // 1. Encode Header part + /* Encode SPgno if interior page */ + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; + nHeader = nHeader + sizeof(SPgno); + } + /* Encode kLen if need */ if (pPage->kLen == TDB_VARIANT_LEN) { nHeader += tdbPutVarInt(pCell + nHeader, kLen); @@ -864,14 +874,6 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo nHeader += tdbPutVarInt(pCell + nHeader, vLen); } - /* Encode SPgno if interior page */ - if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); - - ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; - nHeader = nHeader + sizeof(SPgno); - } - // 2. Encode payload part if (leaf) { ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, pVal, vLen, &nPayload); @@ -930,6 +932,14 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD pDecoder->pgno = 0; // 1. Decode header part + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; + pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); + nHeader = nHeader + sizeof(SPgno); + } + if (pPage->kLen == TDB_VARIANT_LEN) { nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->kLen)); } else { @@ -942,14 +952,6 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD pDecoder->vLen = pPage->vLen; } - if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); - - pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; - pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); - nHeader = nHeader + sizeof(SPgno); - } - // 2. Decode payload part ret = tdbBtreeDecodePayload(pPage, pCell + nHeader, pDecoder); if (ret < 0) { From 1075c230ed0615f80449000afe09feafd9a63723 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:18:04 +0000 Subject: [PATCH 28/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 37 ++++++++++++++++++++++--------- source/libs/tdb/src/inc/tdbPage.h | 3 ++- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 52ea45578b..93fc75269c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -644,19 +644,11 @@ static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; -#if 0 - SBtreeBalanceHelper blh; - - blh.pBt = pBt; - blh.pParent = pParent; - blh.idx = idx; -#endif int nOlds; SPage *pOlds[3]; - { - // Find 3 child pages at most to do balance + { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); int sIdx; if (nCells <= 2) { @@ -695,7 +687,32 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - // >>>>>>>>>>>>>>>>>>> + int nNews = 0; + int cntNews[5] = {0}; // TODO: maybe 5 is not enough + int szNews[5] = {0}; + int maxPageCapacity; // TODO + + { // Get how many new pages are needed and the new distribution + + // loop to find number of pages needed + for (int i = 0; i < nOlds; i++) { + SPage *pPage = pOlds[i]; + SCell *pCell; + int cellBytes; + for (int cIdx = 0; cIdx < TDB_PAGE_TOTAL_CELLS(pPage); cIdx++) { + pCell = tdbPageGetCell(pPage, cIdx); + cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); + + if (szNews[nNews] + cellBytes > maxPageCapacity) { + nNews++; + } + cntNews[nNews]++; + szNews[nNews] = szNews[nNews] + cellBytes; + } + } + + // Loop to make the distribution even + } #if 0 // Step 1: find two sibling pages and get engough info about the old pages diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index e513a63fe4..479c7603e1 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -98,7 +98,8 @@ struct SPage { }) // APIs -#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); From b5b522d66643663ceeabbf5b8a8548153431073a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:23:34 +0000 Subject: [PATCH 29/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 4 ++-- source/libs/tdb/src/inc/tdbPage.h | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 93fc75269c..aa3e4c6b90 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -690,7 +690,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nNews = 0; int cntNews[5] = {0}; // TODO: maybe 5 is not enough int szNews[5] = {0}; - int maxPageCapacity; // TODO { // Get how many new pages are needed and the new distribution @@ -699,11 +698,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SPage *pPage = pOlds[i]; SCell *pCell; int cellBytes; + for (int cIdx = 0; cIdx < TDB_PAGE_TOTAL_CELLS(pPage); cIdx++) { pCell = tdbPageGetCell(pPage, cIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - if (szNews[nNews] + cellBytes > maxPageCapacity) { + if (szNews[nNews] + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { nNews++; } cntNews[nNews]++; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 479c7603e1..258eafcf63 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -99,6 +99,7 @@ struct SPage { // APIs #define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); From bd8c171be1d79dfefd53288739110cb0536ef2a5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:44:18 +0000 Subject: [PATCH 30/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 39 ++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index aa3e4c6b90..4169509280 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -978,10 +978,37 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD return 0; } -#endif - static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { - // TODO - ASSERT(0); - return 0; -} \ No newline at end of file + u8 flags; + u8 isLeaf; + int szCell; + int kLen = 0, vLen = 0; + + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + szCell = 0; + + if (!isLeaf) { + szCell += sizeof(SPgno); + } + + if (pPage->kLen == TDB_VARIANT_LEN) { + szCell += tdbGetVarInt(pCell + szCell, &kLen); + } else { + kLen = pPage->kLen; + } + + if (isLeaf) { + if (pPage->vLen == TDB_VARIANT_LEN) { + szCell += tdbGetVarInt(pCell + szCell, &vLen); + } else { + vLen = pPage->vLen; + } + } + + szCell = szCell + kLen + vLen; + + return szCell; +} + +#endif From d123388a1962630ead69fa7d13e4b92c5f7f4be5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:53:49 +0000 Subject: [PATCH 31/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4169509280..670cdc002a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -711,7 +711,16 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - // Loop to make the distribution even + nNews = nNews + 1; + + // back loop to make the distribution even + for (int iNew = nNews - 1; iNew > 0; iNew--) { + /* code */ + } + } + + SPage *pNews[5]; + { // Allocate the new pages } #if 0 From 925004770202e76b8de27e4a8f4cfbf48c9d67dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 14:56:07 +0000 Subject: [PATCH 32/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 670cdc002a..98a5cf158c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -715,7 +715,34 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { - /* code */ + // TODO: find the last cell of page (iNew-1) + int cIdx; + int iPage; + int cellBytes; + SPage *pPage; + SCell *pCell; + + for (;;) { + pCell = tdbPageGetCell(pPage, cIdx); + cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); + + if (szNews[iNew] + cellBytes >= szNews[iNew - 1] + cellBytes) { + break; + } + + // Move the cell right + szNews[iNew] += cellBytes; + cntNews[iNew]++; + + szNews[iNew - 1] -= cellBytes; + cntNews[iNew - 1]++; + + cIdx--; + if (cIdx < 0) { + pPage = pOlds[--iPage]; + cIdx = TDB_PAGE_TOTAL_CELLS(pPage) - 1; + } + } } } From e2f1434bf2c9d2b5cbe13408f4fbbf876a0ec417 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:08:56 +0000 Subject: [PATCH 33/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 108 ++++++++++-------------------- 1 file changed, 34 insertions(+), 74 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 98a5cf158c..cbb389140f 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -688,26 +688,33 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } int nNews = 0; - int cntNews[5] = {0}; // TODO: maybe 5 is not enough - int szNews[5] = {0}; + struct { + int cnt; + int size; + int oPage; + int oIdx; + } infoNews[5] = {0}; { // Get how many new pages are needed and the new distribution - // loop to find number of pages needed - for (int i = 0; i < nOlds; i++) { - SPage *pPage = pOlds[i]; + // first loop to find minimum number of pages needed + for (int oPage = 0; oPage < nOlds; oPage++) { + SPage *pPage = pOlds[oPage]; SCell *pCell; int cellBytes; - for (int cIdx = 0; cIdx < TDB_PAGE_TOTAL_CELLS(pPage); cIdx++) { - pCell = tdbPageGetCell(pPage, cIdx); + for (int oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { + pCell = tdbPageGetCell(pPage, oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - if (szNews[nNews] + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { + if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { + // page is full, use a new page nNews++; } - cntNews[nNews]++; - szNews[nNews] = szNews[nNews] + cellBytes; + infoNews[nNews].cnt++; + infoNews[nNews].size += cellBytes; + infoNews[nNews].oPage = oPage; + infoNews[nNews].oIdx = oIdx; } } @@ -715,85 +722,38 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { - // TODO: find the last cell of page (iNew-1) - int cIdx; - int iPage; - int cellBytes; - SPage *pPage; SCell *pCell; + SPage *pPage; + int cellBytes; for (;;) { - pCell = tdbPageGetCell(pPage, cIdx); + pPage = pOlds[infoNews[iNew - 1].oPage]; + pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - if (szNews[iNew] + cellBytes >= szNews[iNew - 1] + cellBytes) { - break; + infoNews[iNew].cnt++; + infoNews[iNew].size += cellBytes; + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= cellBytes; + if ((infoNews[iNew - 1].oIdx--) == 0) { + infoNews[iNew - 1].oPage--; + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pOlds[infoNews[iNew - 1].oPage]); } - // Move the cell right - szNews[iNew] += cellBytes; - cntNews[iNew]++; - - szNews[iNew - 1] -= cellBytes; - cntNews[iNew - 1]++; - - cIdx--; - if (cIdx < 0) { - pPage = pOlds[--iPage]; - cIdx = TDB_PAGE_TOTAL_CELLS(pPage) - 1; + if (infoNews[iNew].size > infoNews[iNew - 1].size) { + break; } } } + + int k = 0; } SPage *pNews[5]; - { // Allocate the new pages + { + // Allocate the new pages } -#if 0 - // Step 1: find two sibling pages and get engough info about the old pages - ret = tdbBtreeBalanceStep1(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 2: Load all cells on the old page and the divider cells - ret = tdbBtreeBalanceStep2(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 3: Get the number of pages needed to hold all cells - ret = tdbBtreeBalanceStep3(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 4: Allocate enough new pages. Reuse old pages as much as possible - ret = tdbBtreeBalanceStep4(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 5: Insert new divider cells into pParent - ret = tdbBtreeBalanceStep5(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 6: Update the sibling pages - ret = tdbBtreeBalanceStep6(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } -#endif - { // TODO: Reset states } From d5afc7240a92968e07027c464d1db6caee590084 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:39:12 +0000 Subject: [PATCH 34/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index cbb389140f..e969f15775 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -745,13 +745,29 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } - - int k = 0; } - SPage *pNews[5]; - { - // Allocate the new pages + SPage *pNews[5] = {0}; + { // Allocate new pages, reuse the old page when possible + + SPgno pgno; + SBtreeInitPageArg iarg; + u8 flags; + + flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); + + for (int iNew = 0; iNew < nNews; iNew++) { + if (iNew < nOlds) { + pNews[iNew] = pOlds[iNew]; + } else { + iarg.pBt = pBt; + iarg.flags = flags; + ret = tdbPagerNewPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeZeroPage, &iarg); + if (ret < 0) { + ASSERT(0); + } + } + } } { From 6cd9d2703428e5647fdd64a0c40c94d3cd68724f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:45:59 +0000 Subject: [PATCH 35/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 150 +----------------------------- 1 file changed, 1 insertion(+), 149 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e969f15775..af0f331373 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -498,150 +498,6 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return 0; } -static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { - int nCells; - int i; - int idxStart; - int nChild; - int ret; - SPage *pParent; - SPgno pgno; - SCell *pCell; - SCellDecoder cd; - SBTree *pBt; - - pParent = pBlh->pParent; - nCells = TDB_PAGE_TOTAL_CELLS(pParent); - nChild = nCells + 1; - pBt = pBlh->pBt; - - // TODO: ASSERT below needs to be removed - ASSERT(pParent->nOverflow == 0); - ASSERT(pBlh->idx <= nCells); - - if (nChild < 3) { - idxStart = 0; - pBlh->nOld = nChild; - } else { - if (pBlh->idx == 0) { - idxStart = 0; - } else if (pBlh->idx == nCells) { - idxStart = pBlh->idx - 2; - } else { - idxStart = pBlh->idx - 1; - } - pBlh->nOld = 3; - } - - i = pBlh->nOld - 1; - - if (idxStart + i == nCells) { - // pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; - } else { - pCell = tdbPageGetCell(pParent, idxStart + i); - // TODO: no need to decode the payload part, and even the kLen, vLen part - // we only need the pgno part - ret = tdbBtreeDecodeCell(pParent, pCell, &cd); - if (ret < 0) { - ASSERT(0); - return -1; - } - pgno = cd.pgno; - } - for (;;) { - ret = tdbPagerFetchPage(pBt->pPager, pgno, &(pBlh->pOldPages[i]), tdbBtreeInitPage, pBt); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Loop over - if ((i--) == 0) break; - - { - // TODO - // ASSERT(0); - } - } - - return 0; -} - -static int tdbBtreeBalanceStep2(SBtreeBalanceHelper *pBlh) { -#if 0 - SPage *pPage; - int oidx; - int cidx; - int limit; - SCell *pCell; - - for (int i = 0; i < pBlh->nOld; i++) { - pPage = pBlh->pOldPages[i]; - oidx = 0; - cidx = 0; - - if (oidx < pPage->nOverflow) { - limit = pPage->aiOvfl[oidx]; - } else { - limit = pPage->pPageHdr->nCells; - } - - // Loop to copy each cell pointer out - for (;;) { - if (oidx >= pPage->nOverflow && cidx >= pPage->pPageHdr->nCells) break; - - if (cidx < limit) { - // Get local cells - pCell = tdbPageGetCell(pPage, cidx); - } else if (cidx == limit) { - // Get overflow cells - pCell = pPage->apOvfl[oidx++]; - - if (oidx < pPage->nOverflow) { - limit = pPage->aiOvfl[oidx]; - } else { - limit = pPage->pPageHdr->nCells; - } - } else { - ASSERT(0); - } - } - - { - // TODO: Copy divider cells here - } - } - - /* TODO */ - -#endif - return 0; -} - -static int tdbBtreeBalanceStep3(SBtreeBalanceHelper *pBlh) { - // Figure out number of pages needed after balance - for (int i = 0; i < pBlh->nOld; i++) { - /* TODO */ - } - - return 0; -} - -static int tdbBtreeBalanceStep4(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - -static int tdbBtreeBalanceStep5(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - -static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; @@ -771,11 +627,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } { - // TODO: Reset states - } - - { - // TODO: Clear resources + // Do the actual redistribute } return 0; From f3c10f886c589766bda0ade1f4c9a88b24ef3fd4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:48:32 +0000 Subject: [PATCH 36/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index af0f331373..71a9e70064 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -310,32 +310,6 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p return 0; } -static int tdbBtCursorMoveToRoot(SBtCursor *pCur) { - SBTree *pBt; - SPager *pPager; - SPage *pPage; - int ret; - - pBt = pCur->pBt; - pPager = pBt->pPager; - - // pPage = tdbPagerGet(pPager, pBt->root, true); - // if (pPage == NULL) { - // // TODO: handle error - // } - - // ret = tdbInitBtPage(pPage, &pBtPage); - // if (ret < 0) { - // // TODO - // return 0; - // } - - // pCur->pPage = pBtPage; - // pCur->iPage = 0; - - return 0; -} - static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { int mlen; int cret; From df12ae0d79e73725e54cf4c7ca44743dd21eda71 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 05:28:54 +0000 Subject: [PATCH 37/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 50 ++++++++++++++++++++++++++++-- source/libs/tdb/src/db/tdbPCache.c | 14 +-------- source/libs/tdb/src/inc/tdbUtil.h | 10 ++++++ source/libs/tdb/src/page/tdbPage.c | 7 +++++ 4 files changed, 66 insertions(+), 15 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 71a9e70064..196c4593c5 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -598,10 +598,56 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } + + // TODO: sort the page according to the page number } - { - // Do the actual redistribute + { // Do the actual cell distribution + + SPage *pTPage[2]; + int tPage, tIdx, iOld; + SCell *pCell; + int szCell; + SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; + + for (int i = 0; i < 2; i++) { + ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); + if (ret < 0) { + ASSERT(0); + } + } + + tPage = 0; + tIdx = 0; + iOld = 0; + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + + for (int iNew = 0; iNew < nNews; iNew++) { + // fill the iNew page + tdbBtreeZeroPage(pNews[iNew], &iarg); + + for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { + while (tIdx >= TDB_PAGE_TOTAL_CELLS(pTPage[tPage])) { + tPage = (tPage + 1) % 2; + tIdx = 0; + + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + } + + pCell = tdbPageGetCell(pTPage[tPage], tIdx); + szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + + tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell); + + tIdx++; + } + } + + for (int i = 0; i < 2; i++) { + tdbPageDestroy(pTPage[i], NULL, NULL); + } } return 0; diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 29f7395cbc..1fc0847b38 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -57,8 +57,6 @@ static void tdbPCachePinPage(SPage *pPage); static void tdbPCacheRemovePageFromHash(SPage *pPage); static void tdbPCacheAddPageToHash(SPage *pPage); static void tdbPCacheUnpinPage(SPage *pPage); -static void *tdbOsMalloc(void *arg, size_t size); -static void tdbOsFree(void *arg, void *ptr); int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { SPCache *pCache; @@ -257,7 +255,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { pCache->nFree = 0; pCache->pFree = NULL; for (int i = 0; i < pCache->cacheSize; i++) { - ret = tdbPageCreate(pCache->pageSize, &pPage, tdbOsMalloc, NULL); + ret = tdbPageCreate(pCache->pageSize, &pPage, NULL, NULL); if (ret < 0) { // TODO: handle error return -1; @@ -297,13 +295,3 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { } int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; } - -static void *tdbOsMalloc(void *arg, size_t size) { - void *ptr; - - ptr = malloc(size); - - return ptr; -} - -static void tdbOsFree(void *arg, void *ptr) { free(ptr); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 9576ee6d15..9358aae236 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -39,6 +39,16 @@ int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); int tdbPRead(int fd, void *pData, int count, i64 offset); +static inline void *tdbOsMalloc(void *arg, size_t size) { + void *ptr; + + ptr = malloc(size); + + return ptr; +} + +static inline void tdbOsFree(void *arg, void *ptr) { free(ptr); } + static inline int tdbPutVarInt(u8 *p, int v) { int n = 0; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 0dc7853940..16130ba13f 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -48,6 +48,9 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = NULL; size = pageSize + sizeof(*pPage); + if (xMalloc == NULL) { + xMalloc = tdbOsMalloc; + } ptr = (u8 *)((*xMalloc)(arg, size)); if (pPage == NULL) { @@ -73,6 +76,10 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { u8 *ptr; + if (!xFree) { + xFree = tdbOsFree; + } + ptr = pPage->pData; (*xFree)(arg, ptr); From 3abd4b2e672f5af1e1de3652427dbe840b71d9ce Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 06:08:28 +0000 Subject: [PATCH 38/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 16 +++++++++++++++- source/libs/tdb/src/inc/tdbPage.h | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 196c4593c5..a05202f456 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -602,7 +602,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // TODO: sort the page according to the page number } - { // Do the actual cell distribution + { // Do the real cell distribution SPage *pTPage[2]; int tPage, tIdx, iOld; @@ -650,6 +650,20 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + { // Insert records in parent page + int cIdx; + int szCell; + SCell pCell[128]; // TODO + SCellDecoder cd; + + for (int iNew = 0; iNew < nNews; iNew++) { + tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); + + tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); + tdbPageInsertCell(pParent, cIdx, pCell, szCell); + } + } + return 0; } diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 258eafcf63..61927fabf6 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -100,6 +100,7 @@ struct SPage { // APIs #define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) #define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) +#define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno) #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); From 0b752392aa482ab45cd38d6d78a4112c84782042 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 06:24:16 +0000 Subject: [PATCH 39/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a05202f456..da11eb099a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -495,17 +495,17 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } nOlds = 3; } - for (int i = 0; i < nOlds; i++, sIdx++) { - ASSERT(sIdx <= nCells); + for (int i = 0; i < nOlds; i++) { + ASSERT(sIdx + i <= nCells); SPgno pgno; - if (sIdx == nCells) { + if (sIdx + i == nCells) { ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pParent))); pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { SCell *pCell; - pCell = tdbPageGetCell(pParent, sIdx); + pCell = tdbPageGetCell(pParent, sIdx + i); pgno = *(SPgno *)pCell; } @@ -515,6 +515,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return -1; } } + // drop the cells + for (int i = 0; i < nOlds; i++) { + nCells = TDB_PAGE_TOTAL_CELLS(pParent); + if (sIdx < nCells) { + tdbPageDropCell(pParent, sIdx); + } else { + ((SIntHdr *)pParent->pData)->pgno = 0; + } + } } int nNews = 0; @@ -657,6 +666,16 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SCellDecoder cd; for (int iNew = 0; iNew < nNews; iNew++) { + if (iNew == nNews - 1) { + // The last new page + + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + break; + } + } + tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); @@ -664,6 +683,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + int k = 0; + return 0; } From 2d1e356cc214aa99aed731b17352635f95728f41 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 06:56:29 +0000 Subject: [PATCH 40/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index da11eb099a..e7abbc7161 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -477,10 +477,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nOlds; SPage *pOlds[3]; + int sIdx; { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); - int sIdx; if (nCells <= 2) { sIdx = 0; nOlds = nCells + 1; @@ -515,7 +515,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return -1; } } - // drop the cells + // drop the cells on parent page for (int i = 0; i < nOlds; i++) { nCells = TDB_PAGE_TOTAL_CELLS(pParent); if (sIdx < nCells) { @@ -660,7 +660,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } { // Insert records in parent page - int cIdx; + int cIdx = sIdx; int szCell; SCell pCell[128]; // TODO SCellDecoder cd; @@ -679,12 +679,11 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); + // TODO: the cell here may be used by pParent as an overflow cell tdbPageInsertCell(pParent, cIdx, pCell, szCell); } } - int k = 0; - return 0; } From 238550b0f4d0ceeb987ad41eb7992afba994dacd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 07:50:34 +0000 Subject: [PATCH 41/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 36 ++++++++++++++++++++++++++---- source/libs/tdb/src/inc/tdbPage.h | 1 + source/libs/tdb/src/page/tdbPage.c | 1 - 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e7abbc7161..d1188ee5b3 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -477,10 +477,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nOlds; SPage *pOlds[3]; + SCell *pDivCell[2] = {0}; + int szDivCell[2]; int sIdx; + u8 childLeaf; { // Find 3 child pages at most to do balance - int nCells = TDB_PAGE_TOTAL_CELLS(pParent); + int nCells = TDB_PAGE_TOTAL_CELLS(pParent); + SCell *pCell; + if (nCells <= 2) { sIdx = 0; nOlds = nCells + 1; @@ -503,8 +508,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pParent))); pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { - SCell *pCell; - pCell = tdbPageGetCell(pParent, sIdx + i); pgno = *(SPgno *)pCell; } @@ -515,6 +518,17 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return -1; } } + // copy the parent key out if child pages are not leaf page + childLeaf = TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); + if (childLeaf) { + for (int i = 0; i < nOlds - 1; i++) { + pCell = tdbPageGetCell(pParent, sIdx + i); + + szDivCell[i] = tdbBtreeCellSize(pParent, pCell); + pDivCell[i] = malloc(szDivCell[i]); + memcpy(pDivCell, pCell, szDivCell[i]); + } + } // drop the cells on parent page for (int i = 0; i < nOlds; i++) { nCells = TDB_PAGE_TOTAL_CELLS(pParent); @@ -541,14 +555,28 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SPage *pPage = pOlds[oPage]; SCell *pCell; int cellBytes; + int oIdx; - for (int oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { + for (oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { pCell = tdbPageGetCell(pPage, oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; + // for a child leaf case, this cell is used as the new divider cell + if (childLeaf) continue; + } + infoNews[nNews].cnt++; + infoNews[nNews].size += cellBytes; + infoNews[nNews].oPage = oPage; + infoNews[nNews].oIdx = oIdx; + } + + // For child leaf pages + if (childLeaf && oPage < nOlds - 1) { + if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { + nNews++; } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 61927fabf6..0ba7cf236b 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -102,6 +102,7 @@ struct SPage { #define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) #define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno) #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 16130ba13f..14960de45d 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -18,7 +18,6 @@ extern SPageMethods pageMethods; extern SPageMethods pageLargeMethods; -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) #define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) #define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) #define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) From aee8b603242f0db0facc04cb7b3793815395c521 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 08:13:32 +0000 Subject: [PATCH 42/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index d1188ee5b3..6daecb7908 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -564,7 +564,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for a child leaf case, this cell is used as the new divider cell + // for a child leaf case, this cell is used as the new divider cell to parent if (childLeaf) continue; } infoNews[nNews].cnt++; @@ -591,10 +591,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { for (int iNew = nNews - 1; iNew > 0; iNew--) { SCell *pCell; SPage *pPage; + int nCells; int cellBytes; + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + for (;;) { - pPage = pOlds[infoNews[iNew - 1].oPage]; pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); @@ -604,7 +607,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { infoNews[iNew - 1].size -= cellBytes; if ((infoNews[iNew - 1].oIdx--) == 0) { infoNews[iNew - 1].oPage--; - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pOlds[infoNews[iNew - 1].oPage]); + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); } if (infoNews[iNew].size > infoNews[iNew - 1].size) { From d7b54201bcab335b39a881d32f06af88bb987d72 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 08:26:43 +0000 Subject: [PATCH 43/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 50 +++++++++++++++++-------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 6daecb7908..cda958c578 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -520,7 +520,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } // copy the parent key out if child pages are not leaf page childLeaf = TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); - if (childLeaf) { + if (!childLeaf) { for (int i = 0; i < nOlds - 1; i++) { pCell = tdbPageGetCell(pParent, sIdx + i); @@ -564,8 +564,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for a child leaf case, this cell is used as the new divider cell to parent - if (childLeaf) continue; + // for a internal leaf case, this cell is used as the new divider cell to parent + if (!childLeaf) continue; } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; @@ -573,8 +573,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { infoNews[nNews].oIdx = oIdx; } - // For child leaf pages - if (childLeaf && oPage < nOlds - 1) { + // For internal pages + if (!childLeaf && oPage < nOlds - 1) { if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { nNews++; } @@ -594,27 +594,31 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nCells; int cellBytes; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); + if (childLeaf) { // child leaf + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); - for (;;) { - pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); - cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); + for (;;) { + pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); + cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - infoNews[iNew].cnt++; - infoNews[iNew].size += cellBytes; - infoNews[iNew - 1].cnt--; - infoNews[iNew - 1].size -= cellBytes; - if ((infoNews[iNew - 1].oIdx--) == 0) { - infoNews[iNew - 1].oPage--; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); - } - - if (infoNews[iNew].size > infoNews[iNew - 1].size) { - break; + infoNews[iNew].cnt++; + infoNews[iNew].size += cellBytes; + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= cellBytes; + if ((infoNews[iNew - 1].oIdx--) == 0) { + infoNews[iNew - 1].oPage--; + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); + } + + if (infoNews[iNew].size > infoNews[iNew - 1].size) { + break; + } } + } else { // internal leaf + // TODO } } } From 3b49a863ea18e13f224088a212dc3a522089f770 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 10:43:51 +0000 Subject: [PATCH 44/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 194 ++++++++++++++++++++++++------ 1 file changed, 154 insertions(+), 40 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index cda958c578..4c6d85bc0a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -564,7 +564,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for a internal leaf case, this cell is used as the new divider cell to parent + // for an internal leaf case, this cell is used as the new divider cell to parent if (!childLeaf) continue; } infoNews[nNews].cnt++; @@ -594,10 +594,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nCells; int cellBytes; - if (childLeaf) { // child leaf - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + if (childLeaf) { // child leaf for (;;) { pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); @@ -618,7 +618,55 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } else { // internal leaf - // TODO + SCell *pCellRight; + SCell *pCellLeft; + int szCellRight, szCellLeft; + + // get pCellLeft, szCellLeft + if (infoNews[iNew - 1].oIdx == nCells) { + ASSERT(infoNews[iNew - 1].oPage < nOlds - 1); + pCellLeft = pDivCell[infoNews[iNew - 1].oPage]; + szCellLeft = szDivCell[infoNews[iNew - 1].oPage]; + } else { + pCellLeft = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); + szCellLeft = tdbBtreeCellSize(pPage, pCellLeft); + } + + // get pCellRight, szCellRight + if (infoNews[iNew - 1].oIdx + 1 < nCells) { + pCellRight = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx + 1); + szCellRight = tdbBtreeCellSize(pPage, pCellRight); + } else { + if (infoNews[iNew - 1].oPage < nOlds - 1) { + pCellRight = pDivCell[infoNews[iNew - 1].oPage]; + szCellRight = szDivCell[infoNews[iNew - 1].oPage]; + } else { + // TODO: what if the next old page is empty? + pCellRight = tdbPageGetCell(pOlds[infoNews[iNew - 1].oPage + 1], 0); + szCellRight = tdbBtreeCellSize(pPage, pCellRight); + } + } + + if (infoNews[iNew - 1].size - szCellLeft - TDB_PAGE_OFFSET_SIZE(pPage) <= + infoNews[iNew].size + szCellRight + TDB_PAGE_OFFSET_SIZE(pPage)) { + break; + } + + // TODO: ASSERT(infoNews[iNew].size < PAGE_MAX_CAPACITY); + infoNews[iNew].cnt++; + infoNews[iNew].size += szCellRight; + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= szCellLeft; + if ((infoNews[iNew - 1].oIdx--) == 0) { + infoNews[iNew - 1].oPage--; + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + if (infoNews[iNew - 1].oPage < nOlds - 1) { + infoNews[iNew - 1].oIdx = nCells; + } else { + infoNews[iNew - 1].oIdx = nCells - 1; + } + } } } } @@ -654,6 +702,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int tPage, tIdx, iOld; SCell *pCell; int szCell; + int nCells; + SCellDecoder cd; SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; for (int i = 0; i < 2; i++) { @@ -666,28 +716,117 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tPage = 0; tIdx = 0; iOld = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + if (!childLeaf) { + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + } for (int iNew = 0; iNew < nNews; iNew++) { // fill the iNew page + // TODO: copy current page to tmp space tdbBtreeZeroPage(pNews[iNew], &iarg); for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { - while (tIdx >= TDB_PAGE_TOTAL_CELLS(pTPage[tPage])) { - tPage = (tPage + 1) % 2; - tIdx = 0; + for (;;) { // loop to find the next available cell + if (tIdx < nCells) { + pCell = tdbPageGetCell(pTPage[tPage], tIdx); + szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + tIdx++; + break; + } else { + if (!childLeaf) { + if (iOld < nOlds - 1) { + pCell = pDivCell[iOld]; + szCell = szDivCell[iOld]; + ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + break; + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + } + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + } + } } - pCell = tdbPageGetCell(pTPage[tPage], tIdx); - szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell); + } - tIdx++; + // fill right-most child pgno if internal page + if (!childLeaf) { + if (tIdx < nCells) { + pCell = tdbPageGetCell(pTPage[tPage], tIdx); + szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + tIdx++; + break; + } else { + if (!childLeaf) { + if (iOld < nOlds - 1) { + pCell = pDivCell[iOld]; + szCell = szDivCell[iOld]; + ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + break; + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + } + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + } + } + + ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + } + + // insert divider cell into the parent page + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + } else { + tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); + tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, + &szCell); + // TODO: the cell here may be used by pParent as an overflow cell + tdbPageInsertCell(pParent, sIdx++, pCell, szCell); } } @@ -696,31 +835,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - { // Insert records in parent page - int cIdx = sIdx; - int szCell; - SCell pCell[128]; // TODO - SCellDecoder cd; - - for (int iNew = 0; iNew < nNews; iNew++) { - if (iNew == nNews - 1) { - // The last new page - - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - if (pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - break; - } - } - - tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); - - tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); - // TODO: the cell here may be used by pParent as an overflow cell - tdbPageInsertCell(pParent, cIdx, pCell, szCell); - } - } - return 0; } From 3a173f466069f6cf59f2e9e281eaf9298886766d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 11:47:25 +0000 Subject: [PATCH 45/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4c6d85bc0a..67b11f8ef4 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -204,7 +204,19 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { - // TODO + int ret; + + pCur->pgStack[pCur->iPage] = pCur->pPage; + pCur->idxStack[pCur->iPage] = pCur->idx; + pCur->iPage++; + pCur->pPage = NULL; + pCur->idx = -1; + + ret = tdbPagerFetchPage(pCur->pBt->pPager, pgno, &pCur->pPage, tdbBtreeInitPage, pCur->pBt); + if (ret < 0) { + ASSERT(0); + } + return 0; } @@ -287,14 +299,12 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p pCur->idx = midx; tdbBtCursorMoveToChild(pCur, cd.pgno); } else { + pCur->idx = midx + 1; if (midx == nCells - 1) { /* Move to right-most child */ - pCur->idx = midx + 1; - // tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); + tdbBtCursorMoveToChild(pCur, ((SIntHdr *)pCur->pPage->pData)->pgno); } else { - // TODO: reset cd as uninitialized - pCur->idx = midx + 1; - pCell = tdbPageGetCell(pPage, midx + 1); + pCell = tdbPageGetCell(pPage, pCur->idx); tdbBtreeDecodeCell(pPage, pCell, &cd); tdbBtCursorMoveToChild(pCur, cd.pgno); } From 44daa77d8db957e3d9e7ca150bfce61f26f8764a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 05:32:39 +0000 Subject: [PATCH 46/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 33 ++++++++++++++++++------------ source/libs/tdb/src/inc/tdbPage.h | 2 +- source/libs/tdb/src/page/tdbPage.c | 28 ++++++++++++------------- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 67b11f8ef4..a1f879c729 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -187,7 +187,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } // Insert the cell to the index - ret = tdbPageInsertCell(pCur->pPage, idx, pCell, szCell); + ret = tdbPageInsertCell(pCur->pPage, idx, pCell, szCell, 0); if (ret < 0) { return -1; } @@ -490,7 +490,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SCell *pDivCell[2] = {0}; int szDivCell[2]; int sIdx; - u8 childLeaf; + u8 childNotLeaf; { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); @@ -529,14 +529,21 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } // copy the parent key out if child pages are not leaf page - childLeaf = TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); - if (!childLeaf) { + childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); + if (childNotLeaf) { for (int i = 0; i < nOlds - 1; i++) { pCell = tdbPageGetCell(pParent, sIdx + i); szDivCell[i] = tdbBtreeCellSize(pParent, pCell); pDivCell[i] = malloc(szDivCell[i]); memcpy(pDivCell, pCell, szDivCell[i]); + + ((SPgno *)pDivCell)[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; + ((SIntHdr *)pOlds[i]->pData)->pgno = 0; + + // here we insert the cell as an overflow cell to avoid + // the slow defragment process + tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } } // drop the cells on parent page @@ -575,7 +582,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // page is full, use a new page nNews++; // for an internal leaf case, this cell is used as the new divider cell to parent - if (!childLeaf) continue; + if (childNotLeaf) continue; } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; @@ -584,7 +591,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } // For internal pages - if (!childLeaf && oPage < nOlds - 1) { + if (childNotLeaf && oPage < nOlds - 1) { if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { nNews++; } @@ -607,7 +614,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { pPage = pOlds[infoNews[iNew - 1].oPage]; nCells = TDB_PAGE_TOTAL_CELLS(pPage); - if (childLeaf) { // child leaf + if (!childNotLeaf) { // child leaf for (;;) { pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); @@ -729,7 +736,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); tdbBtreeZeroPage(pTPage[tPage], &iarg); tdbPageCopy(pOlds[iOld], pTPage[tPage]); - if (!childLeaf) { + if (childNotLeaf) { ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; } @@ -746,7 +753,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tIdx++; break; } else { - if (!childLeaf) { + if (childNotLeaf) { if (iOld < nOlds - 1) { pCell = pDivCell[iOld]; szCell = szDivCell[iOld]; @@ -780,18 +787,18 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell); + tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); } // fill right-most child pgno if internal page - if (!childLeaf) { + if (childNotLeaf) { if (tIdx < nCells) { pCell = tdbPageGetCell(pTPage[tPage], tIdx); szCell = tdbBtreeCellSize(pTPage[tPage], pCell); tIdx++; break; } else { - if (!childLeaf) { + if (!childNotLeaf) { if (iOld < nOlds - 1) { pCell = pDivCell[iOld]; szCell = szDivCell[iOld]; @@ -836,7 +843,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); // TODO: the cell here may be used by pParent as an overflow cell - tdbPageInsertCell(pParent, sIdx++, pCell, szCell); + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); } } diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 0ba7cf236b..a6f9fbf615 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -108,7 +108,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); int tdbPageDropCell(SPage *pPage, int idx); void tdbPageCopy(SPage *pFromPage, SPage *pToPage); diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 14960de45d..ee1a021a1b 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -114,7 +114,7 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); } -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl) { int nFree; int nCells; int iOvfl; @@ -135,7 +135,19 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { lidx = idx - iOvfl; - if (nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + if (asOvfl || nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + // TODO: make it extensible + // add the cell as an overflow cell + for (int i = pPage->nOverflow; i > iOvfl; i--) { + pPage->apOvfl[i] = pPage->apOvfl[i - 1]; + pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; + } + + pPage->apOvfl[iOvfl] = pCell; + pPage->aiOvfl[iOvfl] = idx; + pPage->nOverflow++; + iOvfl++; + } else { // page must has enough space to hold the cell locally tdbPageAllocate(pPage, szCell, &pNewCell); @@ -149,18 +161,6 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { TDB_PAGE_NCELLS_SET(pPage, nCells + 1); ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); - } else { - // TODO: make it extensible - // add the cell as an overflow cell - for (int i = pPage->nOverflow; i > iOvfl; i--) { - pPage->apOvfl[i] = pPage->apOvfl[i - 1]; - pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; - } - - pPage->apOvfl[iOvfl] = pCell; - pPage->aiOvfl[iOvfl] = idx; - pPage->nOverflow++; - iOvfl++; } for (; iOvfl < pPage->nOverflow; iOvfl++) { From 5c9e63e5018f42af709ac3a75fa831fb7bacff86 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 06:56:53 +0000 Subject: [PATCH 47/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 327 +++++++++++++++++------------- source/libs/tdb/src/inc/tdbInt.h | 4 +- 2 files changed, 183 insertions(+), 148 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a1f879c729..459154e9fb 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -451,13 +451,15 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPage *pChild; SPgno pgnoChild; int ret; + u8 flags; SIntHdr *pIntHdr; SBtreeInitPageArg zArg; pPager = pRoot->pPager; + flags = TDB_BTREE_PAGE_GET_FLAGS(pRoot); // Allocate a new child page - zArg.flags = TDB_BTREE_LEAF; + zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT); zArg.pBt = pBt; ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg); if (ret < 0) { @@ -559,10 +561,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nNews = 0; struct { - int cnt; - int size; - int oPage; - int oIdx; + int cnt; + int size; + SPage *pPage; + int oIdx; } infoNews[5] = {0}; { // Get how many new pages are needed and the new distribution @@ -581,31 +583,63 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for an internal leaf case, this cell is used as the new divider cell to parent - if (childNotLeaf) continue; - } - infoNews[nNews].cnt++; - infoNews[nNews].size += cellBytes; - infoNews[nNews].oPage = oPage; - infoNews[nNews].oIdx = oIdx; - } - // For internal pages - if (childNotLeaf && oPage < nOlds - 1) { - if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { - nNews++; + ASSERT(infoNews[nNews].size + cellBytes <= TDB_PAGE_USABLE_SIZE(pPage)); + + if (childNotLeaf) { + // for non-child page, this cell is used as the right-most child, + // the divider cell to parent as well + continue; + } } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; - infoNews[nNews].oPage = oPage; + infoNews[nNews].pPage = pPage; infoNews[nNews].oIdx = oIdx; } } - nNews = nNews + 1; + nNews++; // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { + SCell *pLCell, pRCell; + int szLCell, szRCell; + + for (;;) { + if (childNotLeaf) { + pLCell = pRCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); + szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pLCell); + } else { + // For internal page, find the left cell + // TODO: fill pLCell, pRCell, szLCell, szRCell + } + + ASSERT(infoNews[iNew - 1].cnt > 0); + + if (infoNews[iNew].size + szRCell >= infoNews[iNew - 1].size - szRCell) { + break; + } + + // move cell divider left + infoNews[iNew].cnt++; + infoNews[iNew].size += szRCell; + { + // TODO + // infoNews[iNew].oPage = ; + // infoNews[iNew].oIdx = ; + } + + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= szLCell; + { + // TODO + // infoNews[iNew-1].oPage = ; + // infoNews[iNew-1].oIdx = ; + } + } + +#if 0 SCell *pCell; SPage *pPage; int nCells; @@ -685,6 +719,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } +#endif } } @@ -713,146 +748,146 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // TODO: sort the page according to the page number } - { // Do the real cell distribution + // { // Do the real cell distribution - SPage *pTPage[2]; - int tPage, tIdx, iOld; - SCell *pCell; - int szCell; - int nCells; - SCellDecoder cd; - SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; + // SPage *pTPage[2]; + // int tPage, tIdx, iOld; + // SCell *pCell; + // int szCell; + // int nCells; + // SCellDecoder cd; + // SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; - for (int i = 0; i < 2; i++) { - ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); - if (ret < 0) { - ASSERT(0); - } - } + // for (int i = 0; i < 2; i++) { + // ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); + // if (ret < 0) { + // ASSERT(0); + // } + // } - tPage = 0; - tIdx = 0; - iOld = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - if (childNotLeaf) { - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - } + // tPage = 0; + // tIdx = 0; + // iOld = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // if (childNotLeaf) { + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // } - for (int iNew = 0; iNew < nNews; iNew++) { - // fill the iNew page - // TODO: copy current page to tmp space - tdbBtreeZeroPage(pNews[iNew], &iarg); + // for (int iNew = 0; iNew < nNews; iNew++) { + // // fill the iNew page + // // TODO: copy current page to tmp space + // tdbBtreeZeroPage(pNews[iNew], &iarg); - for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { - for (;;) { // loop to find the next available cell - if (tIdx < nCells) { - pCell = tdbPageGetCell(pTPage[tPage], tIdx); - szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - tIdx++; - break; - } else { - if (childNotLeaf) { - if (iOld < nOlds - 1) { - pCell = pDivCell[iOld]; - szCell = szDivCell[iOld]; - ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { + // for (;;) { // loop to find the next available cell + // if (tIdx < nCells) { + // pCell = tdbPageGetCell(pTPage[tPage], tIdx); + // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + // tIdx++; + // break; + // } else { + // if (childNotLeaf) { + // if (iOld < nOlds - 1) { + // pCell = pDivCell[iOld]; + // szCell = szDivCell[iOld]; + // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - break; - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - } - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - } - } - } + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // break; + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // } + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // } + // } + // } - tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); - } + // tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); + // } - // fill right-most child pgno if internal page - if (childNotLeaf) { - if (tIdx < nCells) { - pCell = tdbPageGetCell(pTPage[tPage], tIdx); - szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - tIdx++; - break; - } else { - if (!childNotLeaf) { - if (iOld < nOlds - 1) { - pCell = pDivCell[iOld]; - szCell = szDivCell[iOld]; - ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // // fill right-most child pgno if internal page + // if (childNotLeaf) { + // if (tIdx < nCells) { + // pCell = tdbPageGetCell(pTPage[tPage], tIdx); + // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + // tIdx++; + // break; + // } else { + // if (!childNotLeaf) { + // if (iOld < nOlds - 1) { + // pCell = pDivCell[iOld]; + // szCell = szDivCell[iOld]; + // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - break; - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - } - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - } - } + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // break; + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // } + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // } + // } - ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; - } + // ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + // } - // insert divider cell into the parent page - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - if (iNew == nNews - 1 && pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - } else { - tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); - tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, - &szCell); - // TODO: the cell here may be used by pParent as an overflow cell - tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); - } - } + // // insert divider cell into the parent page + // SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + // if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + // pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + // } else { + // tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); + // tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, + // &szCell); + // // TODO: the cell here may be used by pParent as an overflow cell + // tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + // } + // } - for (int i = 0; i < 2; i++) { - tdbPageDestroy(pTPage[i], NULL, NULL); - } - } + // for (int i = 0; i < 2; i++) { + // tdbPageDestroy(pTPage[i], NULL, NULL); + // } +} - return 0; +return 0; } static int tdbBtreeBalance(SBtCursor *pCur) { diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 5902a6a716..3ebba0a4b5 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -141,8 +141,8 @@ typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, i #define TDB_FLAG_IS(flags, flag) ((flags) == (flag)) #define TDB_FLAG_HAS(flags, flag) (((flags) & (flag)) != 0) #define TDB_FLAG_NO(flags, flag) ((flags) & (flag) == 0) -#define TDB_FLAG_ADD(flags, flag) ((flags) |= (flag)) -#define TDB_FLAG_REMOVE(flags, flag) ((flags) &= (~(flag))) +#define TDB_FLAG_ADD(flags, flag) ((flags) | (flag)) +#define TDB_FLAG_REMOVE(flags, flag) ((flags) & (~(flag))) typedef struct SPager SPager; typedef struct SPCache SPCache; From 9b8a8f12871a85b12d351e4e707c593e3254d770 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:17:26 +0000 Subject: [PATCH 48/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 119 +++++++----------------------- 1 file changed, 25 insertions(+), 94 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 459154e9fb..f649a8e9f6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -603,16 +603,30 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { - SCell *pLCell, pRCell; + SCell *pCell; int szLCell, szRCell; for (;;) { + pCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); + if (childNotLeaf) { - pLCell = pRCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); - szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pLCell); + szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); } else { - // For internal page, find the left cell - // TODO: fill pLCell, pRCell, szLCell, szRCell + szLCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); + + SPage *pPage = infoNews[iNew - 1].pPage; + int oIdx = infoNews[iNew - 1].oIdx + 1; + for (;;) { + if (oIdx < TDB_PAGE_TOTAL_CELLS(pPage)) { + break; + } + + pPage++; + oIdx = 0; + } + + pCell = tdbPageGetCell(pPage, oIdx); + szRCell = tdbBtreeCellSize(pPage, pCell); } ASSERT(infoNews[iNew - 1].cnt > 0); @@ -621,105 +635,22 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { break; } - // move cell divider left - infoNews[iNew].cnt++; - infoNews[iNew].size += szRCell; - { - // TODO - // infoNews[iNew].oPage = ; - // infoNews[iNew].oIdx = ; - } - + // Move a cell right forward infoNews[iNew - 1].cnt--; infoNews[iNew - 1].size -= szLCell; - { - // TODO - // infoNews[iNew-1].oPage = ; - // infoNews[iNew-1].oIdx = ; - } - } - -#if 0 - SCell *pCell; - SPage *pPage; - int nCells; - int cellBytes; - - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - - if (!childNotLeaf) { // child leaf + infoNews[iNew - 1].oIdx--; for (;;) { - pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); - cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - - infoNews[iNew].cnt++; - infoNews[iNew].size += cellBytes; - infoNews[iNew - 1].cnt--; - infoNews[iNew - 1].size -= cellBytes; - if ((infoNews[iNew - 1].oIdx--) == 0) { - infoNews[iNew - 1].oPage--; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); - } - - if (infoNews[iNew].size > infoNews[iNew - 1].size) { + if (infoNews[iNew - 1].oIdx >= 0) { break; } - } - } else { // internal leaf - SCell *pCellRight; - SCell *pCellLeft; - int szCellRight, szCellLeft; - // get pCellLeft, szCellLeft - if (infoNews[iNew - 1].oIdx == nCells) { - ASSERT(infoNews[iNew - 1].oPage < nOlds - 1); - pCellLeft = pDivCell[infoNews[iNew - 1].oPage]; - szCellLeft = szDivCell[infoNews[iNew - 1].oPage]; - } else { - pCellLeft = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); - szCellLeft = tdbBtreeCellSize(pPage, pCellLeft); + infoNews[iNew - 1].pPage--; + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(infoNews[iNew - 1].pPage) - 1; } - // get pCellRight, szCellRight - if (infoNews[iNew - 1].oIdx + 1 < nCells) { - pCellRight = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx + 1); - szCellRight = tdbBtreeCellSize(pPage, pCellRight); - } else { - if (infoNews[iNew - 1].oPage < nOlds - 1) { - pCellRight = pDivCell[infoNews[iNew - 1].oPage]; - szCellRight = szDivCell[infoNews[iNew - 1].oPage]; - } else { - // TODO: what if the next old page is empty? - pCellRight = tdbPageGetCell(pOlds[infoNews[iNew - 1].oPage + 1], 0); - szCellRight = tdbBtreeCellSize(pPage, pCellRight); - } - } - - if (infoNews[iNew - 1].size - szCellLeft - TDB_PAGE_OFFSET_SIZE(pPage) <= - infoNews[iNew].size + szCellRight + TDB_PAGE_OFFSET_SIZE(pPage)) { - break; - } - - // TODO: ASSERT(infoNews[iNew].size < PAGE_MAX_CAPACITY); infoNews[iNew].cnt++; - infoNews[iNew].size += szCellRight; - infoNews[iNew - 1].cnt--; - infoNews[iNew - 1].size -= szCellLeft; - if ((infoNews[iNew - 1].oIdx--) == 0) { - infoNews[iNew - 1].oPage--; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - if (infoNews[iNew - 1].oPage < nOlds - 1) { - infoNews[iNew - 1].oIdx = nCells; - } else { - infoNews[iNew - 1].oIdx = nCells - 1; - } - } + infoNews[iNew].size += szRCell; } -#endif } } From 15fa64ebaab2f3f8237105b83d060cc57c5c3089 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:39:57 +0000 Subject: [PATCH 49/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 177 ++++++++---------------------- 1 file changed, 48 insertions(+), 129 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f649a8e9f6..8912a765fd 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -679,146 +679,65 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // TODO: sort the page according to the page number } - // { // Do the real cell distribution + { // Do the real cell distribution + SPage *pOldsCopy[3]; + SCell *pCell; + int szCell; + SBtreeInitPageArg iarg; + int iNew, nNewCells; - // SPage *pTPage[2]; - // int tPage, tIdx, iOld; - // SCell *pCell; - // int szCell; - // int nCells; - // SCellDecoder cd; - // SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; + iarg.pBt = pBt; + iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); + for (int i = 0; i < nOlds; i++) { + tdbPageCreate(pOlds[0]->pageSize, pOldsCopy + i, NULL, NULL); + tdbBtreeZeroPage(pOldsCopy[i], &iarg); + tdbPageCopy(pOlds[i], pOldsCopy[i]); + } + iNew = 0; + nNewCells = 0; - // for (int i = 0; i < 2; i++) { - // ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); - // if (ret < 0) { - // ASSERT(0); - // } - // } + for (int iOld = 0; iOld < nOlds; iOld++) { + SPage *pPage; - // tPage = 0; - // tIdx = 0; - // iOld = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // if (childNotLeaf) { - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // } + pPage = pOldsCopy[iOld]; - // for (int iNew = 0; iNew < nNews; iNew++) { - // // fill the iNew page - // // TODO: copy current page to tmp space - // tdbBtreeZeroPage(pNews[iNew], &iarg); + for (int oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { + pCell = tdbPageGetCell(pPage, oIdx); + szCell = tdbBtreeCellSize(pPage, pCell); - // for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { - // for (;;) { // loop to find the next available cell - // if (tIdx < nCells) { - // pCell = tdbPageGetCell(pTPage[tPage], tIdx); - // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - // tIdx++; - // break; - // } else { - // if (childNotLeaf) { - // if (iOld < nOlds - 1) { - // pCell = pDivCell[iOld]; - // szCell = szDivCell[iOld]; - // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + ASSERT(nNewCells <= infoNews[iNew].cnt); - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // break; - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // } - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // } - // } - // } + if (nNewCells < infoNews[iNew].cnt) { + tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); + nNewCells++; + } else { + if (childNotLeaf) { + // set current new page right-most child - // tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); - // } + // move to next new page + } else { + // move to next new page - // // fill right-most child pgno if internal page - // if (childNotLeaf) { - // if (tIdx < nCells) { - // pCell = tdbPageGetCell(pTPage[tPage], tIdx); - // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - // tIdx++; - // break; - // } else { - // if (!childNotLeaf) { - // if (iOld < nOlds - 1) { - // pCell = pDivCell[iOld]; - // szCell = szDivCell[iOld]; - // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // insert the cell to the new page + } + } - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // break; - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // } - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // } - // } + if (nNewCells == infoNews[iNew].cnt) { + if (childNotLeaf) { + // TODO + } else { + // TODO + } + } + } + } - // ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; - // } + for (int i = 0; i < nOlds; i++) { + tdbPageDestroy(pOldsCopy[i], NULL, NULL); + } + } - // // insert divider cell into the parent page - // SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - // if (iNew == nNews - 1 && pIntHdr->pgno == 0) { - // pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - // } else { - // tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); - // tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, - // &szCell); - // // TODO: the cell here may be used by pParent as an overflow cell - // tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); - // } - // } - - // for (int i = 0; i < 2; i++) { - // tdbPageDestroy(pTPage[i], NULL, NULL); - // } -} - -return 0; + return 0; } static int tdbBtreeBalance(SBtCursor *pCur) { From f8fc05c76e035de6c0fe2ffa0dbec1eec7b10987 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:48:08 +0000 Subject: [PATCH 50/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8912a765fd..98b9455d13 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -712,13 +712,25 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nNewCells++; } else { if (childNotLeaf) { + // Insert to parent + // set current new page right-most child + ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; // move to next new page + iNew++; + nNewCells = 0; } else { + // TODO: Insert to parent max key and current page number or the right most child + // move to next new page + iNew++; + nNewCells = 0; // insert the cell to the new page + ASSERT(nNewCells < infoNews[iNew].cnt); + tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); + nNewCells++; } } From 7b89447239f1971547d44adfede31d113066784f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:57:15 +0000 Subject: [PATCH 51/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 98b9455d13..c6270d6047 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -733,14 +733,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nNewCells++; } } - - if (nNewCells == infoNews[iNew].cnt) { - if (childNotLeaf) { - // TODO - } else { - // TODO - } - } } } From 09833d2244a0da316a34d3b342513d55fc2a8753 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:21:09 +0000 Subject: [PATCH 52/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 49 ++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c6270d6047..ca50631e90 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -710,28 +710,43 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); nNewCells++; - } else { - if (childNotLeaf) { - // Insert to parent - // set current new page right-most child - ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + // insert parent page + if (!childNotLeaf && nNewCells == infoNews[iNew].cnt) { + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + + if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + } else { + // TODO: + ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + // TODO: pCell here may be inserted as an overflow cell, handle it + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + } // move to next new page iNew++; - nNewCells = 0; - } else { - // TODO: Insert to parent max key and current page number or the right most child - - // move to next new page - iNew++; - nNewCells = 0; - - // insert the cell to the new page - ASSERT(nNewCells < infoNews[iNew].cnt); - tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); - nNewCells++; + nNews = 0; } + } else { + ASSERT(childNotLeaf); + + // set current new page right-most child + ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + + // insert to parent as divider cell + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + + if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + } else { + ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + } + + // move to next new page + iNew++; + nNewCells = 0; } } } From d5a1c7a31f9ea48c7b41a1b7f23f3769b1683123 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:25:57 +0000 Subject: [PATCH 53/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index ca50631e90..a6747e8903 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -685,6 +685,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szCell; SBtreeInitPageArg iarg; int iNew, nNewCells; + SCellDecoder cd; iarg.pBt = pBt; iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); @@ -718,10 +719,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (iNew == nNews - 1 && pIntHdr->pgno == 0) { pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); } else { - // TODO: - ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + tdbBtreeDecodeCell(pPage, pCell, &cd); + // TODO: pCell here may be inserted as an overflow cell, handle it - tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + SCell *pNewCell = malloc(cd.kLen + 9); + int szNewCell; + SPgno pgno; + pgno = TDB_PAGE_PGNO(pNews[iNew]); + tdbBtreeEncodeCell(pPage, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); + tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); } // move to next new page From 72aed8997dfcf56f4a1942b4b58ee651b201bf1a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:39:22 +0000 Subject: [PATCH 54/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a6747e8903..68be6ec8cd 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -493,6 +493,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szDivCell[2]; int sIdx; u8 childNotLeaf; + SPgno rPgno; { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); @@ -547,6 +548,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // the slow defragment process tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } + rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno; } // drop the cells on parent page for (int i = 0; i < nOlds; i++) { @@ -741,14 +743,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; // insert to parent as divider cell - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - - if (iNew == nNews - 1 && pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - } else { - ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); - tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); - } + ASSERT(iNew < nNews - 1); + ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); // move to next new page iNew++; From 795981feae04c4c470768c8de9cff682e10336f4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:54:38 +0000 Subject: [PATCH 55/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 68be6ec8cd..1503d68575 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -737,6 +737,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nNews = 0; } } else { + ASSERT(0); ASSERT(childNotLeaf); // set current new page right-most child From 2bb84eeb2becf4e636d06fff26d9a2443e8903ae Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:59:10 +0000 Subject: [PATCH 56/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 1503d68575..50f585ebd3 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -698,6 +698,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } iNew = 0; nNewCells = 0; + tdbBtreeZeroPage(pNews[iNew], &iarg); for (int iOld = 0; iOld < nOlds; iOld++) { SPage *pPage; @@ -735,6 +736,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; nNews = 0; + tdbBtreeZeroPage(pNews[iNew], &iarg); } } else { ASSERT(0); @@ -751,6 +753,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; nNewCells = 0; + tdbBtreeZeroPage(pNews[iNew], &iarg); } } } From 1fba9d5d3d162085f4dabf76ca79a1d009492c93 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 09:03:01 +0000 Subject: [PATCH 57/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 50f585ebd3..81b8be3588 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -735,8 +735,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; - nNews = 0; - tdbBtreeZeroPage(pNews[iNew], &iarg); + nNewCells = 0; + if (iNew < nNews) { + tdbBtreeZeroPage(pNews[iNew], &iarg); + } } } else { ASSERT(0); @@ -753,7 +755,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; nNewCells = 0; - tdbBtreeZeroPage(pNews[iNew], &iarg); + if (iNew < nNews) { + tdbBtreeZeroPage(pNews[iNew], &iarg); + } } } } From 33e119e9c037c63136bc3729de300e5bc9527e79 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 09:12:21 +0000 Subject: [PATCH 58/92] fix another TDB bug --- source/libs/tdb/src/db/tdbBtree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 81b8be3588..f0055723b6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -729,7 +729,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szNewCell; SPgno pgno; pgno = TDB_PAGE_PGNO(pNews[iNew]); - tdbBtreeEncodeCell(pPage, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); + tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); } From 48306365aed114b632fdf3ce76e7077da130692d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 09:35:25 +0000 Subject: [PATCH 59/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f0055723b6..0f5e6733ed 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -563,10 +563,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nNews = 0; struct { - int cnt; - int size; - SPage *pPage; - int oIdx; + int cnt; + int size; + int iPage; + int oIdx; } infoNews[5] = {0}; { // Get how many new pages are needed and the new distribution @@ -596,7 +596,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; - infoNews[nNews].pPage = pPage; + infoNews[nNews].iPage = oPage; infoNews[nNews].oIdx = oIdx; } } @@ -609,21 +609,23 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szLCell, szRCell; for (;;) { - pCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); + pCell = tdbPageGetCell(pOlds[infoNews[iNew - 1].iPage], infoNews[iNew - 1].oIdx); if (childNotLeaf) { - szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); + szLCell = szRCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); } else { - szLCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); + szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); - SPage *pPage = infoNews[iNew - 1].pPage; + int iPage = infoNews[iNew - 1].iPage; int oIdx = infoNews[iNew - 1].oIdx + 1; + SPage *pPage; for (;;) { + pPage = pOlds[iPage]; if (oIdx < TDB_PAGE_TOTAL_CELLS(pPage)) { break; } - pPage++; + iPage++; oIdx = 0; } @@ -646,8 +648,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { break; } - infoNews[iNew - 1].pPage--; - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(infoNews[iNew - 1].pPage) - 1; + infoNews[iNew - 1].iPage--; + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pOlds[infoNews[iNew - 1].iPage]) - 1; } infoNews[iNew].cnt++; From 27bcf222cec34b67d40b90d33c0d934f6fffb611 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 10:12:43 +0000 Subject: [PATCH 60/92] fix another bug --- source/libs/tdb/src/page/tdbPage.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index ee1a021a1b..5253a1fd67 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -288,8 +288,6 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { } else { TDB_PAGE_FCELL_SET(pPage, pFreeCell - pPage->pData); } - - goto _alloc_finish; } else { if (pPrevFreeCell) { pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, nxFreeCell); @@ -297,6 +295,8 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { TDB_PAGE_FCELL_SET(pPage, nxFreeCell); } } + + goto _alloc_finish; } else { pPrevFreeCell = pFreeCell; szPrevFreeCell = szFreeCell; From b633a96bed37bececfe62551f005a79cedd90644 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 10:17:23 +0000 Subject: [PATCH 61/92] more TDB --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 7bdcc38173..63b53b1732 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -8,7 +8,7 @@ TEST(tdb_test, simple_test) { STDb *pDb; // Open Env - ret = tdbEnvOpen("tdb", 1024, 20, &pEnv); + ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From 2aa6d7e38482a76211f265c6edbc31d52ba7e387 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 10:30:52 +0000 Subject: [PATCH 62/92] start the query --- source/libs/tdb/src/db/tdbDb.c | 5 +++++ source/libs/tdb/src/inc/tdbDb.h | 1 + source/libs/tdb/test/tdbTest.cpp | 35 ++++++++++++++++++++++++-------- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 00f38a19bb..27cbaeaa01 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -85,5 +85,10 @@ int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int v return -1; } + return 0; +} + +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen) { + // TODO return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 06ea74a83e..fdf4c4aca5 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -26,6 +26,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF int tdbDbClose(STDb *pDb); int tdbDbDrop(STDb *pDb); int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 63b53b1732..c5a2787aaf 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,6 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; + int nData = 1000; // Open Env ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); @@ -15,15 +16,33 @@ TEST(tdb_test, simple_test) { ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, NULL, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); - { // Insert some data - char key[64]; - char val[64]; + { + char key[64]; + char val[64]; + void *pVal; + int vLen; - for (int i = 1; i <= 1000; i++) { - sprintf(key, "key%d", i); - sprintf(val, "value%d", i); - ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val)); - GTEST_ASSERT_EQ(ret, 0); + { // Insert some data + + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + sprintf(val, "value%d", i); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val)); + GTEST_ASSERT_EQ(ret, 0); + } + } + + { // Query the data + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + sprintf(val, "value%d", i); + + ret = tdbDbGet(pDb, key, strlen(key), pVal, &vLen); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, strlen(val)); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + } } } From 4383b3b53a45d597b8c2cbcd8a11df46cf923d55 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 02:09:56 +0000 Subject: [PATCH 63/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 28 ++++++++++++++++++++++++++++ source/libs/tdb/src/db/tdbDb.c | 5 ++--- source/libs/tdb/src/inc/tdbBtree.h | 1 + source/libs/tdb/src/inc/tdbDb.h | 2 +- source/libs/tdb/test/tdbTest.cpp | 2 +- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0f5e6733ed..f5715866ed 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -203,6 +203,34 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p return 0; } +int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) { + SBtCursor btc; + SCell *pCell; + int cret; + SCellDecoder cd; + + tdbBtreeCursor(&btc, pBt); + + tdbBtCursorMoveTo(&btc, pKey, kLen, &cret); + + if (cret) { + return cret; + } + + pCell = tdbPageGetCell(btc.pPage, btc.idx); + tdbBtreeDecodeCell(btc.pPage, pCell, &cd); + + *vLen = cd.vLen; + // TODO: here may have memory leak + *ppVal = realloc(*ppVal, *vLen); + if (*ppVal == NULL) { + return -1; + } + + memcpy(*ppVal, cd.pVal, cd.vLen); + return 0; +} + static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { int ret; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 27cbaeaa01..2ba9daaa4a 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -88,7 +88,6 @@ int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int v return 0; } -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen) { - // TODO - return 0; +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { + return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index c1fe77c22e..a0b054a6a8 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -37,6 +37,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SB int tdbBtreeClose(SBTree *pBt); int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt); int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen); +int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index fdf4c4aca5..6a9d653eac 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -26,7 +26,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF int tdbDbClose(STDb *pDb); int tdbDbDrop(STDb *pDb); int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen); +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index c5a2787aaf..3d06d568e4 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -37,7 +37,7 @@ TEST(tdb_test, simple_test) { sprintf(key, "key%d", i); sprintf(val, "value%d", i); - ret = tdbDbGet(pDb, key, strlen(key), pVal, &vLen); + ret = tdbDbGet(pDb, key, strlen(key), &pVal, &vLen); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(vLen, strlen(val)); From 20b1c0d236103f458d356c84d93497dc3858f2bd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 02:27:16 +0000 Subject: [PATCH 64/92] more TDB --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 3d06d568e4..1acd78f188 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,7 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 1000; + int nData = 10000; // Open Env ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); From 820de153a80e5a95e02d91e4dcdf0068103918a9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 05:46:59 +0000 Subject: [PATCH 65/92] fix invalid read --- source/libs/tdb/src/db/tdbBtree.c | 4 ++-- source/libs/tdb/src/page/tdbPage.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f5715866ed..a733c0d981 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -712,7 +712,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } { // Do the real cell distribution - SPage *pOldsCopy[3]; + SPage *pOldsCopy[3] = {0}; SCell *pCell; int szCell; SBtreeInitPageArg iarg; @@ -722,7 +722,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { iarg.pBt = pBt; iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); for (int i = 0; i < nOlds; i++) { - tdbPageCreate(pOlds[0]->pageSize, pOldsCopy + i, NULL, NULL); + tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], NULL, NULL); tdbBtreeZeroPage(pOldsCopy[i], &iarg); tdbPageCopy(pOlds[i], pOldsCopy[i]); } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 5253a1fd67..15b6e218e6 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -52,7 +52,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) } ptr = (u8 *)((*xMalloc)(arg, size)); - if (pPage == NULL) { + if (ptr == NULL) { return -1; } From d59b65fa54d958b2caee0b207687000051e10fc8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 06:00:35 +0000 Subject: [PATCH 66/92] fix memory bug --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 1acd78f188..653efaa19a 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -19,7 +19,7 @@ TEST(tdb_test, simple_test) { { char key[64]; char val[64]; - void *pVal; + void *pVal = NULL; int vLen; { // Insert some data From 08a270777c72cb90c8453a3d734ee085ba1e0e96 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 06:58:52 +0000 Subject: [PATCH 67/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 48 ++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a733c0d981..0bb50fe937 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -517,8 +517,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nOlds; SPage *pOlds[3]; - SCell *pDivCell[2] = {0}; - int szDivCell[2]; + SCell *pDivCell[3] = {0}; + int szDivCell[3]; int sIdx; u8 childNotLeaf; SPgno rPgno; @@ -562,18 +562,18 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // copy the parent key out if child pages are not leaf page childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); if (childNotLeaf) { - for (int i = 0; i < nOlds - 1; i++) { - pCell = tdbPageGetCell(pParent, sIdx + i); + for (int i = 0; i < nOlds; i++) { + if (sIdx + i < TDB_PAGE_TOTAL_CELLS(pParent)) { + pCell = tdbPageGetCell(pParent, sIdx + i); + szDivCell[i] = tdbBtreeCellSize(pParent, pCell); + pDivCell[i] = malloc(szDivCell[i]); + memcpy(pDivCell[i], pCell, szDivCell[i]); + } - szDivCell[i] = tdbBtreeCellSize(pParent, pCell); - pDivCell[i] = malloc(szDivCell[i]); - memcpy(pDivCell, pCell, szDivCell[i]); - - ((SPgno *)pDivCell)[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; - ((SIntHdr *)pOlds[i]->pData)->pgno = 0; - - // here we insert the cell as an overflow cell to avoid - // the slow defragment process + if (i < nOlds - 1) { + ((SPgno *)pDivCell[i])[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; + ((SIntHdr *)pOlds[i]->pData)->pgno = 0; + } tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno; @@ -740,6 +740,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { szCell = tdbBtreeCellSize(pPage, pCell); ASSERT(nNewCells <= infoNews[iNew].cnt); + ASSERT(iNew < nNews - 1); if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); @@ -771,8 +772,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } else { - ASSERT(0); ASSERT(childNotLeaf); + ASSERT(iNew < nNews - 1); // set current new page right-most child ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; @@ -790,6 +791,19 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } + + if (childNotLeaf) { + ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); + ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; + + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[nNews - 1]); + } else { + ((SPgno *)pDivCell[nOlds - 1])[0] = TDB_PAGE_PGNO(pNews[nNews - 1]); + tdbPageInsertCell(pParent, sIdx, pDivCell[nOlds - 1], szDivCell[nOlds - 1], 0); + } + } } for (int i = 0; i < nOlds; i++) { @@ -797,6 +811,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + for (int i = 0; i < 3; i++) { + if (pDivCell[i]) { + free(pDivCell[i]); + } + } + return 0; } From b6d27d3759c55168626212c425b72407211e01c3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 07:43:19 +0000 Subject: [PATCH 68/92] fix another bug --- source/libs/tdb/src/db/tdbBtree.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0bb50fe937..de2ed60ec7 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -516,7 +516,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; int nOlds; - SPage *pOlds[3]; + SPage *pOlds[3] = {0}; SCell *pDivCell[3] = {0}; int szDivCell[3]; int sIdx; @@ -573,8 +573,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (i < nOlds - 1) { ((SPgno *)pDivCell[i])[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; ((SIntHdr *)pOlds[i]->pData)->pgno = 0; + tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } - tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno; } @@ -740,7 +740,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { szCell = tdbBtreeCellSize(pPage, pCell); ASSERT(nNewCells <= infoNews[iNew].cnt); - ASSERT(iNew < nNews - 1); + ASSERT(iNew < nNews); if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); @@ -791,18 +791,18 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } + } - if (childNotLeaf) { - ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); - ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; + if (childNotLeaf) { + ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); + ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - if (pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[nNews - 1]); - } else { - ((SPgno *)pDivCell[nOlds - 1])[0] = TDB_PAGE_PGNO(pNews[nNews - 1]); - tdbPageInsertCell(pParent, sIdx, pDivCell[nOlds - 1], szDivCell[nOlds - 1], 0); - } + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[nNews - 1]); + } else { + ((SPgno *)pDivCell[nOlds - 1])[0] = TDB_PAGE_PGNO(pNews[nNews - 1]); + tdbPageInsertCell(pParent, sIdx, pDivCell[nOlds - 1], szDivCell[nOlds - 1], 0); } } From 493670d2f0ba7c226af5b00b54c8ef4b0447ee0e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 08:11:07 +0000 Subject: [PATCH 69/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index de2ed60ec7..eec75d3a13 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -482,9 +482,11 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { u8 flags; SIntHdr *pIntHdr; SBtreeInitPageArg zArg; + u8 leaf; pPager = pRoot->pPager; flags = TDB_BTREE_PAGE_GET_FLAGS(pRoot); + leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Allocate a new child page zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT); @@ -494,6 +496,10 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } + if (!leaf) { + ((SIntHdr *)pChild->pData)->pgno = ((SIntHdr *)(pRoot->pData))->pgno; + } + // Copy the root page content to the child page tdbPageCopy(pRoot, pChild); From c9fe5301564e14313331b0231b2c344c3961bbad Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 08:16:14 +0000 Subject: [PATCH 70/92] add more test --- source/libs/tdb/test/tdbTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 653efaa19a..5285a5ee94 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 10000; + int nData = 50000; // Open Env - ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); + ret = tdbEnvOpen("tdb", 1024, 2560, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From eb9d3f2196581a1c65534766e582e9dbb7d1e121 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 08:40:37 +0000 Subject: [PATCH 71/92] more TDB --- source/libs/tdb/src/inc/tdbDb.h | 3 ++- source/libs/tdb/test/tdbTest.cpp | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 6a9d653eac..7a65966be3 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -20,7 +20,8 @@ extern "C" { #endif -typedef struct STDb STDb; +typedef struct STDb STDb; +typedef struct STDbC STDbC; int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb); int tdbDbClose(STDb *pDb); diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 5285a5ee94..79532d07b7 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 50000; + int nData = 200000; // Open Env - ret = tdbEnvOpen("tdb", 1024, 2560, &pEnv); + ret = tdbEnvOpen("tdb", 1024, 25600, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -44,6 +44,9 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); } } + + { // Loop to query the data + } } ret = tdbDbDrop(pDb); From 347d1e6f9932ddbdfb2e2d352808cad6552b97df Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 09:03:01 +0000 Subject: [PATCH 72/92] start more fix --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 79532d07b7..cca2082e53 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,7 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 200000; + int nData = 160511; // Open Env ret = tdbEnvOpen("tdb", 1024, 25600, &pEnv); From 61b2f52dca24dd7a7ba66b93194aabfa87704c57 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 09:53:34 +0000 Subject: [PATCH 73/92] now TDB can write 1M records --- source/libs/tdb/src/db/tdbBtree.c | 11 ++++++----- source/libs/tdb/test/tdbTest.cpp | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index eec75d3a13..e244b77f95 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -817,11 +817,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - for (int i = 0; i < 3; i++) { - if (pDivCell[i]) { - free(pDivCell[i]); - } - } + // TODO: here has memory leak + // for (int i = 0; i < 3; i++) { + // if (pDivCell[i]) { + // free(pDivCell[i]); + // } + // } return 0; } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index cca2082e53..0759de54b7 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 160511; + int nData = 1000000; // Open Env - ret = tdbEnvOpen("tdb", 1024, 25600, &pEnv); + ret = tdbEnvOpen("tdb", 4096, 25600, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From e20e13d2dbb9b3234db44b6539a918c6016dcd02 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 10:01:17 +0000 Subject: [PATCH 74/92] more TDB --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 0759de54b7..56284f0271 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,7 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 1000000; + int nData = 2000000; // Open Env ret = tdbEnvOpen("tdb", 4096, 25600, &pEnv); From 488d44a55515152d0982b1a343591343830b4a9c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 10:14:46 +0000 Subject: [PATCH 75/92] w/r 10M K-V pairs --- source/libs/tdb/test/tdbTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 56284f0271..df96a39943 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 2000000; + int nData = 10000000; // Open Env - ret = tdbEnvOpen("tdb", 4096, 25600, &pEnv); + ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From b8ac2d30d5b2658d2dc68560938b07ea55c71b5f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 01:42:03 +0000 Subject: [PATCH 76/92] refact --- source/libs/tdb/src/db/tdbBtree.c | 16 ++++++++-------- source/libs/tdb/src/db/tdbDb.c | 6 +++--- source/libs/tdb/src/inc/tdbBtree.h | 15 ++++++++++----- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e244b77f95..befc021d28 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -67,7 +67,7 @@ typedef struct { u8 *pTmpSpace; } SCellDecoder; -static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst); +static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst); static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); static int tdbBtreeOpenImpl(SBTree *pBt); static int tdbBtreeZeroPage(SPage *pPage, void *arg); @@ -75,7 +75,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg); static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell); static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); -static int tdbBtreeBalance(SBtCursor *pCur); +static int tdbBtreeBalance(SBTC *pCur); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { @@ -131,7 +131,7 @@ int tdbBtreeClose(SBTree *pBt) { return 0; } -int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt) { +int tdbBtreeCursor(SBTC *pCur, SBTree *pBt) { pCur->pBt = pBt; pCur->iPage = -1; pCur->pPage = NULL; @@ -140,7 +140,7 @@ int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt) { return 0; } -int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { +int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { int ret; int idx; SPager *pPager; @@ -204,7 +204,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) { - SBtCursor btc; + SBTC btc; SCell *pCell; int cret; SCellDecoder cd; @@ -231,7 +231,7 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen return 0; } -static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { +static int tdbBtCursorMoveToChild(SBTC *pCur, SPgno pgno) { int ret; pCur->pgStack[pCur->iPage] = pCur->pPage; @@ -248,7 +248,7 @@ static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { return 0; } -static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst) { +static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst) { int ret; SBTree *pBt; SPager *pPager; @@ -827,7 +827,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return 0; } -static int tdbBtreeBalance(SBtCursor *pCur) { +static int tdbBtreeBalance(SBTC *pCur) { int iPage; SPage *pParent; SPage *pPage; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 2ba9daaa4a..9c5fb551f0 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -70,9 +70,9 @@ int tdbDbDrop(STDb *pDb) { } int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { - SBtCursor btc; - SBtCursor *pCur; - int ret; + SBTC btc; + SBTC *pCur; + int ret; pCur = &btc; ret = tdbBtreeCursor(pCur, pDb->pBt); diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index a0b054a6a8..60078f68f4 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -20,10 +20,15 @@ extern "C" { #endif -typedef struct SBTree SBTree; -typedef struct SBtCursor SBtCursor; +typedef struct SBTree SBTree; +typedef struct SBTC SBTC; +typedef struct SBtInfo { + SPgno root; + int nLevel; + int nData; +} SBtInfo; -struct SBtCursor { +struct SBTC { SBTree *pBt; i8 iPage; SPage *pPage; @@ -35,8 +40,8 @@ struct SBtCursor { int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); -int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt); -int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen); +int tdbBtreeCursor(SBTC *pCur, SBTree *pBt); +int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus From 6276f2c33e84e90d5b427a574b444a4413cfacc9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 01:48:35 +0000 Subject: [PATCH 77/92] refact --- source/libs/tdb/src/db/tdbDb.c | 16 ++++++++-------- source/libs/tdb/src/inc/tdbDb.h | 12 ++++++------ source/libs/tdb/src/inc/tdbInt.h | 2 +- source/libs/tdb/test/tdbTest.cpp | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 9c5fb551f0..e889e8772b 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,13 +15,13 @@ #include "tdbInt.h" -struct STDb { +struct STDB { STEnv *pEnv; SBTree *pBt; }; -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb) { - STDb *pDb; +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb) { + STDB *pDb; SPager *pPager; int ret; char fFullName[TDB_FILENAME_LEN]; @@ -30,7 +30,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF *ppDb = NULL; - pDb = (STDb *)calloc(1, sizeof(*pDb)); + pDb = (STDB *)calloc(1, sizeof(*pDb)); if (pDb == NULL) { return -1; } @@ -59,17 +59,17 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF return 0; } -int tdbDbClose(STDb *pDb) { +int tdbDbClose(STDB *pDb) { // TODO return 0; } -int tdbDbDrop(STDb *pDb) { +int tdbDbDrop(STDB *pDb) { // TODO return 0; } -int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { +int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { SBTC btc; SBTC *pCur; int ret; @@ -88,6 +88,6 @@ int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int v return 0; } -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { +int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 7a65966be3..e48de5e8c7 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -20,14 +20,14 @@ extern "C" { #endif -typedef struct STDb STDb; +typedef struct STDB STDB; typedef struct STDbC STDbC; -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb); -int tdbDbClose(STDb *pDb); -int tdbDbDrop(STDb *pDb); -int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb); +int tdbDbClose(STDB *pDb); +int tdbDbDrop(STDB *pDb); +int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); +int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 3ebba0a4b5..98845bb66f 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -95,7 +95,7 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) { // tdb_log #define tdbError(var) -typedef TD_DLIST(STDb) STDbList; +typedef TD_DLIST(STDB) STDbList; typedef TD_DLIST(SPgFile) SPgFileList; typedef TD_DLIST_NODE(SPgFile) SPgFileListNode; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index df96a39943..4f6bde3c96 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -5,7 +5,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; - STDb *pDb; + STDB *pDb; int nData = 10000000; // Open Env From 212e24dda649b879decfd7bf20c0bb03c7065475 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 04:59:20 +0000 Subject: [PATCH 78/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 1 + source/libs/tdb/test/tdbTest.cpp | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index befc021d28..c3adf7d2b6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -914,6 +914,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, u8 *pPayload, const void *pKey, i return 0; } +// TODO: allow vLen = 0 static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell) { u8 flags; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 4f6bde3c96..05a0f3a6ae 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -17,10 +17,8 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(ret, 0); { - char key[64]; - char val[64]; - void *pVal = NULL; - int vLen; + char key[64]; + char val[64]; { // Insert some data @@ -33,6 +31,9 @@ TEST(tdb_test, simple_test) { } { // Query the data + void *pVal = NULL; + int vLen; + for (int i = 1; i <= nData; i++) { sprintf(key, "key%d", i); sprintf(val, "value%d", i); @@ -46,6 +47,8 @@ TEST(tdb_test, simple_test) { } { // Loop to query the data + // STDbC *pDBC; + // tdbDBNext(pDBC, p) } } From f4359a6fb3c82295ac5b8b752e2804b7dd19ad84 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 06:02:38 +0000 Subject: [PATCH 79/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 7 ++++--- source/libs/tdb/src/inc/tdbUtil.h | 22 ++++++++++++++++++++++ source/libs/tdb/test/tdbTest.cpp | 2 ++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c3adf7d2b6..0ef5d64d30 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -207,6 +207,7 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen SBTC btc; SCell *pCell; int cret; + void *pVal; SCellDecoder cd; tdbBtreeCursor(&btc, pBt); @@ -221,12 +222,12 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen tdbBtreeDecodeCell(btc.pPage, pCell, &cd); *vLen = cd.vLen; - // TODO: here may have memory leak - *ppVal = realloc(*ppVal, *vLen); - if (*ppVal == NULL) { + pVal = TDB_REALLOC(*ppVal, *vLen); + if (pVal == NULL) { return -1; } + *ppVal = pVal; memcpy(*ppVal, cd.pVal, cd.vLen); return 0; } diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 9358aae236..88fc846bf1 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -39,6 +39,28 @@ int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); int tdbPRead(int fd, void *pData, int count, i64 offset); +#define TDB_REALLOC(PTR, SIZE) \ + ({ \ + void *nPtr; \ + if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ + nPtr = realloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \ + if (nPtr) { \ + ((int *)nPtr)[0] = (SIZE); \ + nPtr = (char *)nPtr + sizeof(int); \ + } \ + } else { \ + nPtr = (PTR); \ + } \ + nPtr; \ + }) + +#define TDB_FREE(PTR) \ + do { \ + if (PTR) { \ + free((char *)(PTR) - sizeof(int)); \ + } \ + } while (0) + static inline void *tdbOsMalloc(void *arg, size_t size) { void *ptr; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 05a0f3a6ae..46437392cd 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -44,6 +44,8 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(vLen, strlen(val)); GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); } + + TDB_FREE(pVal); } { // Loop to query the data From 9ffdf09986e1ee511d12b97e4e39531ef63fa9df Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 06:15:21 +0000 Subject: [PATCH 80/92] design dbc api --- source/libs/tdb/src/db/tdbDb.c | 15 +++++++++++++++ source/libs/tdb/src/inc/tdbDb.h | 8 +++++++- source/libs/tdb/test/tdbTest.cpp | 25 ++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index e889e8772b..98f22e4d25 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -90,4 +90,19 @@ int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int v int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); +} + +int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc) { + // TODO + return 0; +} + +int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { + // TODO + return 0; +} + +int tdbDbcClose(STDBC *pDbc) { + // TODO + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index e48de5e8c7..4e69de2bfa 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -21,14 +21,20 @@ extern "C" { #endif typedef struct STDB STDB; -typedef struct STDbC STDbC; +typedef struct STDBC STDBC; +// STDB int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb); int tdbDbClose(STDB *pDb); int tdbDbDrop(STDB *pDb); int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); +// STDBC +int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc); +int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbDbcClose(STDBC *pDbc); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 46437392cd..164b2c0b83 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -48,9 +48,28 @@ TEST(tdb_test, simple_test) { TDB_FREE(pVal); } - { // Loop to query the data - // STDbC *pDBC; - // tdbDBNext(pDBC, p) + { // Iterate to query the DB data + STDBC *pDBC; + void *pKey = NULL; + void *pVal = NULL; + int vLen, kLen; + int count = 0; + + ret = tdbDbcOpen(pDb, &pDBC); + GTEST_ASSERT_EQ(ret, 0); + + for (;;) { + ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + count++; + } + + GTEST_ASSERT_EQ(count, nData); + + tdbDbcClose(pDBC); + + TDB_FREE(pKey); + TDB_FREE(pVal); } } From cc176b1037ebed74221b6f6d99f63775d2f80987 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 06:42:42 +0000 Subject: [PATCH 81/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 29 +++++++++++++++++++++-- source/libs/tdb/src/db/tdbDb.c | 37 +++++++++++++++++++++++++----- source/libs/tdb/src/inc/tdbBtree.h | 10 +++++++- source/libs/tdb/src/inc/tdbDb.h | 2 +- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0ef5d64d30..2b0ba62ed3 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -131,7 +131,7 @@ int tdbBtreeClose(SBTree *pBt) { return 0; } -int tdbBtreeCursor(SBTC *pCur, SBTree *pBt) { +int tdbBtcOpen(SBTC *pCur, SBTree *pBt) { pCur->pBt = pBt; pCur->iPage = -1; pCur->pPage = NULL; @@ -210,7 +210,7 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen void *pVal; SCellDecoder cd; - tdbBtreeCursor(&btc, pBt); + tdbBtcOpen(&btc, pBt); tdbBtCursorMoveTo(&btc, pKey, kLen, &cret); @@ -1072,3 +1072,28 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { } #endif + +int tdbBtcMoveToFirst(SBTC *pBtc) { + // TODO + return 0; +} + +int tdbBtcMoveToLast(SBTC *pBtc) { + // TODO + return 0; +} + +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen) { + // TODO + return 0; +} + +int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { + // TODO + return 0; +} + +int tdbBtcClose(SBTC *pBtc) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 98f22e4d25..4e74dc4cbb 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -20,6 +20,10 @@ struct STDB { SBTree *pBt; }; +struct STDBC { + SBTC btc; +}; + int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb) { STDB *pDb; SPager *pPager; @@ -75,7 +79,7 @@ int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int v int ret; pCur = &btc; - ret = tdbBtreeCursor(pCur, pDb->pBt); + ret = tdbBtcOpen(pCur, pDb->pBt); if (ret < 0) { return -1; } @@ -92,17 +96,38 @@ int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } -int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc) { - // TODO +int tdbDbcOpen(STDB *pDb, STDBC **ppDbc) { + int ret; + STDBC *pDbc = NULL; + + *ppDbc = NULL; + pDbc = malloc(sizeof(*pDbc)); + if (pDbc == NULL) { + return -1; + } + + tdbBtcOpen(&pDbc->btc, pDb->pBt); + + // TODO: move to first now, we can move to any key-value + // and in any direction, design new APIs. + ret = tdbBtcMoveToFirst(&pDbc->btc); + if (ret < 0) { + ASSERT(0); + return -1; + } + + *ppDbc = pDbc; return 0; } int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { - // TODO - return 0; + return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); } int tdbDbcClose(STDBC *pDbc) { - // TODO + if (pDbc) { + free(pDbc); + } + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index 60078f68f4..e46e8bb78b 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -38,12 +38,20 @@ struct SBTC { void *pBuf; }; +// SBTree int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); -int tdbBtreeCursor(SBTC *pCur, SBTree *pBt); int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); +// SBTC +int tdbBtcOpen(SBTC *pCur, SBTree *pBt); +int tdbBtcMoveToFirst(SBTC *pBtc); +int tdbBtcMoveToLast(SBTC *pBtc); +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen); +int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbBtcClose(SBTC *pBtc); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 4e69de2bfa..b96076b826 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -31,7 +31,7 @@ int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int v int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); // STDBC -int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc); +int tdbDbcOpen(STDB *pDb, STDBC **ppDbc); int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbDbcClose(STDBC *pDbc); From 07f6d03a5246aaced36ee3272c6852f703779871 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 07:19:48 +0000 Subject: [PATCH 82/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 107 +++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 2b0ba62ed3..7d925d69f4 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -131,15 +131,6 @@ int tdbBtreeClose(SBTree *pBt) { return 0; } -int tdbBtcOpen(SBTC *pCur, SBTree *pBt) { - pCur->pBt = pBt; - pCur->iPage = -1; - pCur->pPage = NULL; - pCur->idx = -1; - - return 0; -} - int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { int ret; int idx; @@ -1073,13 +1064,107 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { #endif +int tdbBtcOpen(SBTC *pCur, SBTree *pBt) { + pCur->pBt = pBt; + pCur->iPage = -1; + pCur->pPage = NULL; + pCur->idx = -1; + + return 0; +} + int tdbBtcMoveToFirst(SBTC *pBtc) { - // TODO + int ret; + SBTree *pBt; + SPager *pPager; + u8 flags; + SCell *pCell; + SPgno pgno; + + pBt = pBtc->pBt; + pPager = pBt->pPager; + + if (pBtc->iPage < 0) { + // move a clean cursor + ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->iPage = 0; + pBtc->idx = 0; + } else { + // move from a position + ASSERT(0); + } + + // move downward + for (;;) { + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + + if (TDB_BTREE_PAGE_IS_LEAF(flags)) break; + + pCell = tdbPageGetCell(pBtc->pPage, 0); + pgno = *(SPgno *)pCell; + + ret = tdbBtCursorMoveToChild(pBtc, pgno); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->idx = 0; + } + return 0; } int tdbBtcMoveToLast(SBTC *pBtc) { - // TODO + int ret; + SBTree *pBt; + SPager *pPager; + u8 flags; + SPgno pgno; + + pBt = pBtc->pBt; + pPager = pBt->pPager; + + if (pBtc->iPage < 0) { + // move a clean cursor + ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->iPage = 0; + } else { + // move from a position + ASSERT(0); + } + + // move downward + for (;;) { + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + + if (TDB_BTREE_PAGE_IS_LEAF(flags)) { + // TODO: handle empty case + ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0); + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1; + break; + } else { + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; + + ret = tdbBtCursorMoveToChild(pBtc, pgno); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + } + return 0; } From f75cb898663afc37a7c45b964d4b16e462c8d663 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 07:37:41 +0000 Subject: [PATCH 83/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 63 ++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 7d925d69f4..8c86c576c2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -77,6 +77,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); static int tdbBtreeBalance(SBTC *pCur); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); +static int tdbBtcMoveToNext(SBTC *pBtc); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; @@ -1174,7 +1175,67 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen) { } int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { - // TODO + SCell *pCell; + SCellDecoder cd; + void *pKey, *pVal; + int ret; + + if (0) { + // TODO: no valid data current cursor is pointing to + return -1; + } + + pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); + + tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd); + + pKey = TDB_REALLOC(*ppKey, cd.kLen); + if (pKey == NULL) { + return -1; + } + + // TODO: vLen may be zero + pVal = TDB_REALLOC(*ppVal, cd.vLen); + if (pVal == NULL) { + TDB_FREE(pKey); + return -1; + } + + *ppKey = pKey; + *ppVal = pVal; + + *kLen = cd.kLen; + *vLen = cd.vLen; + + memcpy(pKey, cd.pKey, cd.kLen); + memcpy(pVal, cd.pVal, cd.vLen); + + ret = tdbBtcMoveToNext(pBtc); + + return 0; +} + +static int tdbBtcMoveToNext(SBTC *pBtc) { + pBtc->idx++; + + if (pBtc->idx < 0) return -1; + + if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { + return 0; + } + + // Move upward + for (;;) { + // TODO: release the page + + /* code */ + } + + // Move downward + for (;;) { + /* code */ + } + return 0; } From af98aa400509287f1f196754f69186bbd0bf582b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 07:48:55 +0000 Subject: [PATCH 84/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 51 ++++++++++++++++------------- source/libs/tdb/src/db/tdbPCache.c | 47 ++++++++++---------------- source/libs/tdb/src/db/tdbPager.c | 4 +++ source/libs/tdb/src/inc/tdbPCache.h | 25 +++++++------- source/libs/tdb/src/inc/tdbPager.h | 19 ++++++----- 5 files changed, 73 insertions(+), 73 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8c86c576c2..282c5a98a6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -78,6 +78,8 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD static int tdbBtreeBalance(SBTC *pCur); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); static int tdbBtcMoveToNext(SBTC *pBtc); +static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno); +static int tdbBtcMoveUpward(SBTC *pBtc); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; @@ -224,23 +226,6 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen return 0; } -static int tdbBtCursorMoveToChild(SBTC *pCur, SPgno pgno) { - int ret; - - pCur->pgStack[pCur->iPage] = pCur->pPage; - pCur->idxStack[pCur->iPage] = pCur->idx; - pCur->iPage++; - pCur->pPage = NULL; - pCur->idx = -1; - - ret = tdbPagerFetchPage(pCur->pBt->pPager, pgno, &pCur->pPage, tdbBtreeInitPage, pCur->pBt); - if (ret < 0) { - ASSERT(0); - } - - return 0; -} - static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst) { int ret; SBTree *pBt; @@ -318,16 +303,16 @@ static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst) } else { if (c <= 0) { pCur->idx = midx; - tdbBtCursorMoveToChild(pCur, cd.pgno); + tdbBtcMoveDownward(pCur, cd.pgno); } else { pCur->idx = midx + 1; if (midx == nCells - 1) { /* Move to right-most child */ - tdbBtCursorMoveToChild(pCur, ((SIntHdr *)pCur->pPage->pData)->pgno); + tdbBtcMoveDownward(pCur, ((SIntHdr *)pCur->pPage->pData)->pgno); } else { pCell = tdbPageGetCell(pPage, pCur->idx); tdbBtreeDecodeCell(pPage, pCell, &cd); - tdbBtCursorMoveToChild(pCur, cd.pgno); + tdbBtcMoveDownward(pCur, cd.pgno); } } } @@ -1109,7 +1094,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { pCell = tdbPageGetCell(pBtc->pPage, 0); pgno = *(SPgno *)pCell; - ret = tdbBtCursorMoveToChild(pBtc, pgno); + ret = tdbBtcMoveDownward(pBtc, pgno); if (ret < 0) { ASSERT(0); return -1; @@ -1158,7 +1143,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; - ret = tdbBtCursorMoveToChild(pBtc, pgno); + ret = tdbBtcMoveDownward(pBtc, pgno); if (ret < 0) { ASSERT(0); return -1; @@ -1242,4 +1227,26 @@ static int tdbBtcMoveToNext(SBTC *pBtc) { int tdbBtcClose(SBTC *pBtc) { // TODO return 0; +} + +static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno) { + int ret; + + pCur->pgStack[pCur->iPage] = pCur->pPage; + pCur->idxStack[pCur->iPage] = pCur->idx; + pCur->iPage++; + pCur->pPage = NULL; + pCur->idx = -1; + + ret = tdbPagerFetchPage(pCur->pBt->pPager, pgno, &pCur->pPage, tdbBtreeInitPage, pCur->pBt); + if (ret < 0) { + ASSERT(0); + } + + return 0; +} + +static int tdbBtcMoveUpward(SBTC *pBtc) { + // TODO + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 1fc0847b38..3c7d037faa 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -53,10 +53,10 @@ static void tdbPCacheLock(SPCache *pCache); static void tdbPCacheUnlock(SPCache *pCache); static bool tdbPCacheLocked(SPCache *pCache); static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); -static void tdbPCachePinPage(SPage *pPage); -static void tdbPCacheRemovePageFromHash(SPage *pPage); -static void tdbPCacheAddPageToHash(SPage *pPage); -static void tdbPCacheUnpinPage(SPage *pPage); +static void tdbPCachePinPage(SPCache *pCache, SPage *pPage); +static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage); +static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage); +static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage); int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { SPCache *pCache; @@ -100,7 +100,7 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) { return pPage; } -void tdbPCacheRelease(SPage *pPage) { +void tdbPCacheRelease(SPCache *pCache, SPage *pPage) { i32 nRef; nRef = TDB_UNREF_PAGE(pPage); @@ -108,7 +108,7 @@ void tdbPCacheRelease(SPage *pPage) { if (nRef == 0) { if (1 /*TODO: page still clean*/) { - tdbPCacheUnpinPage(pPage); + tdbPCacheUnpinPage(pCache, pPage); } else { // TODO ASSERT(0); @@ -142,7 +142,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe if (pPage || !alcNewPage) { if (pPage) { - tdbPCachePinPage(pPage); + tdbPCachePinPage(pCache, pPage); } return pPage; } @@ -158,8 +158,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe // 3. Try to Recycle a page if (!pPage && !pCache->lru.pLruPrev->isAnchor) { pPage = pCache->lru.pLruPrev; - tdbPCacheRemovePageFromHash(pPage); - tdbPCachePinPage(pPage); + tdbPCacheRemovePageFromHash(pCache, pPage); + tdbPCachePinPage(pCache, pPage); } // 4. Try a stress allocation (TODO) @@ -171,16 +171,13 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid)); pPage->pLruNext = NULL; pPage->pPager = NULL; - tdbPCacheAddPageToHash(pPage); + tdbPCacheAddPageToHash(pCache, pPage); } return pPage; } -static void tdbPCachePinPage(SPage *pPage) { - SPCache *pCache; - - pCache = pPage->pCache; +static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { if (!PAGE_IS_PINNED(pPage)) { pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; @@ -190,11 +187,8 @@ static void tdbPCachePinPage(SPage *pPage) { } } -static void tdbPCacheUnpinPage(SPage *pPage) { - SPCache *pCache; - i32 nRef; - - pCache = pPage->pCache; +static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { + i32 nRef; tdbPCacheLock(pCache); @@ -215,12 +209,10 @@ static void tdbPCacheUnpinPage(SPage *pPage) { tdbPCacheUnlock(pCache); } -static void tdbPCacheRemovePageFromHash(SPage *pPage) { - SPCache *pCache; - SPage **ppPage; - int h; +static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { + SPage **ppPage; + int h; - pCache = pPage->pCache; h = PCACHE_PAGE_HASH(&(pPage->pgid)); for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) ; @@ -230,11 +222,9 @@ static void tdbPCacheRemovePageFromHash(SPage *pPage) { pCache->nPage--; } -static void tdbPCacheAddPageToHash(SPage *pPage) { - SPCache *pCache; - int h; +static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { + int h; - pCache = pPage->pCache; h = PCACHE_PAGE_HASH(&(pPage->pgid)) % pCache->nHash; pPage->pHashNext = pCache->pgHash[h]; @@ -264,7 +254,6 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { // pPage->pgid = 0; pPage->isAnchor = 0; pPage->isLocalPage = 1; - pPage->pCache = pCache; TDB_INIT_PAGE_REF(pPage); pPage->pHashNext = NULL; pPage->pLruNext = NULL; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index ac0bc15e1d..fe4b9aa123 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -255,6 +255,10 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage return 0; } +void tdbPagerReturnPage(SPager *pPager, SPage *pPage) { + tdbPCacheRelease(pPager->pCache, pPage); +} + static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { // TODO: Allocate a page from the free list return 0; diff --git a/source/libs/tdb/src/inc/tdbPCache.h b/source/libs/tdb/src/inc/tdbPCache.h index ff4f1acbb6..c7fa155615 100644 --- a/source/libs/tdb/src/inc/tdbPCache.h +++ b/source/libs/tdb/src/inc/tdbPCache.h @@ -21,23 +21,22 @@ extern "C" { #endif #define TDB_PCACHE_PAGE \ - u8 isAnchor; \ - u8 isLocalPage; \ - u8 isDirty; \ - i32 nRef; \ - SPCache *pCache; \ - SPage *pFreeNext; \ - SPage *pHashNext; \ - SPage *pLruNext; \ - SPage *pLruPrev; \ - SPage *pDirtyNext; \ - SPager *pPager; \ - SPgid pgid; + u8 isAnchor; \ + u8 isLocalPage; \ + u8 isDirty; \ + i32 nRef; \ + SPage *pFreeNext; \ + SPage *pHashNext; \ + SPage *pLruNext; \ + SPage *pLruPrev; \ + SPage *pDirtyNext; \ + SPager *pPager; \ + SPgid pgid; int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache); int tdbPCacheClose(SPCache *pCache); SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); -void tdbPCacheRelease(SPage *pPage); +void tdbPCacheRelease(SPCache *pCache, SPage *pPage); int tdbPCacheGetPageSize(SPCache *pCache); #ifdef __cplusplus diff --git a/source/libs/tdb/src/inc/tdbPager.h b/source/libs/tdb/src/inc/tdbPager.h index e4ed8552fd..f4cc822f27 100644 --- a/source/libs/tdb/src/inc/tdbPager.h +++ b/source/libs/tdb/src/inc/tdbPager.h @@ -20,15 +20,16 @@ extern "C" { #endif -int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); -int tdbPagerClose(SPager *pPager); -int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); -int tdbPagerWrite(SPager *pPager, SPage *pPage); -int tdbPagerBegin(SPager *pPager); -int tdbPagerCommit(SPager *pPager); -int tdbPagerGetPageSize(SPager *pPager); -int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); -int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); +int tdbPagerClose(SPager *pPager); +int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); +int tdbPagerWrite(SPager *pPager, SPage *pPage); +int tdbPagerBegin(SPager *pPager); +int tdbPagerCommit(SPager *pPager); +int tdbPagerGetPageSize(SPager *pPager); +int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +void tdbPagerReturnPage(SPager *pPager, SPage *pPage); #ifdef __cplusplus } From cbc8720d293a05ecb3fec048c45bd62d8f470b72 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 08:35:40 +0000 Subject: [PATCH 85/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 51 ++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 282c5a98a6..d135eedb7c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1201,24 +1201,58 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { } static int tdbBtcMoveToNext(SBTC *pBtc) { - pBtc->idx++; + int nCells; + SPgno pgno; + SCell *pCell; + u8 flags; + + ASSERT(TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage))); if (pBtc->idx < 0) return -1; + pBtc->idx++; if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { return 0; } + if (pBtc->iPage == 0) { + pBtc->idx = -1; + return 0; + } + // Move upward for (;;) { - // TODO: release the page + tdbBtcMoveUpward(pBtc); + pBtc->idx++; - /* code */ + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + if (pBtc->idx <= nCells) { + break; + } + + if (pBtc->iPage == 0) { + pBtc->idx = -1; + return 0; + } } // Move downward for (;;) { - /* code */ + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + if (pBtc->idx < nCells) { + pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); + pgno = *(SPgno *)pCell; + } else { + pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; + } + + tdbBtcMoveDownward(pBtc, pgno); + pBtc->idx = 0; + + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + if (TDB_BTREE_PAGE_IS_LEAF(flags)) { + break; + } } return 0; @@ -1247,6 +1281,13 @@ static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno) { } static int tdbBtcMoveUpward(SBTC *pBtc) { - // TODO + if (pBtc->iPage == 0) return -1; + + // tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage); + + pBtc->iPage--; + pBtc->pPage = pBtc->pgStack[pBtc->iPage]; + pBtc->idx = pBtc->idxStack[pBtc->iPage]; + return 0; } \ No newline at end of file From a9acfce8860d6174a734262e010f8108b469b406 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 08:41:00 +0000 Subject: [PATCH 86/92] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index d135eedb7c..de77fb2649 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1165,8 +1165,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { void *pKey, *pVal; int ret; - if (0) { - // TODO: no valid data current cursor is pointing to + if (pBtc->idx < 0) { return -1; } From f47953797845ae9d17548ef457cee4cb40b3e57f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 09:01:09 +0000 Subject: [PATCH 87/92] more --- source/libs/tdb/test/tdbTest.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 164b2c0b83..b9ae2dff7d 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -2,6 +2,8 @@ #include "tdbInt.h" +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); + TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; @@ -61,6 +63,11 @@ TEST(tdb_test, simple_test) { for (;;) { ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; + + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; + count++; } @@ -82,4 +89,19 @@ TEST(tdb_test, simple_test) { // Close Env ret = tdbEnvClose(pEnv); GTEST_ASSERT_EQ(ret, 0); +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + k1 = std::strtol((char *)pKey1 + 3, nullptr, kLen1 - 3); + k2 = std::strtol((char *)pKey2 + 3, nullptr, kLen2 - 3); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } } \ No newline at end of file From 4ea8c88810ed1f578f7817e483fd2a7b0751513a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 09:30:07 +0000 Subject: [PATCH 88/92] more TDB --- source/libs/tdb/test/tdbTest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index b9ae2dff7d..4c7af58b14 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -15,7 +15,7 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(ret, 0); // Create a database - ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, NULL, pEnv, &pDb); + ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, tKeyCmpr, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -94,8 +94,8 @@ TEST(tdb_test, simple_test) { static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { int k1, k2; - k1 = std::strtol((char *)pKey1 + 3, nullptr, kLen1 - 3); - k2 = std::strtol((char *)pKey2 + 3, nullptr, kLen2 - 3); + k1 = std::strtol((char *)pKey1 + 3, nullptr, 10); + k2 = std::strtol((char *)pKey2 + 3, nullptr, 10); if (k1 < k2) { return -1; From 72f1719a93254028be4f9d2248557acf7e4c380b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 10:28:23 +0000 Subject: [PATCH 89/92] add btree debug helper function --- source/libs/tdb/src/db/tdbBtree.c | 37 ++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index de77fb2649..45706ae3dc 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1289,4 +1289,39 @@ static int tdbBtcMoveUpward(SBTC *pBtc) { pBtc->idx = pBtc->idxStack[pBtc->iPage]; return 0; -} \ No newline at end of file +} + +#ifndef NODEBUG +typedef struct { + SPgno pgno; + u8 root; + u8 leaf; + SPgno rChild; + int nCells; + int nOvfl; +} SBtPageInfo; + +SBtPageInfo btPageInfos[20]; + +void tdbBtPageInfo(SPage *pPage, int idx) { + u8 flags; + SBtPageInfo *pBtPageInfo; + + pBtPageInfo = btPageInfos + idx; + + pBtPageInfo->pgno = TDB_PAGE_PGNO(pPage); + + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + + pBtPageInfo->root = TDB_BTREE_PAGE_IS_ROOT(flags); + pBtPageInfo->leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + + pBtPageInfo->rChild = 0; + if (!pBtPageInfo->leaf) { + pBtPageInfo->rChild = *(SPgno *)(pPage->pData + 1); + } + + pBtPageInfo->nCells = TDB_PAGE_TOTAL_CELLS(pPage) - pPage->nOverflow; + pBtPageInfo->nOvfl = pPage->nOverflow; +} +#endif \ No newline at end of file From fc64a2c72d00464356d2d4f8b7612a98610568fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 10:54:54 +0000 Subject: [PATCH 90/92] more --- source/libs/tdb/test/tdbTest.cpp | 34 +++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 4c7af58b14..c256f07cf6 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -3,19 +3,23 @@ #include "tdbInt.h" static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); TEST(tdb_test, simple_test) { - int ret; - STEnv *pEnv; - STDB *pDb; - int nData = 10000000; + int ret; + STEnv *pEnv; + STDB *pDb; + FKeyComparator compFunc; + int nData = 10000000; + // int nData = 8508; // Open Env ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database - ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, tKeyCmpr, pEnv, &pDb); + compFunc = tKeyCmpr; + ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -104,4 +108,24 @@ static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) } else { return 0; } +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; } \ No newline at end of file From 0b12117fb263c35179fb85942183e3cad431e313 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 26 Mar 2022 02:53:38 +0000 Subject: [PATCH 91/92] fix a test bug --- source/libs/tdb/test/tdbTest.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index c256f07cf6..52bf26f0ef 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -2,6 +2,8 @@ #include "tdbInt.h" +#include + static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); @@ -11,7 +13,6 @@ TEST(tdb_test, simple_test) { STDB *pDb; FKeyComparator compFunc; int nData = 10000000; - // int nData = 8508; // Open Env ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv); @@ -98,8 +99,10 @@ TEST(tdb_test, simple_test) { static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { int k1, k2; - k1 = std::strtol((char *)pKey1 + 3, nullptr, 10); - k2 = std::strtol((char *)pKey2 + 3, nullptr, 10); + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); if (k1 < k2) { return -1; From dbec5f1fa4044c4198452b6dcf32b0d0e843ba27 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 26 Mar 2022 09:29:49 +0000 Subject: [PATCH 92/92] fix logical bug with memory leak --- source/libs/tdb/src/db/tdbBtree.c | 12 +-- source/libs/tdb/src/page/tdbPage.c | 6 +- source/libs/tdb/test/tdbTest.cpp | 145 +++++++++++++++++++++-------- 3 files changed, 117 insertions(+), 46 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 45706ae3dc..5980c2b531 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -746,6 +746,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { pgno = TDB_PAGE_PGNO(pNews[iNew]); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); + free(pNewCell); } // move to next new page @@ -795,12 +796,11 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - // TODO: here has memory leak - // for (int i = 0; i < 3; i++) { - // if (pDivCell[i]) { - // free(pDivCell[i]); - // } - // } + for (int i = 0; i < 3; i++) { + if (pDivCell[i]) { + free(pDivCell[i]); + } + } return 0; } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 15b6e218e6..516330e4e6 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -143,7 +143,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; } - pPage->apOvfl[iOvfl] = pCell; + // TODO: here has memory leak + pNewCell = (SCell *)malloc(szCell); + memcpy(pNewCell, pCell, szCell); + + pPage->apOvfl[iOvfl] = pNewCell; pPage->aiOvfl[iOvfl] = idx; pPage->nOverflow++; iOvfl++; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 52bf26f0ef..63341e5430 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -4,8 +4,112 @@ #include -static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); -static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)malloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void closePool(SPoolMem *pPool) { + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + free(pMem); + } while (1); + + assert(pPool->size == 0); + + free(pPool); +} + +static void *poolMalloc(void *arg, int size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)malloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + free(pMem); +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} TEST(tdb_test, simple_test) { int ret; @@ -94,41 +198,4 @@ TEST(tdb_test, simple_test) { // Close Env ret = tdbEnvClose(pEnv); GTEST_ASSERT_EQ(ret, 0); -} - -static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { - int k1, k2; - - std::string s1((char *)pKey1 + 3, kLen1 - 3); - std::string s2((char *)pKey2 + 3, kLen2 - 3); - k1 = stoi(s1); - k2 = stoi(s2); - - if (k1 < k2) { - return -1; - } else if (k1 > k2) { - return 1; - } else { - return 0; - } -} - -static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { - int mlen; - int cret; - - ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); - - mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; - cret = memcmp(pKey1, pKey2, mlen); - if (cret == 0) { - if (keyLen1 < keyLen2) { - cret = -1; - } else if (keyLen1 > keyLen2) { - cret = 1; - } else { - cret = 0; - } - } - return cret; } \ No newline at end of file