From d3446a5c7aa1daeabba0bf3959d7ad9bc8da46c9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 09:32:21 +0000 Subject: [PATCH 001/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 2 +- source/libs/tdb/src/db/tdbPage.c | 253 ---------------------------- source/libs/tdb/src/inc/tdbPage.h | 26 +-- source/libs/tdb/src/page/tdbPage.c | 244 +++++++++++++++++---------- source/libs/tdb/src/page/tdbPageL.c | 45 +++-- 5 files changed, 198 insertions(+), 372 deletions(-) delete mode 100644 source/libs/tdb/src/db/tdbPage.c diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c6e8c9dca9..423de0b53c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -414,7 +414,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { } // TODO: need to update the SPage.nFree - pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; + // pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; pPage->nOverflow = 0; return 0; diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c deleted file mode 100644 index df158de756..0000000000 --- a/source/libs/tdb/src/db/tdbPage.c +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -typedef struct __attribute__((__packed__)) { - u8 szCell[2]; - u8 nxOffset[2]; -} SFreeCell; - -typedef struct __attribute__((__packed__)) { - u8 szCell[3]; - u8 nxOffset[3]; -} SFreeCellL; - -/* For small page */ -#define TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCell *)(PCELL))->szCell) -#define TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCell *)(PCELL))->nxOffset) - -#define TDB_SPAGE_FREE_CELL_SIZE(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL))[0] -#define TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))[0] - -#define TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) (TDB_SPAGE_FREE_CELL_SIZE(PCELL) = (SIZE)) -#define TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) (TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) = (OFFSET)) - -/* For large page */ -#define TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCellL *)(PCELL))->szCell) -#define TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCellL *)(PCELL))->nxOffset) - -#define TDB_LPAGE_FREE_CELL_SIZE(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL)) -#define TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) - -#define TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL), SIZE) -#define TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL), OFFSET) - -/* For page */ -#define TDB_PAGE_FREE_CELL_SIZE_PTR(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) : TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL)) -#define TDB_PAGE_FREE_CELL_NXOFFSET_PTR(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) - -#define TDB_PAGE_FREE_CELL_SIZE(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE(PCELL) : TDB_SPAGE_FREE_CELL_SIZE(PCELL)) -#define TDB_PAGE_FREE_CELL_NXOFFSET(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL)) - -#define TDB_PAGE_FREE_CELL_SIZE_SET(PPAGE, PCELL, SIZE) \ - do { \ - if (TDB_IS_LARGE_PAGE(PPAGE)) { \ - TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ - } else { \ - TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ - } \ - } while (0) -#define TDB_PAGE_FREE_CELL_NXOFFSET_SET(PPAGE, PCELL, OFFSET) \ - do { \ - if (TDB_IS_LARGE_PAGE(PPAGE)) { \ - TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ - } else { \ - TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ - } \ - } while (0) - -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); -static int tdbPageDefragment(SPage *pPage); - -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { - SPage *pPage; - u8 *ptr; - int size; - - ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); - - *ppPage = NULL; - size = pageSize + sizeof(*pPage); - - ptr = (u8 *)((*xMalloc)(arg, size)); - if (pPage == NULL) { - return -1; - } - - memset(ptr, 0, size); - pPage = (SPage *)(ptr + pageSize); - - pPage->pData = ptr; - pPage->pageSize = pageSize; - if (pageSize < 65536) { - pPage->szOffset = 2; - pPage->szPageHdr = sizeof(SPageHdr); - pPage->szFreeCell = sizeof(SFreeCell); - } else { - pPage->szOffset = 3; - pPage->szPageHdr = sizeof(SPageHdrL); - pPage->szFreeCell = sizeof(SFreeCellL); - } - TDB_INIT_PAGE_LOCK(pPage); - - /* TODO */ - - *ppPage = pPage; - return 0; -} - -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { - u8 *ptr; - - ptr = pPage->pData; - (*xFree)(arg, ptr); - - return 0; -} - -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { - int ret; - SCell *pTarget; - u8 *pTmp; - int j; - - if (pPage->nOverflow || szCell + pPage->szOffset > pPage->nFree) { - // TODO: need to figure out if pCell may be used by outside of this function - j = pPage->nOverflow++; - - pPage->apOvfl[j] = pCell; - pPage->aiOvfl[j] = idx; - } else { - ret = tdbPageAllocate(pPage, szCell, &pTarget); - if (ret < 0) { - return -1; - } - - memcpy(pTarget, pCell, szCell); - pTmp = pPage->pCellIdx + idx * pPage->szOffset; - memmove(pTmp + pPage->szOffset, pTmp, pPage->pFreeStart - pTmp - pPage->szOffset); - TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); - TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); - } - - return 0; -} - -int tdbPageDropCell(SPage *pPage, int idx) { - // TODO - return 0; -} - -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { - SCell *pCell; - SFreeCell *pFreeCell; - u8 *pOffset; - int ret; - - ASSERT(pPage->nFree > size + pPage->szOffset); - - pCell = NULL; - *ppCell = NULL; - - // 1. Try to allocate from the free space area - if (pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset) { - pPage->pFreeEnd -= size; - pPage->pFreeStart += pPage->szOffset; - pCell = pPage->pFreeEnd; - } - - // 2. Try to allocate from the page free list - if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= pPage->szOffset) && TDB_PAGE_FCELL(pPage)) { - int szCell; - int nxOffset; - - pCell = pPage->pData + TDB_PAGE_FCELL(pPage); - pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell - : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - - for (;;) { - // Find a cell - if (szCell >= size) { - if (szCell - size >= pPage->szFreeCell) { - SCell *pTmpCell = pCell + size; - - TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); - TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); - // TODO: *pOffset = pTmpCell - pPage->pData; - } else { - TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); - // TODO: *pOffset = nxOffset; - } - break; - } - - // Not find a cell yet - if (nxOffset > 0) { - pCell = pPage->pData + nxOffset; - pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - continue; - } else { - pCell = NULL; - break; - } - } - - if (pCell) { - pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; - } - } - - // 3. Try to dfragment and allocate again - if (pCell == NULL) { - ret = tdbPageDefragment(pPage); - if (ret < 0) { - return -1; - } - - ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset); - ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); - - // Allocate from the free space area again - pPage->pFreeEnd -= size; - pPage->pFreeStart += pPage->szOffset; - pCell = pPage->pFreeEnd; - } - - ASSERT(pCell != NULL); - - pPage->nFree = pPage->nFree - size - pPage->szOffset; - *ppCell = pCell; - return 0; -} - -static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { - // TODO - return 0; -} - -static int tdbPageDefragment(SPage *pPage) { - // TODO - ASSERT(0); - return 0; -} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 6479574abe..418a1b6626 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -45,6 +45,9 @@ typedef struct { // cell offset at idx int (*getCellOffset)(SPage *, int); void (*setCellOffset)(SPage *, int, int); + // free cell info + void (*getFreeCellInfo)(SCell *pCell, int *szCell, int *nxOffset); + void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset); } SPageMethods; // Page footer @@ -59,20 +62,19 @@ struct SPage { SPageMethods *pPageMethods; // Fields below used by pager and am u8 szAmHdr; - u8 *pPageHdr; u8 *pAmHdr; + u8 *pPageHdr; u8 *pCellIdx; u8 *pFreeStart; u8 *pFreeEnd; SPageFtr *pPageFtr; - int kLen; // key length of the page, -1 for unknown - int vLen; // value length of the page, -1 for unknown - int nFree; - int maxLocal; - int minLocal; int nOverflow; SCell *apOvfl[4]; int aiOvfl[4]; + int kLen; // key length of the page, -1 for unknown + int vLen; // value length of the page, -1 for unknown + int maxLocal; + int minLocal; // Fields used by SPCache TDB_PCACHE_PAGE }; @@ -92,8 +94,6 @@ struct SPage { #define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) #define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) - #define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) // For page lock @@ -119,10 +119,12 @@ struct SPage { }) // APIs -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); -int tdbPageDropCell(SPage *pPage, int idx); +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); +void tdbPageZero(SPage *pPage); +void tdbPageInit(SPage *pPage); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); +int tdbPageDropCell(SPage *pPage, int idx); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 4ec3a895e7..e198891338 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -18,10 +18,11 @@ extern SPageMethods pageMethods; extern SPageMethods pageLargeMethods; -typedef struct __attribute__((__packed__)) { - u16 szCell; - u16 nxOffset; -} SFreeCell; +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) +#define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) +#define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) +#define TDB_PAGE_MAX_FREE_BLOCK(pPage) \ + ((pPage)->pageSize - (pPage)->szAmHdr - TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); static int tdbPageDefragment(SPage *pPage); @@ -44,6 +45,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) memset(ptr, 0, size); pPage = (SPage *)(ptr + pageSize); + TDB_INIT_PAGE_LOCK(pPage); pPage->pData = ptr; pPage->pageSize = pageSize; if (pageSize < 65536) { @@ -51,9 +53,6 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) } else { pPage->pPageMethods = &pageLargeMethods; } - TDB_INIT_PAGE_LOCK(pPage); - - /* TODO */ *ppPage = pPage; return 0; @@ -68,29 +67,64 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) return 0; } +void tdbPageZero(SPage *pPage) { + TDB_PAGE_NCELLS_SET(pPage, 0); + TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); + TDB_PAGE_FCELL_SET(pPage, 0); + TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_MAX_FREE_BLOCK(pPage)); + tdbPageInit(pPage); +} + +void tdbPageInit(SPage *pPage) { + pPage->pAmHdr = pPage->pData; + pPage->pPageHdr = pPage->pAmHdr + pPage->szAmHdr; + pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); + pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); + pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); + pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + pPage->nOverflow = 0; + + ASSERT(pPage->pFreeEnd >= pPage->pFreeStart); + ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); +} + int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { + int nFree; int ret; - SCell *pTarget; - u8 *pTmp; - int j; + int nCells; + int lidx; // local idx + SCell *pNewCell; - if (pPage->nOverflow || szCell + TDB_PAGE_OFFSET_SIZE(pPage) > pPage->nFree) { - // TODO: need to figure out if pCell may be used by outside of this function - j = pPage->nOverflow++; + ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage)); - pPage->apOvfl[j] = pCell; - pPage->aiOvfl[j] = idx; - } else { - ret = tdbPageAllocate(pPage, szCell, &pTarget); - if (ret < 0) { - return -1; + nFree = TDB_PAGE_NFREE(pPage); + nCells = TDB_PAGE_NCELLS(pPage); + + if (nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + // page must has enough space to hold the cell locally + ret = tdbPageAllocate(pPage, szCell, &pNewCell); + ASSERT(ret == 0); + + memcpy(pNewCell, pCell, szCell); + + if (pPage->nOverflow == 0) { + lidx = idx; + } else { + // TODO + // lidx = ; } - memcpy(pTarget, pCell, szCell); - pTmp = pPage->pCellIdx + idx * TDB_PAGE_OFFSET_SIZE(pPage); - memmove(pTmp + TDB_PAGE_OFFSET_SIZE(pPage), pTmp, pPage->pFreeStart - pTmp - TDB_PAGE_OFFSET_SIZE(pPage)); - TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); - TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); + // no overflow cell exists in this page + u8 *src = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * lidx; + u8 *dest = src + TDB_PAGE_OFFSET_SIZE(pPage); + memmove(dest, src, pPage->pFreeStart - dest); + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, TDB_PAGE_OFFSET_SIZE(pPage) * lidx); + TDB_PAGE_NCELLS_SET(pPage, nCells + 1); + } else { + // TODO: page not has enough space + pPage->apOvfl[pPage->nOverflow] = pCell; + pPage->aiOvfl[pPage->nOverflow] = idx; + pPage->nOverflow++; } return 0; @@ -101,73 +135,80 @@ int tdbPageDropCell(SPage *pPage, int idx) { return 0; } -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { - SCell *pCell; - SFreeCell *pFreeCell; - u8 *pOffset; - int ret; +static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { + SCell *pFreeCell; + u8 *pOffset; + int nFree; + int ret; + int cellFree; - ASSERT(pPage->nFree > size + TDB_PAGE_OFFSET_SIZE(pPage)); - - pCell = NULL; *ppCell = NULL; + nFree = TDB_PAGE_NFREE(pPage); - // 1. Try to allocate from the free space area - if (pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)) { - pPage->pFreeEnd -= size; + ASSERT(nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)); + ASSERT(TDB_PAGE_CCELLS(pPage) == pPage->pFreeEnd - pPage->pData); + + // 1. Try to allocate from the free space block area + if (pPage->pFreeEnd - pPage->pFreeStart >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); - pCell = pPage->pFreeEnd; + pPage->pFreeEnd -= szCell; + + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); + + *ppCell = pPage->pFreeEnd; + return 0; } // 2. Try to allocate from the page free list - if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) && - TDB_PAGE_FCELL(pPage)) { -#if 0 - int szCell; - int nxOffset; - - pCell = pPage->pData + TDB_PAGE_FCELL(pPage); - pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell - : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + cellFree = TDB_PAGE_FCELL(pPage); + ASSERT(cellFree == 0 || cellFree > pPage->pFreeEnd - pPage->pData); + if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { + SCell *pPrevFreeCell = NULL; + int szPrevFreeCell; + int szFreeCell; + int nxFreeCell; + int newSize; for (;;) { - // Find a cell - if (szCell >= size) { - if (szCell - size >= pPage->szFreeCell) { - SCell *pTmpCell = pCell + size; + if (cellFree == 0) break; - TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); - TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); - // TODO: *pOffset = pTmpCell - pPage->pData; + pFreeCell = pPage->pData + cellFree; + pPage->pPageMethods->getFreeCellInfo(pFreeCell, &szFreeCell, &nxFreeCell); + + if (szFreeCell >= szCell) { + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + TDB_PAGE_NFREE_SET(pPage, nFree - TDB_PAGE_OFFSET_SIZE(pPage) - szCell); + *ppCell = pFreeCell; + + newSize = szFreeCell - szCell; + pFreeCell += szCell; + if (newSize >= TDB_PAGE_FREE_CELL_SIZE(pPage)) { + pPage->pPageMethods->setFreeCellInfo(pFreeCell, newSize, nxFreeCell); + if (pPrevFreeCell) { + pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, pFreeCell - pPage->pData); + } else { + TDB_PAGE_FCELL_SET(pPage, pFreeCell - pPage->pData); + } + + return 0; } else { - TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); - // TODO: *pOffset = nxOffset; + if (pPrevFreeCell) { + pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, nxFreeCell); + } else { + TDB_PAGE_FCELL_SET(pPage, nxFreeCell); + } } - break; - } - - // Not find a cell yet - if (nxOffset > 0) { - pCell = pPage->pData + nxOffset; - pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - continue; } else { - pCell = NULL; - break; + pPrevFreeCell = pFreeCell; + szPrevFreeCell = szFreeCell; + cellFree = nxFreeCell; } } - - if (pCell) { - pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; - } -#endif } - // 3. Try to dfragment and allocate again +// 3. Try to dfragment and allocate again +#if 0 if (pCell == NULL) { ret = tdbPageDefragment(pPage); if (ret < 0) { @@ -175,7 +216,7 @@ static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { } ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)); - ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); + // ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); // Allocate from the free space area again pPage->pFreeEnd -= size; @@ -185,8 +226,9 @@ static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { ASSERT(pCell != NULL); - pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); + // pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); *ppCell = pCell; +#endif return 0; } @@ -210,6 +252,11 @@ typedef struct __attribute__((__packed__)) { u16 nFree; } SPageHdr; +typedef struct __attribute__((__packed__)) { + u16 szCell; + u16 nxOffset; +} SFreeCell; + // flags static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } @@ -253,20 +300,35 @@ static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { ((u16 *)pPage->pCellIdx)[idx] = (u16)offset; } +// free cell info +static inline void getPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { + SFreeCell *pFreeCell = (SFreeCell *)pCell; + *szCell = pFreeCell->szCell; + *nxOffset = pFreeCell->nxOffset; +} + +static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { + SFreeCell *pFreeCell = (SFreeCell *)pCell; + pFreeCell->szCell = szCell; + pFreeCell->nxOffset = nxOffset; +} + SPageMethods pageMethods = { - 2, // szOffset - sizeof(SPageHdr), // szPageHdr - sizeof(SFreeCell), // szFreeCell - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp - getPageCellNum, // getCellNum - setPageCellNum, // setCellNum - getPageCellBody, // getCellBody - setPageCellBody, // setCellBody - getPageCellFree, // getCellFree - setPageCellFree, // setCellFree - getPageNFree, // getFreeBytes - setPageNFree, // setFreeBytes - getPageCellOffset, // getCellOffset - setPageCellOffset // setCellOffset + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell + getPageFlags, // getPageFlags + setPageFlags, // setFlagsp + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset, // setCellOffset + getPageFreeCellInfo, // getFreeCellInfo + setPageFreeCellInfo // setFreeCellInfo }; \ No newline at end of file diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index e7c60118d2..e3c1bb8d37 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -66,20 +66,35 @@ static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset); } +// free cell info +static inline void getPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + *szCell = TDB_GET_U24(pFreeCell->szCell); + *nxOffset = TDB_GET_U24(pFreeCell->nxOffset); +} + +static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + TDB_PUT_U24(pFreeCell->szCell, szCell); + TDB_PUT_U24(pFreeCell->nxOffset, nxOffset); +} + SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp - getPageCellNum, // getCellNum - setPageCellNum, // setCellNum - getPageCellBody, // getCellBody - setPageCellBody, // setCellBody - getPageCellFree, // getCellFree - setPageCellFree, // setCellFree - getPageNFree, // getFreeBytes - setPageNFree, // setFreeBytes - getPageCellOffset, // getCellOffset - setPageCellOffset // setCellOffset + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell + getPageFlags, // getPageFlags + setPageFlags, // setFlagsp + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset, // setCellOffset + getPageFreeCellInfo, // getFreeCellInfo + setPageFreeCellInfo // setFreeCellInfo }; \ No newline at end of file From 57a92869ae28c7ce6f2776e521edcb879f9a4c77 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 09:45:49 +0000 Subject: [PATCH 002/105] more TDB --- source/libs/tdb/src/page/tdbPage.c | 46 +++++++++++------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index e198891338..892d7ecb31 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -141,6 +141,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { int nFree; int ret; int cellFree; + SCell *pCell = NULL; *ppCell = NULL; nFree = TDB_PAGE_NFREE(pPage); @@ -150,14 +151,10 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { // 1. Try to allocate from the free space block area if (pPage->pFreeEnd - pPage->pFreeStart >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { - pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); pPage->pFreeEnd -= szCell; - + pCell = pPage->pFreeEnd; TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); - TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); - - *ppCell = pPage->pFreeEnd; - return 0; + goto _alloc_finish; } // 2. Try to allocate from the page free list @@ -177,9 +174,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { pPage->pPageMethods->getFreeCellInfo(pFreeCell, &szFreeCell, &nxFreeCell); if (szFreeCell >= szCell) { - pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); - TDB_PAGE_NFREE_SET(pPage, nFree - TDB_PAGE_OFFSET_SIZE(pPage) - szCell); - *ppCell = pFreeCell; + pCell = pFreeCell; newSize = szFreeCell - szCell; pFreeCell += szCell; @@ -191,7 +186,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { TDB_PAGE_FCELL_SET(pPage, pFreeCell - pPage->pData); } - return 0; + goto _alloc_finish; } else { if (pPrevFreeCell) { pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, nxFreeCell); @@ -207,28 +202,21 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { } } -// 3. Try to dfragment and allocate again -#if 0 - if (pCell == NULL) { - ret = tdbPageDefragment(pPage); - if (ret < 0) { - return -1; - } + // 3. Try to dfragment and allocate again + tdbPageDefragment(pPage); + ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + ASSERT(nFree == TDB_PAGE_NFREE(pPage)); + ASSERT(pPage->pFreeEnd - pPage->pData == TDB_PAGE_CCELLS(pPage)); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)); - // ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); + pPage->pFreeEnd -= szCell; + pCell = pPage->pFreeEnd; + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); - // Allocate from the free space area again - pPage->pFreeEnd -= size; - pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); - pCell = pPage->pFreeEnd; - } - - ASSERT(pCell != NULL); - - // pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); +_alloc_finish: + ASSERT(pCell); + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); *ppCell = pCell; -#endif return 0; } From 0353e590814cfec20350627f07e5e11e3bcf6c67 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 12:02:26 +0000 Subject: [PATCH 003/105] more TDB --- source/libs/tdb/src/page/tdbPage.c | 34 ++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 892d7ecb31..1c9984cd01 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -220,8 +220,38 @@ _alloc_finish: return 0; } -static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { - // TODO +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { + int nFree; + int cellFree; + u8 *dest; + u8 *src; + + ASSERT(pCell >= pPage->pFreeEnd); + ASSERT(pCell + szCell <= (u8 *)(pPage->pPageFtr)); + ASSERT(pCell == TDB_PAGE_CELL_AT(pPage, idx)); + + nFree = TDB_PAGE_NFREE(pPage); + + if (pCell == pPage->pFreeEnd) { + pPage->pFreeEnd += szCell; + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + } else { + if (szCell >= TDB_PAGE_FREE_CELL_SIZE(pPage)) { + cellFree = TDB_PAGE_FCELL(pPage); + pPage->pPageMethods->setFreeCellInfo(pCell, szCell, cellFree); + TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); + } else { + ASSERT(0); + } + } + + dest = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * idx; + src = dest + TDB_PAGE_OFFSET_SIZE(pPage); + memmove(dest, src, pPage->pFreeStart - src); + + pPage->pFreeStart -= TDB_PAGE_OFFSET_SIZE(pPage); + nFree = nFree + szCell + TDB_PAGE_OFFSET_SIZE(pPage); + TDB_PAGE_NFREE_SET(pPage, nFree); return 0; } From dd7344c7800d88c6f444031319965130824cc7b7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 12:41:15 +0000 Subject: [PATCH 004/105] more TDB --- source/libs/tdb/src/inc/tdbPage.h | 1 + source/libs/tdb/src/page/tdbPage.c | 58 ++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 418a1b6626..9c40e9d5c1 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -75,6 +75,7 @@ struct SPage { int vLen; // value length of the page, -1 for unknown int maxLocal; int minLocal; + int (*xCellSize)(SCell *pCell); // Fields used by SPCache TDB_PCACHE_PAGE }; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 1c9984cd01..b10cd7b0ec 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -256,8 +256,62 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { } static int tdbPageDefragment(SPage *pPage) { - // TODO - ASSERT(0); + int nFree; + int nCells; + SCell *pCell; + SCell *pNextCell; + SCell *pTCell; + int szCell; + int idx; + int iCell; + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart < nFree); + + nFree = TDB_PAGE_NFREE(pPage); + nCells = TDB_PAGE_NCELLS(pPage); + + // Loop to compact the page content + // Here we use an O(n^2) algorithm to do the job since + // this is a low frequency job. + pNextCell = (u8 *)pPage->pPageFtr; + pCell = NULL; + for (iCell = 0;; iCell++) { + // compact over + if (iCell == nCells) { + pPage->pFreeEnd = pNextCell; + break; + } + + for (int i = 0; i < nCells; i++) { + if (TDB_PAGE_CELL_OFFSET_AT(pPage, i) < pNextCell - pPage->pData) { + pTCell = TDB_PAGE_CELL_AT(pPage, i); + if (pCell == NULL || pCell < pTCell) { + pCell = pTCell; + idx = i; + } + } else { + continue; + } + } + + ASSERT(pCell != NULL); + + szCell = (*pPage->xCellSize)(pCell); + + ASSERT(pCell + szCell <= pNextCell); + if (pCell + szCell < pNextCell) { + memmove(pNextCell - szCell, pCell, szCell); + } + + pCell = NULL; + pNextCell = pNextCell - szCell; + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pNextCell - pPage->pData); + } + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + TDB_PAGE_FCELL_SET(pPage, 0); + return 0; } From f91124ef8edc8f2ae300270b46798a9885f7b81b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 13:00:51 +0000 Subject: [PATCH 005/105] more TDB --- source/libs/tdb/src/page/tdbPage.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index b10cd7b0ec..9d679118e7 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -26,6 +26,7 @@ extern SPageMethods pageLargeMethods; static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); static int tdbPageDefragment(SPage *pPage); +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { SPage *pPage; @@ -131,7 +132,25 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { } int tdbPageDropCell(SPage *pPage, int idx) { - // TODO + int lidx; + SCell *pCell; + int szCell; + int nCells; + + nCells = TDB_PAGE_NCELLS(pPage); + + if (pPage->nOverflow == 0) { + lidx = idx; + } else { + // TODO + } + + pCell = TDB_PAGE_CELL_AT(pPage, lidx); + szCell = (*pPage->xCellSize)(pCell); + tdbPageFree(pPage, lidx, pCell, szCell); + + TDB_PAGE_NCELLS_SET(pPage, nCells - 1); + return 0; } From 6eb2e1b288d1363abf998b1604978a64340ad20a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 13:13:08 +0000 Subject: [PATCH 006/105] more TDB --- source/libs/tdb/src/page/tdbPage.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 9d679118e7..d91a32cb7c 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -136,19 +136,26 @@ int tdbPageDropCell(SPage *pPage, int idx) { SCell *pCell; int szCell; int nCells; + int iOvfl; nCells = TDB_PAGE_NCELLS(pPage); - if (pPage->nOverflow == 0) { - lidx = idx; - } else { - // TODO + ASSERT(idx >= 0 && idx < nCells + pPage->nOverflow); + + iOvfl = 0; + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] == idx) { + // TODO: remove the over flow cell + return 0; + } else if (pPage->aiOvfl[iOvfl] > idx) { + break; + } } + lidx = idx - iOvfl; pCell = TDB_PAGE_CELL_AT(pPage, lidx); szCell = (*pPage->xCellSize)(pCell); tdbPageFree(pPage, lidx, pCell, szCell); - TDB_PAGE_NCELLS_SET(pPage, nCells - 1); return 0; From 7d5030bc7fc2576411698f718f50fbaf7ff697ec Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 15:04:29 +0000 Subject: [PATCH 007/105] more TDB --- source/libs/tdb/src/page/tdbPage.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index d91a32cb7c..2c9563d36d 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -145,7 +145,13 @@ int tdbPageDropCell(SPage *pPage, int idx) { iOvfl = 0; for (; iOvfl < pPage->nOverflow; iOvfl++) { if (pPage->aiOvfl[iOvfl] == idx) { - // TODO: remove the over flow cell + // remove the over flow cell + for (; (++iOvfl) < pPage->nOverflow;) { + pPage->aiOvfl[iOvfl - 1] = pPage->aiOvfl[iOvfl] - 1; + pPage->apOvfl[iOvfl - 1] = pPage->apOvfl[iOvfl]; + } + + pPage->nOverflow--; return 0; } else if (pPage->aiOvfl[iOvfl] > idx) { break; @@ -158,6 +164,11 @@ int tdbPageDropCell(SPage *pPage, int idx) { tdbPageFree(pPage, lidx, pCell, szCell); TDB_PAGE_NCELLS_SET(pPage, nCells - 1); + for (; iOvfl < pPage->nOverflow; iOvfl++) { + pPage->aiOvfl[iOvfl]--; + ASSERT(pPage->aiOvfl[iOvfl] > 0); + } + return 0; } From 7a5985ae46de83a3fbe359aa5d79e21e54b4a8de Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 15:39:51 +0000 Subject: [PATCH 008/105] more TDB --- source/libs/tdb/src/page/tdbPage.c | 39 ++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 2c9563d36d..4673a561e6 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -91,8 +91,8 @@ void tdbPageInit(SPage *pPage) { int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { int nFree; - int ret; int nCells; + int iOvfl; int lidx; // local idx SCell *pNewCell; @@ -100,32 +100,45 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); + iOvfl = 0; + + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] >= idx) { + break; + } + } + + lidx = idx - iOvfl; if (nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { // page must has enough space to hold the cell locally - ret = tdbPageAllocate(pPage, szCell, &pNewCell); - ASSERT(ret == 0); + tdbPageAllocate(pPage, szCell, &pNewCell); memcpy(pNewCell, pCell, szCell); - if (pPage->nOverflow == 0) { - lidx = idx; - } else { - // TODO - // lidx = ; - } - // no overflow cell exists in this page u8 *src = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * lidx; u8 *dest = src + TDB_PAGE_OFFSET_SIZE(pPage); memmove(dest, src, pPage->pFreeStart - dest); TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, TDB_PAGE_OFFSET_SIZE(pPage) * lidx); TDB_PAGE_NCELLS_SET(pPage, nCells + 1); + + ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); } else { - // TODO: page not has enough space - pPage->apOvfl[pPage->nOverflow] = pCell; - pPage->aiOvfl[pPage->nOverflow] = idx; + // add the cell as an overflow cell + for (int i = pPage->nOverflow; i > iOvfl; i--) { + pPage->apOvfl[i] = pPage->apOvfl[i - 1]; + pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; + } + + pPage->apOvfl[iOvfl] = pCell; + pPage->aiOvfl[iOvfl] = idx; pPage->nOverflow++; + iOvfl++; + } + + for (; iOvfl < pPage->nOverflow; iOvfl++) { + pPage->aiOvfl[iOvfl]++; } return 0; From 13b7ba3edb940b226754215ecd17af97025eb727 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 02:27:46 +0000 Subject: [PATCH 009/105] refact TDB --- source/libs/tdb/src/db/tdbBtree.c | 63 +++++++++++++++++------------- source/libs/tdb/src/inc/tdbPage.h | 44 +++++++++++++-------- source/libs/tdb/src/page/tdbPage.c | 20 ++++++++-- 3 files changed, 80 insertions(+), 47 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 423de0b53c..621afc9dfb 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -145,7 +145,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } if (pCur->idx == -1) { - ASSERT(TDB_PAGE_NCELLS(pCur->pPage) == 0); + ASSERT(TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0); idx = 0; } else { if (cret > 0) { @@ -218,7 +218,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p pCur->iPage = 0; - if (TDB_PAGE_NCELLS(pCur->pPage) == 0) { + if (TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0) { // Current page is empty ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); return 0; @@ -231,7 +231,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p SCellDecoder cd = {0}; pPage = pCur->pPage; - nCells = TDB_PAGE_NCELLS(pPage); + nCells = TDB_PAGE_TOTAL_CELLS(pPage); lidx = 0; ridx = nCells - 1; @@ -242,7 +242,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p midx = (lidx + ridx) >> 1; - pCell = TDB_PAGE_CELL_AT(pPage, midx); + pCell = tdbPageGetCell(pPage, midx); ret = tdbBtreeDecodeCell(pPage, pCell, &cd); if (ret < 0) { // TODO: handle error @@ -283,7 +283,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p } else { // TODO: reset cd as uninitialized pCur->idx = midx + 1; - pCell = TDB_PAGE_CELL_AT(pPage, midx + 1); + pCell = tdbPageGetCell(pPage, midx + 1); tdbBtreeDecodeCell(pPage, pCell, &cd); tdbBtCursorMoveToChild(pCur, cd.pgno); } @@ -391,16 +391,12 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { } else { pPage->szAmHdr = sizeof(SBtPageHdr); } - pPage->pPageHdr = pPage->pData; - pPage->pAmHdr = pPage->pPageHdr + pPage->pPageMethods->szPageHdr; - pPage->pCellIdx = pPage->pAmHdr + pPage->szAmHdr; - pPage->pFreeStart = pPage->pCellIdx + pPage->pPageMethods->szOffset * TDB_PAGE_NCELLS(pPage); - pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); - pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + pPage->xCellSize = NULL; // TODO + + tdbPageInit(pPage); TDB_BTREE_ASSERT_FLAG(flags); - // Init other fields if (isLeaf) { pPage->kLen = pBt->keyLen; pPage->vLen = pBt->valLen; @@ -413,30 +409,41 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { pPage->minLocal = pBt->minLocal; } - // TODO: need to update the SPage.nFree - // pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; - pPage->nOverflow = 0; - return 0; } static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u16 flags; SBTree *pBt; + u8 isLeaf; flags = ((SBtreeZeroPageArg *)arg)->flags; pBt = ((SBtreeZeroPageArg *)arg)->pBt; - - pPage->pPageHdr = pPage->pData; + isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Init the page header TDB_PAGE_FLAGS_SET(pPage, flags); - TDB_PAGE_NCELLS_SET(pPage, 0); - TDB_PAGE_CCELLS_SET(pPage, pBt->pageSize - sizeof(SPageFtr)); - TDB_PAGE_FCELL_SET(pPage, 0); - TDB_PAGE_NFREE_SET(pPage, 0); + // Set szAmHdr + if (isLeaf) { + pPage->szAmHdr = 0; + } else { + pPage->szAmHdr = sizeof(SBtPageHdr); + } + pPage->xCellSize = NULL; // TODO - tdbBtreeInitPage(pPage, (void *)pBt); + tdbPageZero(pPage); + + if (isLeaf) { + pPage->kLen = pBt->keyLen; + pPage->vLen = pBt->valLen; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + } else { + pPage->kLen = pBt->keyLen; + pPage->vLen = sizeof(SPgno); + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + } return 0; } @@ -453,7 +460,8 @@ typedef struct { } SBtreeBalanceHelper; static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { - int nCells = TDB_PAGE_NCELLS(pFrom); +#if 0 + int nCells = TDB_PAGE_TOTAL_CELLS(pFrom); int cCells = TDB_PAGE_CCELLS(pFrom); int fCell = TDB_PAGE_FCELL(pFrom); int nFree = TDB_PAGE_NFREE(pFrom); @@ -469,6 +477,7 @@ static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { TDB_PAGE_NFREE_SET(pTo, nFree); // TODO: update other fields +#endif return 0; } @@ -529,7 +538,7 @@ static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { SBTree *pBt; pParent = pBlh->pParent; - nCells = TDB_PAGE_NCELLS(pParent); + nCells = TDB_PAGE_TOTAL_CELLS(pParent); nChild = nCells + 1; pBt = pBlh->pBt; @@ -556,7 +565,7 @@ static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { if (idxStart + i == nCells) { pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; } else { - pCell = TDB_PAGE_CELL_AT(pParent, idxStart + i); + pCell = tdbPageGetCell(pParent, idxStart + i); // TODO: no need to decode the payload part, and even the kLen, vLen part // we only need the pgno part ret = tdbBtreeDecodeCell(pParent, pCell, &cd); @@ -610,7 +619,7 @@ static int tdbBtreeBalanceStep2(SBtreeBalanceHelper *pBlh) { if (cidx < limit) { // Get local cells - pCell = TDB_PAGE_CELL_AT(pPage, cidx); + pCell = tdbPageGetCell(pPage, cidx); } else if (cidx == limit) { // Get overflow cells pCell = pPage->apOvfl[oidx++]; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 9c40e9d5c1..82287b0921 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -57,8 +57,8 @@ typedef struct __attribute__((__packed__)) { struct SPage { pthread_spinlock_t lock; - u8 *pData; int pageSize; + u8 *pData; SPageMethods *pPageMethods; // Fields below used by pager and am u8 szAmHdr; @@ -81,21 +81,8 @@ struct SPage { }; /* For page */ -#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) -#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) -#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage) -#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage) -#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) -#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx) - -#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) -#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS) -#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS) -#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL) -#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) -#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) - -#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) +#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) +#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) // For page lock #define P_LOCK_SUCC 0 @@ -120,6 +107,8 @@ struct SPage { }) // APIs +#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) + int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); void tdbPageZero(SPage *pPage); @@ -127,6 +116,29 @@ void tdbPageInit(SPage *pPage); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); +static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { + SCell *pCell; + int iOvfl; + int lidx; + + ASSERT(idx >= 0 && idx < pPage->nOverflow + pPage->pPageMethods->getCellNum(pPage)); + + iOvfl = 0; + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] == idx) { + pCell = pPage->apOvfl[iOvfl]; + return pCell; + } else if (pPage->aiOvfl[iOvfl] > idx) { + break; + } + } + + lidx = idx - iOvfl; + pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); + + return pCell; +} + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 4673a561e6..2b0e573aae 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -18,9 +18,20 @@ extern SPageMethods pageMethods; extern SPageMethods pageLargeMethods; -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) -#define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) -#define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) +#define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) +#define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) +#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) +#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage) +#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage) +#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) +#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx) +#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS) +#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS) +#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL) +#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) +#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) +#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) #define TDB_PAGE_MAX_FREE_BLOCK(pPage) \ ((pPage)->pageSize - (pPage)->szAmHdr - TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) @@ -47,8 +58,8 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) pPage = (SPage *)(ptr + pageSize); TDB_INIT_PAGE_LOCK(pPage); - pPage->pData = ptr; pPage->pageSize = pageSize; + pPage->pData = ptr; if (pageSize < 65536) { pPage->pPageMethods = &pageMethods; } else { @@ -125,6 +136,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); } else { + // TODO: make it extensible // add the cell as an overflow cell for (int i = pPage->nOverflow; i > iOvfl; i--) { pPage->apOvfl[i] = pPage->apOvfl[i - 1]; From cde7a7c6a1558a8bb76ad3d7bd83fef44fc89172 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 02:54:00 +0000 Subject: [PATCH 010/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 14 ++++++++------ source/libs/tdb/src/inc/tdbPage.h | 6 +++--- source/libs/tdb/src/page/tdbPage.c | 24 ++++++++++++++++-------- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 621afc9dfb..0c83dd609c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -381,19 +381,20 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { SBTree *pBt; u16 flags; u8 isLeaf; + u8 szAmHdr; pBt = (SBTree *)arg; flags = TDB_PAGE_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { - pPage->szAmHdr = 0; + szAmHdr = 0; } else { - pPage->szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SBtPageHdr); } pPage->xCellSize = NULL; // TODO - tdbPageInit(pPage); + tdbPageInit(pPage, szAmHdr); TDB_BTREE_ASSERT_FLAG(flags); @@ -416,6 +417,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u16 flags; SBTree *pBt; u8 isLeaf; + u8 szAmHdr; flags = ((SBtreeZeroPageArg *)arg)->flags; pBt = ((SBtreeZeroPageArg *)arg)->pBt; @@ -425,13 +427,13 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { TDB_PAGE_FLAGS_SET(pPage, flags); // Set szAmHdr if (isLeaf) { - pPage->szAmHdr = 0; + szAmHdr = 0; } else { - pPage->szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SBtPageHdr); } pPage->xCellSize = NULL; // TODO - tdbPageZero(pPage); + tdbPageZero(pPage, szAmHdr); if (isLeaf) { pPage->kLen = pBt->keyLen; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 82287b0921..58ae1f3963 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -61,7 +61,6 @@ struct SPage { u8 *pData; SPageMethods *pPageMethods; // Fields below used by pager and am - u8 szAmHdr; u8 *pAmHdr; u8 *pPageHdr; u8 *pCellIdx; @@ -111,8 +110,8 @@ struct SPage { int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -void tdbPageZero(SPage *pPage); -void tdbPageInit(SPage *pPage); +void tdbPageZero(SPage *pPage, u8 szAmHdr); +void tdbPageInit(SPage *pPage, u8 szAmHdr); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); @@ -134,6 +133,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { } lidx = idx - iOvfl; + ASSERT(lidx >= 0 && lidx < pPage->pPageMethods->getCellNum(pPage)); pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); return pCell; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 2b0e573aae..11c9a9247e 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -32,8 +32,8 @@ extern SPageMethods pageLargeMethods; #define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) #define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) #define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) -#define TDB_PAGE_MAX_FREE_BLOCK(pPage) \ - ((pPage)->pageSize - (pPage)->szAmHdr - TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) +#define TDB_PAGE_MAX_FREE_BLOCK(pPage, szAmHdr) \ + ((pPage)->pageSize - (szAmHdr)-TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); static int tdbPageDefragment(SPage *pPage); @@ -79,17 +79,25 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) return 0; } -void tdbPageZero(SPage *pPage) { +void tdbPageZero(SPage *pPage, u8 szAmHdr) { + pPage->pAmHdr = pPage->pData; + pPage->pPageHdr = pPage->pAmHdr + szAmHdr; TDB_PAGE_NCELLS_SET(pPage, 0); TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); TDB_PAGE_FCELL_SET(pPage, 0); - TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_MAX_FREE_BLOCK(pPage)); - tdbPageInit(pPage); + TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_MAX_FREE_BLOCK(pPage, szAmHdr)); + pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); + pPage->pFreeStart = pPage->pCellIdx; + pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); + pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + pPage->nOverflow = 0; + + ASSERT((u8 *)pPage->pPageFtr == pPage->pFreeEnd); } -void tdbPageInit(SPage *pPage) { +void tdbPageInit(SPage *pPage, u8 szAmHdr) { pPage->pAmHdr = pPage->pData; - pPage->pPageHdr = pPage->pAmHdr + pPage->szAmHdr; + pPage->pPageHdr = pPage->pAmHdr + szAmHdr; pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); @@ -107,7 +115,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { int lidx; // local idx SCell *pNewCell; - ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage)); + ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pAmHdr)); nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); From 50aa291ff60089a8270f3b85f3e4bd84d0a2aae9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 03:29:39 +0000 Subject: [PATCH 011/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0c83dd609c..59c132ecb2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -423,9 +423,6 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { pBt = ((SBtreeZeroPageArg *)arg)->pBt; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); - // Init the page header - TDB_PAGE_FLAGS_SET(pPage, flags); - // Set szAmHdr if (isLeaf) { szAmHdr = 0; } else { @@ -433,6 +430,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { } pPage->xCellSize = NULL; // TODO + TDB_PAGE_FLAGS_SET(pPage, flags); tdbPageZero(pPage, szAmHdr); if (isLeaf) { From 03eaa648d68c2b9f11a54f780d88e13630809d99 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 03:42:18 +0000 Subject: [PATCH 012/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 59c132ecb2..52f0c7d466 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -385,6 +385,9 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { pBt = (SBTree *)arg; + ASSERT(0); + + // TODO: here has problem flags = TDB_PAGE_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { @@ -430,8 +433,8 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { } pPage->xCellSize = NULL; // TODO - TDB_PAGE_FLAGS_SET(pPage, flags); tdbPageZero(pPage, szAmHdr); + TDB_PAGE_FLAGS_SET(pPage, flags); if (isLeaf) { pPage->kLen = pBt->keyLen; From 4e8f8445de7f719dbef9a75321253df3d4b1fff5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 05:47:06 +0000 Subject: [PATCH 013/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 41 +++++++++++++++++------------ source/libs/tdb/src/inc/tdbPage.h | 4 +++ source/libs/tdb/src/page/tdbPage.c | 10 ++++--- source/libs/tdb/src/page/tdbPageL.c | 10 ++++--- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 52f0c7d466..4b2394f861 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -39,9 +39,16 @@ struct SBTree { u8 *pTmp; }; +#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; + typedef struct __attribute__((__packed__)) { - SPgno rChild; -} SBtPageHdr; + TDB_BTREE_PAGE_COMMON_HDR +} SLeafHdr; + +typedef struct __attribute__((__packed__)) { + TDB_BTREE_PAGE_COMMON_HDR; + SPgno pgno; // right-most child +} SIntHdr; typedef struct { u16 flags; @@ -220,7 +227,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p if (TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0) { // Current page is empty - ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); + // ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); return 0; } @@ -265,7 +272,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p } // Move downward or break - u16 flags = TDB_PAGE_FLAGS(pPage); + u16 flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (leaf) { pCur->idx = midx; @@ -279,7 +286,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p if (midx == nCells - 1) { /* Move to right-most child */ pCur->idx = midx + 1; - tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); + // tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); } else { // TODO: reset cd as uninitialized pCur->idx = midx + 1; @@ -388,12 +395,12 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { ASSERT(0); // TODO: here has problem - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { - szAmHdr = 0; + szAmHdr = sizeof(SLeafHdr); } else { - szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SIntHdr); } pPage->xCellSize = NULL; // TODO @@ -427,14 +434,14 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { - szAmHdr = 0; + szAmHdr = sizeof(SLeafHdr); } else { - szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SIntHdr); } pPage->xCellSize = NULL; // TODO tdbPageZero(pPage, szAmHdr); - TDB_PAGE_FLAGS_SET(pPage, flags); + // TDB_PAGE_FLAGS_SET(pPage, flags); if (isLeaf) { pPage->kLen = pBt->keyLen; @@ -522,7 +529,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } - ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; + // TODO: ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; *ppChild = pChild; return 0; @@ -566,7 +573,7 @@ static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { i = pBlh->nOld - 1; if (idxStart + i == nCells) { - pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; + // pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; } else { pCell = tdbPageGetCell(pParent, idxStart + i); // TODO: no need to decode the payload part, and even the kLen, vLen part @@ -676,7 +683,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; SBtreeBalanceHelper blh; - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); + // ASSERT(!TDB_BTREE_PGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); blh.pBt = pBt; blh.pParent = pParent; @@ -748,7 +755,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { for (;;) { iPage = pCur->iPage; pPage = pCur->pPage; - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); root = TDB_BTREE_PAGE_IS_ROOT(flags); @@ -842,7 +849,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo nPayload = 0; nHeader = 0; - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // 1. Encode Header part @@ -911,7 +918,7 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD int ret; nHeader = 0; - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Clear the state of decoder diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 58ae1f3963..155e7b2e37 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -27,9 +27,11 @@ typedef struct { int szOffset; int szPageHdr; int szFreeCell; +#if 0 // flags u16 (*getFlags)(SPage *); void (*setFlags)(SPage *, u16); +#endif // cell number int (*getCellNum)(SPage *); void (*setCellNum)(SPage *, int); @@ -79,9 +81,11 @@ struct SPage { TDB_PCACHE_PAGE }; +#if 0 /* For page */ #define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) #define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) +#endif // For page lock #define P_LOCK_SUCC 0 diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 11c9a9247e..61793b6025 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -399,9 +399,11 @@ typedef struct __attribute__((__packed__)) { u16 nxOffset; } SFreeCell; +#if 0 // flags static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } +#endif // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } @@ -456,11 +458,13 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageMethods = { - 2, // szOffset - sizeof(SPageHdr), // szPageHdr - sizeof(SFreeCell), // szFreeCell + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell +#if 0 getPageFlags, // getPageFlags setPageFlags, // setFlagsp +#endif getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index e3c1bb8d37..f4bcc62494 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -28,9 +28,11 @@ typedef struct __attribute__((__packed__)) { u8 nxOffset[3]; } SFreeCellL; +#if 0 // flags static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdrL *)(pPage->pPageHdr))[0].flags; } static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdrL *)(pPage->pPageHdr))[0].flags = flags; } +#endif // cellNum static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } @@ -80,11 +82,13 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell +#if 0 getPageFlags, // getPageFlags setPageFlags, // setFlagsp +#endif getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody From df306127a644ebec5d8dc6ff37264ed8c4d9fed3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 05:56:33 +0000 Subject: [PATCH 014/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 3 ++- source/libs/tdb/src/inc/tdbPage.h | 11 ----------- source/libs/tdb/src/page/tdbPage.c | 16 +++------------- source/libs/tdb/src/page/tdbPageL.c | 16 +++------------- 4 files changed, 8 insertions(+), 38 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4b2394f861..68319da3b2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -39,7 +39,8 @@ struct SBTree { u8 *pTmp; }; -#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; +#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; +#define TDB_BTREE_PAGE_FLAGS(PAGE) (PAGE)->pAmHdr[0] typedef struct __attribute__((__packed__)) { TDB_BTREE_PAGE_COMMON_HDR diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 155e7b2e37..dc3b803708 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -27,11 +27,6 @@ typedef struct { int szOffset; int szPageHdr; int szFreeCell; -#if 0 - // flags - u16 (*getFlags)(SPage *); - void (*setFlags)(SPage *, u16); -#endif // cell number int (*getCellNum)(SPage *); void (*setCellNum)(SPage *, int); @@ -81,12 +76,6 @@ struct SPage { TDB_PCACHE_PAGE }; -#if 0 -/* For page */ -#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) -#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) -#endif - // For page lock #define P_LOCK_SUCC 0 #define P_LOCK_BUSY 1 diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 61793b6025..b3a3bdbdcc 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -399,12 +399,6 @@ typedef struct __attribute__((__packed__)) { u16 nxOffset; } SFreeCell; -#if 0 -// flags -static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } -static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } -#endif - // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } static inline void setPageCellNum(SPage *pPage, int cellNum) { @@ -458,13 +452,9 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageMethods = { - 2, // szOffset - sizeof(SPageHdr), // szPageHdr - sizeof(SFreeCell), // szFreeCell -#if 0 - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp -#endif + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index f4bcc62494..cfd8c919df 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -28,12 +28,6 @@ typedef struct __attribute__((__packed__)) { u8 nxOffset[3]; } SFreeCellL; -#if 0 -// flags -static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdrL *)(pPage->pPageHdr))[0].flags; } -static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdrL *)(pPage->pPageHdr))[0].flags = flags; } -#endif - // cellNum static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } static inline void setPageCellNum(SPage *pPage, int cellNum) { @@ -82,13 +76,9 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell -#if 0 - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp -#endif + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody From 2b7da2a79b1972c35853419cb564e9e429f40609 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:05:32 +0000 Subject: [PATCH 015/105] more TDB --- source/libs/tdb/src/btree/tdbBtreeBalance.c | 14 +++++++++++ source/libs/tdb/src/btree/tdbBtreeCommon.c | 14 +++++++++++ source/libs/tdb/src/btree/tdbBtreeInt.h | 27 +++++++++++++++++++++ source/libs/tdb/src/btree/tdbBtreeOpen.c | 14 +++++++++++ 4 files changed, 69 insertions(+) create mode 100644 source/libs/tdb/src/btree/tdbBtreeBalance.c create mode 100644 source/libs/tdb/src/btree/tdbBtreeCommon.c create mode 100644 source/libs/tdb/src/btree/tdbBtreeInt.h create mode 100644 source/libs/tdb/src/btree/tdbBtreeOpen.c diff --git a/source/libs/tdb/src/btree/tdbBtreeBalance.c b/source/libs/tdb/src/btree/tdbBtreeBalance.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeBalance.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeCommon.c b/source/libs/tdb/src/btree/tdbBtreeCommon.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeCommon.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeInt.h b/source/libs/tdb/src/btree/tdbBtreeInt.h new file mode 100644 index 0000000000..b8a935a614 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeInt.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TDB_BTREE_INT_H_ +#define _TDB_BTREE_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TDB_BTREE_INT_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeOpen.c b/source/libs/tdb/src/btree/tdbBtreeOpen.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeOpen.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file From d9c4bcab6ccb19c9494587472c7ee6598e7cdd94 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:14:47 +0000 Subject: [PATCH 016/105] more TDB --- source/libs/tdb/CMakeLists.txt | 1 - source/libs/tdb/src/btree/tdbBtreeDelete.c | 14 ++++++++++++++ source/libs/tdb/src/btree/tdbBtreeInsert.c | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 source/libs/tdb/src/btree/tdbBtreeDelete.c create mode 100644 source/libs/tdb/src/btree/tdbBtreeInsert.c diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 978649499a..a9b56d42b8 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -8,7 +8,6 @@ target_sources(tdb "src/db/tdbBtree.c" "src/db/tdbDb.c" "src/db/tdbEnv.c" - # "src/db/tdbPage.c" "src/page/tdbPage.c" "src/page/tdbPageL.c" ) diff --git a/source/libs/tdb/src/btree/tdbBtreeDelete.c b/source/libs/tdb/src/btree/tdbBtreeDelete.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeDelete.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeInsert.c b/source/libs/tdb/src/btree/tdbBtreeInsert.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeInsert.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file From a84155b951b24d87b204a2a4009c9c19cbe6f2cc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:23:32 +0000 Subject: [PATCH 017/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 21 ++++++++++++++------- source/libs/tdb/src/inc/tdbPage.h | 2 +- source/libs/tdb/src/page/tdbPage.c | 4 ++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 68319da3b2..2a2d630cd8 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -52,9 +52,9 @@ typedef struct __attribute__((__packed__)) { } SIntHdr; typedef struct { - u16 flags; + u8 flags; SBTree *pBt; -} SBtreeZeroPageArg; +} SBtreeInitPageArg; typedef struct { int kLen; @@ -74,6 +74,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo int *szCell); static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); static int tdbBtreeBalance(SBtCursor *pCur); +static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; @@ -371,7 +372,7 @@ static int tdbBtreeOpenImpl(SBTree *pBt) { } // Try to create a new database - SBtreeZeroPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; + SBtreeInitPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg); if (ret < 0) { return -1; @@ -430,8 +431,8 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u8 isLeaf; u8 szAmHdr; - flags = ((SBtreeZeroPageArg *)arg)->flags; - pBt = ((SBtreeZeroPageArg *)arg)->pBt; + flags = ((SBtreeInitPageArg *)arg)->flags; + pBt = ((SBtreeInitPageArg *)arg)->pBt; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { @@ -498,7 +499,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPage *pChild; SPgno pgnoChild; int ret; - SBtreeZeroPageArg zArg; + SBtreeInitPageArg zArg; pPager = pRoot->pPager; @@ -959,4 +960,10 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD return 0; } -#endif \ No newline at end of file +#endif + +static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { + // TODO + ASSERT(0); + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index dc3b803708..0960d88c33 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -71,7 +71,7 @@ struct SPage { int vLen; // value length of the page, -1 for unknown int maxLocal; int minLocal; - int (*xCellSize)(SCell *pCell); + int (*xCellSize)(const SPage *, SCell *); // Fields used by SPCache TDB_PCACHE_PAGE }; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index b3a3bdbdcc..7088904285 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -193,7 +193,7 @@ int tdbPageDropCell(SPage *pPage, int idx) { lidx = idx - iOvfl; pCell = TDB_PAGE_CELL_AT(pPage, lidx); - szCell = (*pPage->xCellSize)(pCell); + szCell = (*pPage->xCellSize)(pPage, pCell); tdbPageFree(pPage, lidx, pCell, szCell); TDB_PAGE_NCELLS_SET(pPage, nCells - 1); @@ -366,7 +366,7 @@ static int tdbPageDefragment(SPage *pPage) { ASSERT(pCell != NULL); - szCell = (*pPage->xCellSize)(pCell); + szCell = (*pPage->xCellSize)(pPage, pCell); ASSERT(pCell + szCell <= pNextCell); if (pCell + szCell < pNextCell) { From 0a6be10ff1d1f5b35c0ffceb095485347f774194 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:34:34 +0000 Subject: [PATCH 018/105] more tdb --- source/libs/tdb/src/db/tdbBtree.c | 44 ++++++++++++++---------------- source/libs/tdb/src/inc/tdbPage.h | 4 +-- source/libs/tdb/src/page/tdbPage.c | 6 ++-- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 2a2d630cd8..06f96e5498 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -39,8 +39,10 @@ struct SBTree { u8 *pTmp; }; -#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; -#define TDB_BTREE_PAGE_FLAGS(PAGE) (PAGE)->pAmHdr[0] +#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; + +#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pAmHdr[0] +#define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pAmHdr[0] = (flags)) typedef struct __attribute__((__packed__)) { TDB_BTREE_PAGE_COMMON_HDR @@ -274,8 +276,8 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p } // Move downward or break - u16 flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); - u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + u8 flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (leaf) { pCur->idx = midx; *pCRst = c; @@ -388,25 +390,23 @@ static int tdbBtreeOpenImpl(SBTree *pBt) { static int tdbBtreeInitPage(SPage *pPage, void *arg) { SBTree *pBt; - u16 flags; + u8 flags; u8 isLeaf; u8 szAmHdr; - pBt = (SBTree *)arg; - - ASSERT(0); - - // TODO: here has problem - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + pBt = ((SBtreeInitPageArg *)arg)->pBt; + flags = ((SBtreeInitPageArg *)arg)->flags; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + + ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); + if (isLeaf) { szAmHdr = sizeof(SLeafHdr); } else { szAmHdr = sizeof(SIntHdr); } - pPage->xCellSize = NULL; // TODO - tdbPageInit(pPage, szAmHdr); + tdbPageInit(pPage, szAmHdr, tdbBtreeCellSize); TDB_BTREE_ASSERT_FLAG(flags); @@ -426,7 +426,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { } static int tdbBtreeZeroPage(SPage *pPage, void *arg) { - u16 flags; + u8 flags; SBTree *pBt; u8 isLeaf; u8 szAmHdr; @@ -440,10 +440,8 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { } else { szAmHdr = sizeof(SIntHdr); } - pPage->xCellSize = NULL; // TODO - tdbPageZero(pPage, szAmHdr); - // TDB_PAGE_FLAGS_SET(pPage, flags); + tdbPageZero(pPage, szAmHdr, tdbBtreeCellSize); if (isLeaf) { pPage->kLen = pBt->keyLen; @@ -749,7 +747,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { SPage *pParent; SPage *pPage; int ret; - u16 flags; + u8 flags; u8 leaf; u8 root; @@ -757,7 +755,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { for (;;) { iPage = pCur->iPage; pPage = pCur->pPage; - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); root = TDB_BTREE_PAGE_IS_ROOT(flags); @@ -840,7 +838,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, u8 *pPayload, const void *pKey, i static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell) { - u16 flags; + u8 flags; u8 leaf; int nHeader; int nPayload; @@ -851,7 +849,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo nPayload = 0; nHeader = 0; - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // 1. Encode Header part @@ -914,13 +912,13 @@ static int tdbBtreeDecodePayload(SPage *pPage, const u8 *pPayload, SCellDecoder } static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder) { - u16 flags; + u8 flags; u8 leaf; int nHeader; int ret; nHeader = 0; - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Clear the state of decoder diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 0960d88c33..b9007b99e0 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -103,8 +103,8 @@ struct SPage { int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -void tdbPageZero(SPage *pPage, u8 szAmHdr); -void tdbPageInit(SPage *pPage, u8 szAmHdr); +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 7088904285..97ed9bb703 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -79,7 +79,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) return 0; } -void tdbPageZero(SPage *pPage, u8 szAmHdr) { +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { pPage->pAmHdr = pPage->pData; pPage->pPageHdr = pPage->pAmHdr + szAmHdr; TDB_PAGE_NCELLS_SET(pPage, 0); @@ -91,11 +91,12 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr) { pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); pPage->nOverflow = 0; + pPage->xCellSize = xCellSize; ASSERT((u8 *)pPage->pPageFtr == pPage->pFreeEnd); } -void tdbPageInit(SPage *pPage, u8 szAmHdr) { +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { pPage->pAmHdr = pPage->pData; pPage->pPageHdr = pPage->pAmHdr + szAmHdr; pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); @@ -103,6 +104,7 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr) { pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); pPage->nOverflow = 0; + pPage->xCellSize = xCellSize; ASSERT(pPage->pFreeEnd >= pPage->pFreeStart); ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); From ad0e3a007bba08799cf7b193db4b2f97323edf27 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:45:08 +0000 Subject: [PATCH 019/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 06f96e5498..8b4bef50a5 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -392,7 +392,6 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { SBTree *pBt; u8 flags; u8 isLeaf; - u8 szAmHdr; pBt = ((SBtreeInitPageArg *)arg)->pBt; flags = ((SBtreeInitPageArg *)arg)->flags; @@ -400,13 +399,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); - if (isLeaf) { - szAmHdr = sizeof(SLeafHdr); - } else { - szAmHdr = sizeof(SIntHdr); - } - - tdbPageInit(pPage, szAmHdr, tdbBtreeCellSize); + tdbPageInit(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); TDB_BTREE_ASSERT_FLAG(flags); @@ -429,26 +422,26 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u8 flags; SBTree *pBt; u8 isLeaf; - u8 szAmHdr; flags = ((SBtreeInitPageArg *)arg)->flags; pBt = ((SBtreeInitPageArg *)arg)->pBt; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); - if (isLeaf) { - szAmHdr = sizeof(SLeafHdr); - } else { - szAmHdr = sizeof(SIntHdr); - } - - tdbPageZero(pPage, szAmHdr, tdbBtreeCellSize); + tdbPageZero(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); if (isLeaf) { + SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pPageHdr); + pLeafHdr->flags = flags; + pPage->kLen = pBt->keyLen; pPage->vLen = pBt->valLen; pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; } else { + SIntHdr *pIntHdr = (SIntHdr *)(pPage->pPageHdr); + pIntHdr->flags = flags; + pIntHdr->pgno = 0; + pPage->kLen = pBt->keyLen; pPage->vLen = sizeof(SPgno); pPage->maxLocal = pBt->maxLocal; From 9c8102027fc921e0c9d35f50b16e4579a5df6569 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:46:37 +0000 Subject: [PATCH 020/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8b4bef50a5..f72ce41355 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -430,7 +430,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { tdbPageZero(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); if (isLeaf) { - SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pPageHdr); + SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pAmHdr); pLeafHdr->flags = flags; pPage->kLen = pBt->keyLen; @@ -438,7 +438,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; } else { - SIntHdr *pIntHdr = (SIntHdr *)(pPage->pPageHdr); + SIntHdr *pIntHdr = (SIntHdr *)(pPage->pAmHdr); pIntHdr->flags = flags; pIntHdr->pgno = 0; From 0cd8cca66dd1aa03b16079de53514c4545ec9d31 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:55:17 +0000 Subject: [PATCH 021/105] more TDB --- source/libs/tdb/src/page/tdbPage.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 97ed9bb703..4a067bb978 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -141,7 +141,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { u8 *src = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * lidx; u8 *dest = src + TDB_PAGE_OFFSET_SIZE(pPage); memmove(dest, src, pPage->pFreeStart - dest); - TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, TDB_PAGE_OFFSET_SIZE(pPage) * lidx); + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, pNewCell - pPage->pData); TDB_PAGE_NCELLS_SET(pPage, nCells + 1); ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); From 8becb5421110425ce073ce2e8fb70176087862b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 07:30:34 +0000 Subject: [PATCH 022/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 7 ------- source/libs/tdb/src/inc/tdbPage.h | 1 + source/libs/tdb/src/page/tdbPage.c | 10 ++++++++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f72ce41355..4c7cddf311 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -752,13 +752,6 @@ static int tdbBtreeBalance(SBtCursor *pCur) { leaf = TDB_BTREE_PAGE_IS_LEAF(flags); root = TDB_BTREE_PAGE_IS_ROOT(flags); - // TODO: Get the page free space if not get yet - // if (pPage->nFree < 0) { - // if (tdbBtreeComputeFreeSpace(pPage) < 0) { - // return -1; - // } - // } - // when the page is not overflow and not too empty, the balance work // is finished. Just break out the balance loop. if (pPage->nOverflow == 0 /* TODO: && pPage->nFree <= */) { diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index b9007b99e0..00322067df 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -107,6 +107,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); +void tdbPageCopy(SPage *pFromPage, SPage *pToPage); static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { SCell *pCell; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 4a067bb978..c3fec25d81 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -207,6 +207,16 @@ int tdbPageDropCell(SPage *pPage, int idx) { return 0; } +void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { + pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); + pToPage->pFreeEnd = (u8 *)(pToPage->pPageFtr) - ((u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); + + ASSERT(pToPage->pFreeEnd >= pToPage->pFreeStart); + + memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); + memcpy(pToPage->pFreeEnd, pFromPage->pFreeEnd, (u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); +} + static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { SCell *pFreeCell; u8 *pOffset; From b8954ae1c5036a91293b385c80fc69501a8e5e82 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 07:55:56 +0000 Subject: [PATCH 023/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 28 +--------------------------- source/libs/tdb/src/page/tdbPage.c | 12 ++++++++++++ 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4c7cddf311..9e6ea91a4c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -462,29 +462,6 @@ typedef struct { SPage *pNewPages[5]; } SBtreeBalanceHelper; -static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { -#if 0 - int nCells = TDB_PAGE_TOTAL_CELLS(pFrom); - int cCells = TDB_PAGE_CCELLS(pFrom); - int fCell = TDB_PAGE_FCELL(pFrom); - int nFree = TDB_PAGE_NFREE(pFrom); - - pTo->pFreeStart = pTo->pCellIdx + nCells * pFrom->pPageMethods->szOffset; - memcpy(pTo->pCellIdx, pFrom->pCellIdx, nCells * pFrom->pPageMethods->szOffset); - pTo->pFreeEnd = (u8 *)pTo->pPageFtr - (u8 *)(pFrom->pPageFtr) + pFrom->pFreeEnd; - memcpy(pTo->pFreeEnd, pFrom->pFreeEnd, (u8 *)pFrom->pPageFtr - pFrom->pFreeEnd); - - TDB_PAGE_NCELLS_SET(pTo, nCells); - TDB_PAGE_CCELLS_SET(pTo, cCells); - TDB_PAGE_FCELL_SET(pTo, fCell); - TDB_PAGE_NFREE_SET(pTo, nFree); - - // TODO: update other fields -#endif - - return 0; -} - static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPager *pPager; SPage *pChild; @@ -503,10 +480,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { } // Copy the root page content to the child page - ret = tdbBtreeCopyPageContent(pRoot, pChild); - if (ret < 0) { - return -1; - } + tdbPageCopy(pRoot, pChild); pChild->nOverflow = pRoot->nOverflow; for (int i = 0; i < pChild->nOverflow; i++) { diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index c3fec25d81..803bf168cf 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -208,6 +208,8 @@ int tdbPageDropCell(SPage *pPage, int idx) { } void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { + int delta, nFree; + pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); pToPage->pFreeEnd = (u8 *)(pToPage->pPageFtr) - ((u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); @@ -215,6 +217,16 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); memcpy(pToPage->pFreeEnd, pFromPage->pFreeEnd, (u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); + + ASSERT(TDB_PAGE_CCELLS(pToPage) == pToPage->pFreeEnd - pToPage->pData); + + delta = (pToPage->pPageHdr - pToPage->pAmHdr) - (pFromPage->pPageHdr - pFromPage->pAmHdr); + if (delta != 0) { + nFree = TDB_PAGE_NFREE(pFromPage); + TDB_PAGE_NFREE_SET(pToPage, nFree - delta); + } + + // TODO: do we need to copy the overflow part ??? } static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { From 2e6f7f15d5934ab67f6ee2cb1f2e7bb435bccb9e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 07:59:37 +0000 Subject: [PATCH 024/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 6 ------ source/libs/tdb/src/page/tdbPage.c | 7 ++++++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 9e6ea91a4c..5f56b2f1e7 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -482,12 +482,6 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { // Copy the root page content to the child page tdbPageCopy(pRoot, pChild); - pChild->nOverflow = pRoot->nOverflow; - for (int i = 0; i < pChild->nOverflow; i++) { - pChild->apOvfl[i] = pRoot->apOvfl[i]; - pChild->aiOvfl[i] = pRoot->aiOvfl[i]; - } - // Reinitialize the root page zArg.flags = TDB_BTREE_ROOT; zArg.pBt = pBt; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 803bf168cf..2f6d4101a5 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -226,7 +226,12 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { TDB_PAGE_NFREE_SET(pToPage, nFree - delta); } - // TODO: do we need to copy the overflow part ??? + // Copy the overflow cells + for (int iOvfl = 0; iOvfl < pFromPage->nOverflow; iOvfl++) { + pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl]; + pToPage->apOvfl[iOvfl] = pFromPage->apOvfl[iOvfl]; + } + pToPage->nOverflow = pFromPage->nOverflow; } static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { From 5ca3e29cf19ed454ec9a0a4e7c00c3d474d01c99 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 08:17:58 +0000 Subject: [PATCH 025/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 6 ++++-- source/libs/tdb/src/page/tdbPage.c | 1 - source/libs/tdb/src/page/tdbPageL.c | 9 ++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 5f56b2f1e7..4e4b2275f8 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -467,6 +467,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPage *pChild; SPgno pgnoChild; int ret; + SIntHdr *pIntHdr; SBtreeInitPageArg zArg; pPager = pRoot->pPager; @@ -490,7 +491,8 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } - // TODO: ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; + pIntHdr = (SIntHdr *)(pRoot->pAmHdr); + pIntHdr->pgno = pgnoChild; *ppChild = pChild; return 0; @@ -731,7 +733,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { // ignore the case of empty if (pPage->nOverflow == 0) break; - ret = tdbBtreeBalanceDeeper(pCur->pBt, pCur->pPage, &(pCur->pgStack[1])); + ret = tdbBtreeBalanceDeeper(pCur->pBt, pPage, &(pCur->pgStack[1])); if (ret < 0) { return -1; } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 2f6d4101a5..bc00da9381 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -416,7 +416,6 @@ static int tdbPageDefragment(SPage *pPage) { /* ---------------------------------------------------------------------------------------------------------- */ typedef struct __attribute__((__packed__)) { - u16 flags; u16 cellNum; u16 cellBody; u16 cellFree; diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index cfd8c919df..c5d4a6047f 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -16,11 +16,10 @@ #include "tdbInt.h" typedef struct __attribute__((__packed__)) { - u16 flags; - u8 cellNum[3]; - u8 cellBody[3]; - u8 cellFree[3]; - u8 nFree[3]; + u8 cellNum[3]; + u8 cellBody[3]; + u8 cellFree[3]; + u8 nFree[3]; } SPageHdrL; typedef struct __attribute__((__packed__)) { From db870af8b7afb105417d4d3d7e6acaaf1bc90235 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 09:33:10 +0000 Subject: [PATCH 026/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 65 +++++++++++++++++++++++++----- source/libs/tdb/src/inc/tdbPage.h | 3 +- source/libs/tdb/src/page/tdbPage.c | 10 ++--- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4e4b2275f8..b801002822 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -41,8 +41,8 @@ struct SBTree { #define TDB_BTREE_PAGE_COMMON_HDR u8 flags; -#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pAmHdr[0] -#define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pAmHdr[0] = (flags)) +#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pData[0] +#define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pData[0] = (flags)) typedef struct __attribute__((__packed__)) { TDB_BTREE_PAGE_COMMON_HDR @@ -393,8 +393,8 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { u8 flags; u8 isLeaf; - pBt = ((SBtreeInitPageArg *)arg)->pBt; - flags = ((SBtreeInitPageArg *)arg)->flags; + pBt = (SBTree *)arg; + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); @@ -430,7 +430,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { tdbPageZero(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); if (isLeaf) { - SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pAmHdr); + SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pData); pLeafHdr->flags = flags; pPage->kLen = pBt->keyLen; @@ -438,7 +438,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; } else { - SIntHdr *pIntHdr = (SIntHdr *)(pPage->pAmHdr); + SIntHdr *pIntHdr = (SIntHdr *)(pPage->pData); pIntHdr->flags = flags; pIntHdr->pgno = 0; @@ -491,7 +491,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } - pIntHdr = (SIntHdr *)(pRoot->pAmHdr); + pIntHdr = (SIntHdr *)(pRoot->pData); pIntHdr->pgno = pgnoChild; *ppChild = pChild; @@ -643,15 +643,59 @@ static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { } static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { - int ret; + int ret; +#if 0 SBtreeBalanceHelper blh; - // ASSERT(!TDB_BTREE_PGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); - blh.pBt = pBt; blh.pParent = pParent; blh.idx = idx; +#endif + int nOlds; + SPage *pOlds[3]; + + { + // Find 3 child pages at most to do balance + int nCells = TDB_PAGE_TOTAL_CELLS(pParent); + int sIdx; + if (nCells <= 2) { + sIdx = 0; + nOlds = nCells + 1; + } else { + // has more than three child pages + if (idx == 0) { + sIdx = 0; + } else if (idx == nCells) { + sIdx = idx - 2; + } else { + sIdx = idx - 1; + } + nOlds = 3; + } + for (int i = 0; i < nOlds; i++, sIdx++) { + ASSERT(sIdx <= nCells); + + SPgno pgno; + if (sIdx == nCells) { + ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pParent))); + pgno = ((SIntHdr *)(pParent->pData))->pgno; + } else { + SCell *pCell; + + pCell = tdbPageGetCell(pParent, sIdx); + pgno = *(SPgno *)pCell; + } + + ret = tdbPagerFetchPage(pBt->pPager, pgno, pOlds + i, tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + } + +#if 0 // Step 1: find two sibling pages and get engough info about the old pages ret = tdbBtreeBalanceStep1(&blh); if (ret < 0) { @@ -693,6 +737,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { ASSERT(0); return -1; } +#endif { // TODO: Reset states diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 00322067df..e513a63fe4 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -58,7 +58,6 @@ struct SPage { u8 *pData; SPageMethods *pPageMethods; // Fields below used by pager and am - u8 *pAmHdr; u8 *pPageHdr; u8 *pCellIdx; u8 *pFreeStart; @@ -114,7 +113,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { int iOvfl; int lidx; - ASSERT(idx >= 0 && idx < pPage->nOverflow + pPage->pPageMethods->getCellNum(pPage)); + ASSERT(idx >= 0 && idx < TDB_PAGE_TOTAL_CELLS(pPage)); iOvfl = 0; for (; iOvfl < pPage->nOverflow; iOvfl++) { diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index bc00da9381..0dc7853940 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -80,8 +80,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) } void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { - pPage->pAmHdr = pPage->pData; - pPage->pPageHdr = pPage->pAmHdr + szAmHdr; + pPage->pPageHdr = pPage->pData + szAmHdr; TDB_PAGE_NCELLS_SET(pPage, 0); TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); TDB_PAGE_FCELL_SET(pPage, 0); @@ -97,8 +96,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell } void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { - pPage->pAmHdr = pPage->pData; - pPage->pPageHdr = pPage->pAmHdr + szAmHdr; + pPage->pPageHdr = pPage->pData + szAmHdr; pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); @@ -117,7 +115,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { int lidx; // local idx SCell *pNewCell; - ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pAmHdr)); + ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); @@ -220,7 +218,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { ASSERT(TDB_PAGE_CCELLS(pToPage) == pToPage->pFreeEnd - pToPage->pData); - delta = (pToPage->pPageHdr - pToPage->pAmHdr) - (pFromPage->pPageHdr - pFromPage->pAmHdr); + delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); if (delta != 0) { nFree = TDB_PAGE_NFREE(pFromPage); TDB_PAGE_NFREE_SET(pToPage, nFree - delta); From b3b5298510d28f1a75ec917dcf8cec1978538697 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 09:47:46 +0000 Subject: [PATCH 027/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 34 ++++++++++++++++--------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index b801002822..52ea45578b 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -695,6 +695,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + // >>>>>>>>>>>>>>>>>>> + #if 0 // Step 1: find two sibling pages and get engough info about the old pages ret = tdbBtreeBalanceStep1(&blh); @@ -854,6 +856,14 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // 1. Encode Header part + /* Encode SPgno if interior page */ + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; + nHeader = nHeader + sizeof(SPgno); + } + /* Encode kLen if need */ if (pPage->kLen == TDB_VARIANT_LEN) { nHeader += tdbPutVarInt(pCell + nHeader, kLen); @@ -864,14 +874,6 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo nHeader += tdbPutVarInt(pCell + nHeader, vLen); } - /* Encode SPgno if interior page */ - if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); - - ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; - nHeader = nHeader + sizeof(SPgno); - } - // 2. Encode payload part if (leaf) { ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, pVal, vLen, &nPayload); @@ -930,6 +932,14 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD pDecoder->pgno = 0; // 1. Decode header part + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; + pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); + nHeader = nHeader + sizeof(SPgno); + } + if (pPage->kLen == TDB_VARIANT_LEN) { nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->kLen)); } else { @@ -942,14 +952,6 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD pDecoder->vLen = pPage->vLen; } - if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); - - pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; - pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); - nHeader = nHeader + sizeof(SPgno); - } - // 2. Decode payload part ret = tdbBtreeDecodePayload(pPage, pCell + nHeader, pDecoder); if (ret < 0) { From 1075c230ed0615f80449000afe09feafd9a63723 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:18:04 +0000 Subject: [PATCH 028/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 37 ++++++++++++++++++++++--------- source/libs/tdb/src/inc/tdbPage.h | 3 ++- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 52ea45578b..93fc75269c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -644,19 +644,11 @@ static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; -#if 0 - SBtreeBalanceHelper blh; - - blh.pBt = pBt; - blh.pParent = pParent; - blh.idx = idx; -#endif int nOlds; SPage *pOlds[3]; - { - // Find 3 child pages at most to do balance + { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); int sIdx; if (nCells <= 2) { @@ -695,7 +687,32 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - // >>>>>>>>>>>>>>>>>>> + int nNews = 0; + int cntNews[5] = {0}; // TODO: maybe 5 is not enough + int szNews[5] = {0}; + int maxPageCapacity; // TODO + + { // Get how many new pages are needed and the new distribution + + // loop to find number of pages needed + for (int i = 0; i < nOlds; i++) { + SPage *pPage = pOlds[i]; + SCell *pCell; + int cellBytes; + for (int cIdx = 0; cIdx < TDB_PAGE_TOTAL_CELLS(pPage); cIdx++) { + pCell = tdbPageGetCell(pPage, cIdx); + cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); + + if (szNews[nNews] + cellBytes > maxPageCapacity) { + nNews++; + } + cntNews[nNews]++; + szNews[nNews] = szNews[nNews] + cellBytes; + } + } + + // Loop to make the distribution even + } #if 0 // Step 1: find two sibling pages and get engough info about the old pages diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index e513a63fe4..479c7603e1 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -98,7 +98,8 @@ struct SPage { }) // APIs -#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); From b5b522d66643663ceeabbf5b8a8548153431073a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:23:34 +0000 Subject: [PATCH 029/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 4 ++-- source/libs/tdb/src/inc/tdbPage.h | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 93fc75269c..aa3e4c6b90 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -690,7 +690,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nNews = 0; int cntNews[5] = {0}; // TODO: maybe 5 is not enough int szNews[5] = {0}; - int maxPageCapacity; // TODO { // Get how many new pages are needed and the new distribution @@ -699,11 +698,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SPage *pPage = pOlds[i]; SCell *pCell; int cellBytes; + for (int cIdx = 0; cIdx < TDB_PAGE_TOTAL_CELLS(pPage); cIdx++) { pCell = tdbPageGetCell(pPage, cIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - if (szNews[nNews] + cellBytes > maxPageCapacity) { + if (szNews[nNews] + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { nNews++; } cntNews[nNews]++; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 479c7603e1..258eafcf63 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -99,6 +99,7 @@ struct SPage { // APIs #define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); From bd8c171be1d79dfefd53288739110cb0536ef2a5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:44:18 +0000 Subject: [PATCH 030/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 39 ++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index aa3e4c6b90..4169509280 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -978,10 +978,37 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD return 0; } -#endif - static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { - // TODO - ASSERT(0); - return 0; -} \ No newline at end of file + u8 flags; + u8 isLeaf; + int szCell; + int kLen = 0, vLen = 0; + + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + szCell = 0; + + if (!isLeaf) { + szCell += sizeof(SPgno); + } + + if (pPage->kLen == TDB_VARIANT_LEN) { + szCell += tdbGetVarInt(pCell + szCell, &kLen); + } else { + kLen = pPage->kLen; + } + + if (isLeaf) { + if (pPage->vLen == TDB_VARIANT_LEN) { + szCell += tdbGetVarInt(pCell + szCell, &vLen); + } else { + vLen = pPage->vLen; + } + } + + szCell = szCell + kLen + vLen; + + return szCell; +} + +#endif From d123388a1962630ead69fa7d13e4b92c5f7f4be5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:53:49 +0000 Subject: [PATCH 031/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4169509280..670cdc002a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -711,7 +711,16 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - // Loop to make the distribution even + nNews = nNews + 1; + + // back loop to make the distribution even + for (int iNew = nNews - 1; iNew > 0; iNew--) { + /* code */ + } + } + + SPage *pNews[5]; + { // Allocate the new pages } #if 0 From 925004770202e76b8de27e4a8f4cfbf48c9d67dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 14:56:07 +0000 Subject: [PATCH 032/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 670cdc002a..98a5cf158c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -715,7 +715,34 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { - /* code */ + // TODO: find the last cell of page (iNew-1) + int cIdx; + int iPage; + int cellBytes; + SPage *pPage; + SCell *pCell; + + for (;;) { + pCell = tdbPageGetCell(pPage, cIdx); + cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); + + if (szNews[iNew] + cellBytes >= szNews[iNew - 1] + cellBytes) { + break; + } + + // Move the cell right + szNews[iNew] += cellBytes; + cntNews[iNew]++; + + szNews[iNew - 1] -= cellBytes; + cntNews[iNew - 1]++; + + cIdx--; + if (cIdx < 0) { + pPage = pOlds[--iPage]; + cIdx = TDB_PAGE_TOTAL_CELLS(pPage) - 1; + } + } } } From e2f1434bf2c9d2b5cbe13408f4fbbf876a0ec417 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:08:56 +0000 Subject: [PATCH 033/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 108 ++++++++++-------------------- 1 file changed, 34 insertions(+), 74 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 98a5cf158c..cbb389140f 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -688,26 +688,33 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } int nNews = 0; - int cntNews[5] = {0}; // TODO: maybe 5 is not enough - int szNews[5] = {0}; + struct { + int cnt; + int size; + int oPage; + int oIdx; + } infoNews[5] = {0}; { // Get how many new pages are needed and the new distribution - // loop to find number of pages needed - for (int i = 0; i < nOlds; i++) { - SPage *pPage = pOlds[i]; + // first loop to find minimum number of pages needed + for (int oPage = 0; oPage < nOlds; oPage++) { + SPage *pPage = pOlds[oPage]; SCell *pCell; int cellBytes; - for (int cIdx = 0; cIdx < TDB_PAGE_TOTAL_CELLS(pPage); cIdx++) { - pCell = tdbPageGetCell(pPage, cIdx); + for (int oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { + pCell = tdbPageGetCell(pPage, oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - if (szNews[nNews] + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { + if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { + // page is full, use a new page nNews++; } - cntNews[nNews]++; - szNews[nNews] = szNews[nNews] + cellBytes; + infoNews[nNews].cnt++; + infoNews[nNews].size += cellBytes; + infoNews[nNews].oPage = oPage; + infoNews[nNews].oIdx = oIdx; } } @@ -715,85 +722,38 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { - // TODO: find the last cell of page (iNew-1) - int cIdx; - int iPage; - int cellBytes; - SPage *pPage; SCell *pCell; + SPage *pPage; + int cellBytes; for (;;) { - pCell = tdbPageGetCell(pPage, cIdx); + pPage = pOlds[infoNews[iNew - 1].oPage]; + pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - if (szNews[iNew] + cellBytes >= szNews[iNew - 1] + cellBytes) { - break; + infoNews[iNew].cnt++; + infoNews[iNew].size += cellBytes; + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= cellBytes; + if ((infoNews[iNew - 1].oIdx--) == 0) { + infoNews[iNew - 1].oPage--; + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pOlds[infoNews[iNew - 1].oPage]); } - // Move the cell right - szNews[iNew] += cellBytes; - cntNews[iNew]++; - - szNews[iNew - 1] -= cellBytes; - cntNews[iNew - 1]++; - - cIdx--; - if (cIdx < 0) { - pPage = pOlds[--iPage]; - cIdx = TDB_PAGE_TOTAL_CELLS(pPage) - 1; + if (infoNews[iNew].size > infoNews[iNew - 1].size) { + break; } } } + + int k = 0; } SPage *pNews[5]; - { // Allocate the new pages + { + // Allocate the new pages } -#if 0 - // Step 1: find two sibling pages and get engough info about the old pages - ret = tdbBtreeBalanceStep1(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 2: Load all cells on the old page and the divider cells - ret = tdbBtreeBalanceStep2(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 3: Get the number of pages needed to hold all cells - ret = tdbBtreeBalanceStep3(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 4: Allocate enough new pages. Reuse old pages as much as possible - ret = tdbBtreeBalanceStep4(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 5: Insert new divider cells into pParent - ret = tdbBtreeBalanceStep5(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 6: Update the sibling pages - ret = tdbBtreeBalanceStep6(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } -#endif - { // TODO: Reset states } From d5afc7240a92968e07027c464d1db6caee590084 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:39:12 +0000 Subject: [PATCH 034/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index cbb389140f..e969f15775 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -745,13 +745,29 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } - - int k = 0; } - SPage *pNews[5]; - { - // Allocate the new pages + SPage *pNews[5] = {0}; + { // Allocate new pages, reuse the old page when possible + + SPgno pgno; + SBtreeInitPageArg iarg; + u8 flags; + + flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); + + for (int iNew = 0; iNew < nNews; iNew++) { + if (iNew < nOlds) { + pNews[iNew] = pOlds[iNew]; + } else { + iarg.pBt = pBt; + iarg.flags = flags; + ret = tdbPagerNewPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeZeroPage, &iarg); + if (ret < 0) { + ASSERT(0); + } + } + } } { From 6cd9d2703428e5647fdd64a0c40c94d3cd68724f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:45:59 +0000 Subject: [PATCH 035/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 150 +----------------------------- 1 file changed, 1 insertion(+), 149 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e969f15775..af0f331373 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -498,150 +498,6 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return 0; } -static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { - int nCells; - int i; - int idxStart; - int nChild; - int ret; - SPage *pParent; - SPgno pgno; - SCell *pCell; - SCellDecoder cd; - SBTree *pBt; - - pParent = pBlh->pParent; - nCells = TDB_PAGE_TOTAL_CELLS(pParent); - nChild = nCells + 1; - pBt = pBlh->pBt; - - // TODO: ASSERT below needs to be removed - ASSERT(pParent->nOverflow == 0); - ASSERT(pBlh->idx <= nCells); - - if (nChild < 3) { - idxStart = 0; - pBlh->nOld = nChild; - } else { - if (pBlh->idx == 0) { - idxStart = 0; - } else if (pBlh->idx == nCells) { - idxStart = pBlh->idx - 2; - } else { - idxStart = pBlh->idx - 1; - } - pBlh->nOld = 3; - } - - i = pBlh->nOld - 1; - - if (idxStart + i == nCells) { - // pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; - } else { - pCell = tdbPageGetCell(pParent, idxStart + i); - // TODO: no need to decode the payload part, and even the kLen, vLen part - // we only need the pgno part - ret = tdbBtreeDecodeCell(pParent, pCell, &cd); - if (ret < 0) { - ASSERT(0); - return -1; - } - pgno = cd.pgno; - } - for (;;) { - ret = tdbPagerFetchPage(pBt->pPager, pgno, &(pBlh->pOldPages[i]), tdbBtreeInitPage, pBt); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Loop over - if ((i--) == 0) break; - - { - // TODO - // ASSERT(0); - } - } - - return 0; -} - -static int tdbBtreeBalanceStep2(SBtreeBalanceHelper *pBlh) { -#if 0 - SPage *pPage; - int oidx; - int cidx; - int limit; - SCell *pCell; - - for (int i = 0; i < pBlh->nOld; i++) { - pPage = pBlh->pOldPages[i]; - oidx = 0; - cidx = 0; - - if (oidx < pPage->nOverflow) { - limit = pPage->aiOvfl[oidx]; - } else { - limit = pPage->pPageHdr->nCells; - } - - // Loop to copy each cell pointer out - for (;;) { - if (oidx >= pPage->nOverflow && cidx >= pPage->pPageHdr->nCells) break; - - if (cidx < limit) { - // Get local cells - pCell = tdbPageGetCell(pPage, cidx); - } else if (cidx == limit) { - // Get overflow cells - pCell = pPage->apOvfl[oidx++]; - - if (oidx < pPage->nOverflow) { - limit = pPage->aiOvfl[oidx]; - } else { - limit = pPage->pPageHdr->nCells; - } - } else { - ASSERT(0); - } - } - - { - // TODO: Copy divider cells here - } - } - - /* TODO */ - -#endif - return 0; -} - -static int tdbBtreeBalanceStep3(SBtreeBalanceHelper *pBlh) { - // Figure out number of pages needed after balance - for (int i = 0; i < pBlh->nOld; i++) { - /* TODO */ - } - - return 0; -} - -static int tdbBtreeBalanceStep4(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - -static int tdbBtreeBalanceStep5(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - -static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; @@ -771,11 +627,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } { - // TODO: Reset states - } - - { - // TODO: Clear resources + // Do the actual redistribute } return 0; From f3c10f886c589766bda0ade1f4c9a88b24ef3fd4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:48:32 +0000 Subject: [PATCH 036/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index af0f331373..71a9e70064 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -310,32 +310,6 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p return 0; } -static int tdbBtCursorMoveToRoot(SBtCursor *pCur) { - SBTree *pBt; - SPager *pPager; - SPage *pPage; - int ret; - - pBt = pCur->pBt; - pPager = pBt->pPager; - - // pPage = tdbPagerGet(pPager, pBt->root, true); - // if (pPage == NULL) { - // // TODO: handle error - // } - - // ret = tdbInitBtPage(pPage, &pBtPage); - // if (ret < 0) { - // // TODO - // return 0; - // } - - // pCur->pPage = pBtPage; - // pCur->iPage = 0; - - return 0; -} - static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { int mlen; int cret; From df12ae0d79e73725e54cf4c7ca44743dd21eda71 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 05:28:54 +0000 Subject: [PATCH 037/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 50 ++++++++++++++++++++++++++++-- source/libs/tdb/src/db/tdbPCache.c | 14 +-------- source/libs/tdb/src/inc/tdbUtil.h | 10 ++++++ source/libs/tdb/src/page/tdbPage.c | 7 +++++ 4 files changed, 66 insertions(+), 15 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 71a9e70064..196c4593c5 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -598,10 +598,56 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } + + // TODO: sort the page according to the page number } - { - // Do the actual redistribute + { // Do the actual cell distribution + + SPage *pTPage[2]; + int tPage, tIdx, iOld; + SCell *pCell; + int szCell; + SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; + + for (int i = 0; i < 2; i++) { + ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); + if (ret < 0) { + ASSERT(0); + } + } + + tPage = 0; + tIdx = 0; + iOld = 0; + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + + for (int iNew = 0; iNew < nNews; iNew++) { + // fill the iNew page + tdbBtreeZeroPage(pNews[iNew], &iarg); + + for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { + while (tIdx >= TDB_PAGE_TOTAL_CELLS(pTPage[tPage])) { + tPage = (tPage + 1) % 2; + tIdx = 0; + + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + } + + pCell = tdbPageGetCell(pTPage[tPage], tIdx); + szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + + tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell); + + tIdx++; + } + } + + for (int i = 0; i < 2; i++) { + tdbPageDestroy(pTPage[i], NULL, NULL); + } } return 0; diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 29f7395cbc..1fc0847b38 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -57,8 +57,6 @@ static void tdbPCachePinPage(SPage *pPage); static void tdbPCacheRemovePageFromHash(SPage *pPage); static void tdbPCacheAddPageToHash(SPage *pPage); static void tdbPCacheUnpinPage(SPage *pPage); -static void *tdbOsMalloc(void *arg, size_t size); -static void tdbOsFree(void *arg, void *ptr); int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { SPCache *pCache; @@ -257,7 +255,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { pCache->nFree = 0; pCache->pFree = NULL; for (int i = 0; i < pCache->cacheSize; i++) { - ret = tdbPageCreate(pCache->pageSize, &pPage, tdbOsMalloc, NULL); + ret = tdbPageCreate(pCache->pageSize, &pPage, NULL, NULL); if (ret < 0) { // TODO: handle error return -1; @@ -297,13 +295,3 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { } int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; } - -static void *tdbOsMalloc(void *arg, size_t size) { - void *ptr; - - ptr = malloc(size); - - return ptr; -} - -static void tdbOsFree(void *arg, void *ptr) { free(ptr); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 9576ee6d15..9358aae236 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -39,6 +39,16 @@ int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); int tdbPRead(int fd, void *pData, int count, i64 offset); +static inline void *tdbOsMalloc(void *arg, size_t size) { + void *ptr; + + ptr = malloc(size); + + return ptr; +} + +static inline void tdbOsFree(void *arg, void *ptr) { free(ptr); } + static inline int tdbPutVarInt(u8 *p, int v) { int n = 0; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 0dc7853940..16130ba13f 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -48,6 +48,9 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = NULL; size = pageSize + sizeof(*pPage); + if (xMalloc == NULL) { + xMalloc = tdbOsMalloc; + } ptr = (u8 *)((*xMalloc)(arg, size)); if (pPage == NULL) { @@ -73,6 +76,10 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { u8 *ptr; + if (!xFree) { + xFree = tdbOsFree; + } + ptr = pPage->pData; (*xFree)(arg, ptr); From 3abd4b2e672f5af1e1de3652427dbe840b71d9ce Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 06:08:28 +0000 Subject: [PATCH 038/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 16 +++++++++++++++- source/libs/tdb/src/inc/tdbPage.h | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 196c4593c5..a05202f456 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -602,7 +602,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // TODO: sort the page according to the page number } - { // Do the actual cell distribution + { // Do the real cell distribution SPage *pTPage[2]; int tPage, tIdx, iOld; @@ -650,6 +650,20 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + { // Insert records in parent page + int cIdx; + int szCell; + SCell pCell[128]; // TODO + SCellDecoder cd; + + for (int iNew = 0; iNew < nNews; iNew++) { + tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); + + tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); + tdbPageInsertCell(pParent, cIdx, pCell, szCell); + } + } + return 0; } diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 258eafcf63..61927fabf6 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -100,6 +100,7 @@ struct SPage { // APIs #define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) #define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) +#define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno) #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); From 0b752392aa482ab45cd38d6d78a4112c84782042 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 06:24:16 +0000 Subject: [PATCH 039/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a05202f456..da11eb099a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -495,17 +495,17 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } nOlds = 3; } - for (int i = 0; i < nOlds; i++, sIdx++) { - ASSERT(sIdx <= nCells); + for (int i = 0; i < nOlds; i++) { + ASSERT(sIdx + i <= nCells); SPgno pgno; - if (sIdx == nCells) { + if (sIdx + i == nCells) { ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pParent))); pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { SCell *pCell; - pCell = tdbPageGetCell(pParent, sIdx); + pCell = tdbPageGetCell(pParent, sIdx + i); pgno = *(SPgno *)pCell; } @@ -515,6 +515,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return -1; } } + // drop the cells + for (int i = 0; i < nOlds; i++) { + nCells = TDB_PAGE_TOTAL_CELLS(pParent); + if (sIdx < nCells) { + tdbPageDropCell(pParent, sIdx); + } else { + ((SIntHdr *)pParent->pData)->pgno = 0; + } + } } int nNews = 0; @@ -657,6 +666,16 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SCellDecoder cd; for (int iNew = 0; iNew < nNews; iNew++) { + if (iNew == nNews - 1) { + // The last new page + + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + break; + } + } + tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); @@ -664,6 +683,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + int k = 0; + return 0; } From 2d1e356cc214aa99aed731b17352635f95728f41 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 06:56:29 +0000 Subject: [PATCH 040/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index da11eb099a..e7abbc7161 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -477,10 +477,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nOlds; SPage *pOlds[3]; + int sIdx; { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); - int sIdx; if (nCells <= 2) { sIdx = 0; nOlds = nCells + 1; @@ -515,7 +515,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return -1; } } - // drop the cells + // drop the cells on parent page for (int i = 0; i < nOlds; i++) { nCells = TDB_PAGE_TOTAL_CELLS(pParent); if (sIdx < nCells) { @@ -660,7 +660,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } { // Insert records in parent page - int cIdx; + int cIdx = sIdx; int szCell; SCell pCell[128]; // TODO SCellDecoder cd; @@ -679,12 +679,11 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); + // TODO: the cell here may be used by pParent as an overflow cell tdbPageInsertCell(pParent, cIdx, pCell, szCell); } } - int k = 0; - return 0; } From 238550b0f4d0ceeb987ad41eb7992afba994dacd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 07:50:34 +0000 Subject: [PATCH 041/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 36 ++++++++++++++++++++++++++---- source/libs/tdb/src/inc/tdbPage.h | 1 + source/libs/tdb/src/page/tdbPage.c | 1 - 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e7abbc7161..d1188ee5b3 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -477,10 +477,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nOlds; SPage *pOlds[3]; + SCell *pDivCell[2] = {0}; + int szDivCell[2]; int sIdx; + u8 childLeaf; { // Find 3 child pages at most to do balance - int nCells = TDB_PAGE_TOTAL_CELLS(pParent); + int nCells = TDB_PAGE_TOTAL_CELLS(pParent); + SCell *pCell; + if (nCells <= 2) { sIdx = 0; nOlds = nCells + 1; @@ -503,8 +508,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pParent))); pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { - SCell *pCell; - pCell = tdbPageGetCell(pParent, sIdx + i); pgno = *(SPgno *)pCell; } @@ -515,6 +518,17 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return -1; } } + // copy the parent key out if child pages are not leaf page + childLeaf = TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); + if (childLeaf) { + for (int i = 0; i < nOlds - 1; i++) { + pCell = tdbPageGetCell(pParent, sIdx + i); + + szDivCell[i] = tdbBtreeCellSize(pParent, pCell); + pDivCell[i] = malloc(szDivCell[i]); + memcpy(pDivCell, pCell, szDivCell[i]); + } + } // drop the cells on parent page for (int i = 0; i < nOlds; i++) { nCells = TDB_PAGE_TOTAL_CELLS(pParent); @@ -541,14 +555,28 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SPage *pPage = pOlds[oPage]; SCell *pCell; int cellBytes; + int oIdx; - for (int oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { + for (oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { pCell = tdbPageGetCell(pPage, oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; + // for a child leaf case, this cell is used as the new divider cell + if (childLeaf) continue; + } + infoNews[nNews].cnt++; + infoNews[nNews].size += cellBytes; + infoNews[nNews].oPage = oPage; + infoNews[nNews].oIdx = oIdx; + } + + // For child leaf pages + if (childLeaf && oPage < nOlds - 1) { + if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { + nNews++; } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 61927fabf6..0ba7cf236b 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -102,6 +102,7 @@ struct SPage { #define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) #define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno) #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 16130ba13f..14960de45d 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -18,7 +18,6 @@ extern SPageMethods pageMethods; extern SPageMethods pageLargeMethods; -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) #define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) #define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) #define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) From aee8b603242f0db0facc04cb7b3793815395c521 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 08:13:32 +0000 Subject: [PATCH 042/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index d1188ee5b3..6daecb7908 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -564,7 +564,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for a child leaf case, this cell is used as the new divider cell + // for a child leaf case, this cell is used as the new divider cell to parent if (childLeaf) continue; } infoNews[nNews].cnt++; @@ -591,10 +591,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { for (int iNew = nNews - 1; iNew > 0; iNew--) { SCell *pCell; SPage *pPage; + int nCells; int cellBytes; + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + for (;;) { - pPage = pOlds[infoNews[iNew - 1].oPage]; pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); @@ -604,7 +607,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { infoNews[iNew - 1].size -= cellBytes; if ((infoNews[iNew - 1].oIdx--) == 0) { infoNews[iNew - 1].oPage--; - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pOlds[infoNews[iNew - 1].oPage]); + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); } if (infoNews[iNew].size > infoNews[iNew - 1].size) { From d7b54201bcab335b39a881d32f06af88bb987d72 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 08:26:43 +0000 Subject: [PATCH 043/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 50 +++++++++++++++++-------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 6daecb7908..cda958c578 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -520,7 +520,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } // copy the parent key out if child pages are not leaf page childLeaf = TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); - if (childLeaf) { + if (!childLeaf) { for (int i = 0; i < nOlds - 1; i++) { pCell = tdbPageGetCell(pParent, sIdx + i); @@ -564,8 +564,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for a child leaf case, this cell is used as the new divider cell to parent - if (childLeaf) continue; + // for a internal leaf case, this cell is used as the new divider cell to parent + if (!childLeaf) continue; } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; @@ -573,8 +573,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { infoNews[nNews].oIdx = oIdx; } - // For child leaf pages - if (childLeaf && oPage < nOlds - 1) { + // For internal pages + if (!childLeaf && oPage < nOlds - 1) { if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { nNews++; } @@ -594,27 +594,31 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nCells; int cellBytes; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); + if (childLeaf) { // child leaf + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); - for (;;) { - pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); - cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); + for (;;) { + pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); + cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - infoNews[iNew].cnt++; - infoNews[iNew].size += cellBytes; - infoNews[iNew - 1].cnt--; - infoNews[iNew - 1].size -= cellBytes; - if ((infoNews[iNew - 1].oIdx--) == 0) { - infoNews[iNew - 1].oPage--; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); - } - - if (infoNews[iNew].size > infoNews[iNew - 1].size) { - break; + infoNews[iNew].cnt++; + infoNews[iNew].size += cellBytes; + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= cellBytes; + if ((infoNews[iNew - 1].oIdx--) == 0) { + infoNews[iNew - 1].oPage--; + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); + } + + if (infoNews[iNew].size > infoNews[iNew - 1].size) { + break; + } } + } else { // internal leaf + // TODO } } } From 3b49a863ea18e13f224088a212dc3a522089f770 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 10:43:51 +0000 Subject: [PATCH 044/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 194 ++++++++++++++++++++++++------ 1 file changed, 154 insertions(+), 40 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index cda958c578..4c6d85bc0a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -564,7 +564,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for a internal leaf case, this cell is used as the new divider cell to parent + // for an internal leaf case, this cell is used as the new divider cell to parent if (!childLeaf) continue; } infoNews[nNews].cnt++; @@ -594,10 +594,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nCells; int cellBytes; - if (childLeaf) { // child leaf - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + if (childLeaf) { // child leaf for (;;) { pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); @@ -618,7 +618,55 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } else { // internal leaf - // TODO + SCell *pCellRight; + SCell *pCellLeft; + int szCellRight, szCellLeft; + + // get pCellLeft, szCellLeft + if (infoNews[iNew - 1].oIdx == nCells) { + ASSERT(infoNews[iNew - 1].oPage < nOlds - 1); + pCellLeft = pDivCell[infoNews[iNew - 1].oPage]; + szCellLeft = szDivCell[infoNews[iNew - 1].oPage]; + } else { + pCellLeft = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); + szCellLeft = tdbBtreeCellSize(pPage, pCellLeft); + } + + // get pCellRight, szCellRight + if (infoNews[iNew - 1].oIdx + 1 < nCells) { + pCellRight = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx + 1); + szCellRight = tdbBtreeCellSize(pPage, pCellRight); + } else { + if (infoNews[iNew - 1].oPage < nOlds - 1) { + pCellRight = pDivCell[infoNews[iNew - 1].oPage]; + szCellRight = szDivCell[infoNews[iNew - 1].oPage]; + } else { + // TODO: what if the next old page is empty? + pCellRight = tdbPageGetCell(pOlds[infoNews[iNew - 1].oPage + 1], 0); + szCellRight = tdbBtreeCellSize(pPage, pCellRight); + } + } + + if (infoNews[iNew - 1].size - szCellLeft - TDB_PAGE_OFFSET_SIZE(pPage) <= + infoNews[iNew].size + szCellRight + TDB_PAGE_OFFSET_SIZE(pPage)) { + break; + } + + // TODO: ASSERT(infoNews[iNew].size < PAGE_MAX_CAPACITY); + infoNews[iNew].cnt++; + infoNews[iNew].size += szCellRight; + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= szCellLeft; + if ((infoNews[iNew - 1].oIdx--) == 0) { + infoNews[iNew - 1].oPage--; + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + if (infoNews[iNew - 1].oPage < nOlds - 1) { + infoNews[iNew - 1].oIdx = nCells; + } else { + infoNews[iNew - 1].oIdx = nCells - 1; + } + } } } } @@ -654,6 +702,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int tPage, tIdx, iOld; SCell *pCell; int szCell; + int nCells; + SCellDecoder cd; SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; for (int i = 0; i < 2; i++) { @@ -666,28 +716,117 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tPage = 0; tIdx = 0; iOld = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + if (!childLeaf) { + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + } for (int iNew = 0; iNew < nNews; iNew++) { // fill the iNew page + // TODO: copy current page to tmp space tdbBtreeZeroPage(pNews[iNew], &iarg); for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { - while (tIdx >= TDB_PAGE_TOTAL_CELLS(pTPage[tPage])) { - tPage = (tPage + 1) % 2; - tIdx = 0; + for (;;) { // loop to find the next available cell + if (tIdx < nCells) { + pCell = tdbPageGetCell(pTPage[tPage], tIdx); + szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + tIdx++; + break; + } else { + if (!childLeaf) { + if (iOld < nOlds - 1) { + pCell = pDivCell[iOld]; + szCell = szDivCell[iOld]; + ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + break; + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + } + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + } + } } - pCell = tdbPageGetCell(pTPage[tPage], tIdx); - szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell); + } - tIdx++; + // fill right-most child pgno if internal page + if (!childLeaf) { + if (tIdx < nCells) { + pCell = tdbPageGetCell(pTPage[tPage], tIdx); + szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + tIdx++; + break; + } else { + if (!childLeaf) { + if (iOld < nOlds - 1) { + pCell = pDivCell[iOld]; + szCell = szDivCell[iOld]; + ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + break; + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + } + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + } + } + + ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + } + + // insert divider cell into the parent page + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + } else { + tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); + tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, + &szCell); + // TODO: the cell here may be used by pParent as an overflow cell + tdbPageInsertCell(pParent, sIdx++, pCell, szCell); } } @@ -696,31 +835,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - { // Insert records in parent page - int cIdx = sIdx; - int szCell; - SCell pCell[128]; // TODO - SCellDecoder cd; - - for (int iNew = 0; iNew < nNews; iNew++) { - if (iNew == nNews - 1) { - // The last new page - - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - if (pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - break; - } - } - - tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); - - tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); - // TODO: the cell here may be used by pParent as an overflow cell - tdbPageInsertCell(pParent, cIdx, pCell, szCell); - } - } - return 0; } From 3a173f466069f6cf59f2e9e281eaf9298886766d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 11:47:25 +0000 Subject: [PATCH 045/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4c6d85bc0a..67b11f8ef4 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -204,7 +204,19 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { - // TODO + int ret; + + pCur->pgStack[pCur->iPage] = pCur->pPage; + pCur->idxStack[pCur->iPage] = pCur->idx; + pCur->iPage++; + pCur->pPage = NULL; + pCur->idx = -1; + + ret = tdbPagerFetchPage(pCur->pBt->pPager, pgno, &pCur->pPage, tdbBtreeInitPage, pCur->pBt); + if (ret < 0) { + ASSERT(0); + } + return 0; } @@ -287,14 +299,12 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p pCur->idx = midx; tdbBtCursorMoveToChild(pCur, cd.pgno); } else { + pCur->idx = midx + 1; if (midx == nCells - 1) { /* Move to right-most child */ - pCur->idx = midx + 1; - // tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); + tdbBtCursorMoveToChild(pCur, ((SIntHdr *)pCur->pPage->pData)->pgno); } else { - // TODO: reset cd as uninitialized - pCur->idx = midx + 1; - pCell = tdbPageGetCell(pPage, midx + 1); + pCell = tdbPageGetCell(pPage, pCur->idx); tdbBtreeDecodeCell(pPage, pCell, &cd); tdbBtCursorMoveToChild(pCur, cd.pgno); } From 44daa77d8db957e3d9e7ca150bfce61f26f8764a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 05:32:39 +0000 Subject: [PATCH 046/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 33 ++++++++++++++++++------------ source/libs/tdb/src/inc/tdbPage.h | 2 +- source/libs/tdb/src/page/tdbPage.c | 28 ++++++++++++------------- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 67b11f8ef4..a1f879c729 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -187,7 +187,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } // Insert the cell to the index - ret = tdbPageInsertCell(pCur->pPage, idx, pCell, szCell); + ret = tdbPageInsertCell(pCur->pPage, idx, pCell, szCell, 0); if (ret < 0) { return -1; } @@ -490,7 +490,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SCell *pDivCell[2] = {0}; int szDivCell[2]; int sIdx; - u8 childLeaf; + u8 childNotLeaf; { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); @@ -529,14 +529,21 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } // copy the parent key out if child pages are not leaf page - childLeaf = TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); - if (!childLeaf) { + childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); + if (childNotLeaf) { for (int i = 0; i < nOlds - 1; i++) { pCell = tdbPageGetCell(pParent, sIdx + i); szDivCell[i] = tdbBtreeCellSize(pParent, pCell); pDivCell[i] = malloc(szDivCell[i]); memcpy(pDivCell, pCell, szDivCell[i]); + + ((SPgno *)pDivCell)[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; + ((SIntHdr *)pOlds[i]->pData)->pgno = 0; + + // here we insert the cell as an overflow cell to avoid + // the slow defragment process + tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } } // drop the cells on parent page @@ -575,7 +582,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // page is full, use a new page nNews++; // for an internal leaf case, this cell is used as the new divider cell to parent - if (!childLeaf) continue; + if (childNotLeaf) continue; } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; @@ -584,7 +591,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } // For internal pages - if (!childLeaf && oPage < nOlds - 1) { + if (childNotLeaf && oPage < nOlds - 1) { if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { nNews++; } @@ -607,7 +614,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { pPage = pOlds[infoNews[iNew - 1].oPage]; nCells = TDB_PAGE_TOTAL_CELLS(pPage); - if (childLeaf) { // child leaf + if (!childNotLeaf) { // child leaf for (;;) { pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); @@ -729,7 +736,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); tdbBtreeZeroPage(pTPage[tPage], &iarg); tdbPageCopy(pOlds[iOld], pTPage[tPage]); - if (!childLeaf) { + if (childNotLeaf) { ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; } @@ -746,7 +753,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tIdx++; break; } else { - if (!childLeaf) { + if (childNotLeaf) { if (iOld < nOlds - 1) { pCell = pDivCell[iOld]; szCell = szDivCell[iOld]; @@ -780,18 +787,18 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell); + tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); } // fill right-most child pgno if internal page - if (!childLeaf) { + if (childNotLeaf) { if (tIdx < nCells) { pCell = tdbPageGetCell(pTPage[tPage], tIdx); szCell = tdbBtreeCellSize(pTPage[tPage], pCell); tIdx++; break; } else { - if (!childLeaf) { + if (!childNotLeaf) { if (iOld < nOlds - 1) { pCell = pDivCell[iOld]; szCell = szDivCell[iOld]; @@ -836,7 +843,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); // TODO: the cell here may be used by pParent as an overflow cell - tdbPageInsertCell(pParent, sIdx++, pCell, szCell); + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); } } diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 0ba7cf236b..a6f9fbf615 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -108,7 +108,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); int tdbPageDropCell(SPage *pPage, int idx); void tdbPageCopy(SPage *pFromPage, SPage *pToPage); diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 14960de45d..ee1a021a1b 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -114,7 +114,7 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); } -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl) { int nFree; int nCells; int iOvfl; @@ -135,7 +135,19 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { lidx = idx - iOvfl; - if (nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + if (asOvfl || nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + // TODO: make it extensible + // add the cell as an overflow cell + for (int i = pPage->nOverflow; i > iOvfl; i--) { + pPage->apOvfl[i] = pPage->apOvfl[i - 1]; + pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; + } + + pPage->apOvfl[iOvfl] = pCell; + pPage->aiOvfl[iOvfl] = idx; + pPage->nOverflow++; + iOvfl++; + } else { // page must has enough space to hold the cell locally tdbPageAllocate(pPage, szCell, &pNewCell); @@ -149,18 +161,6 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { TDB_PAGE_NCELLS_SET(pPage, nCells + 1); ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); - } else { - // TODO: make it extensible - // add the cell as an overflow cell - for (int i = pPage->nOverflow; i > iOvfl; i--) { - pPage->apOvfl[i] = pPage->apOvfl[i - 1]; - pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; - } - - pPage->apOvfl[iOvfl] = pCell; - pPage->aiOvfl[iOvfl] = idx; - pPage->nOverflow++; - iOvfl++; } for (; iOvfl < pPage->nOverflow; iOvfl++) { From 5c9e63e5018f42af709ac3a75fa831fb7bacff86 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 06:56:53 +0000 Subject: [PATCH 047/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 327 +++++++++++++++++------------- source/libs/tdb/src/inc/tdbInt.h | 4 +- 2 files changed, 183 insertions(+), 148 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a1f879c729..459154e9fb 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -451,13 +451,15 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPage *pChild; SPgno pgnoChild; int ret; + u8 flags; SIntHdr *pIntHdr; SBtreeInitPageArg zArg; pPager = pRoot->pPager; + flags = TDB_BTREE_PAGE_GET_FLAGS(pRoot); // Allocate a new child page - zArg.flags = TDB_BTREE_LEAF; + zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT); zArg.pBt = pBt; ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg); if (ret < 0) { @@ -559,10 +561,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nNews = 0; struct { - int cnt; - int size; - int oPage; - int oIdx; + int cnt; + int size; + SPage *pPage; + int oIdx; } infoNews[5] = {0}; { // Get how many new pages are needed and the new distribution @@ -581,31 +583,63 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for an internal leaf case, this cell is used as the new divider cell to parent - if (childNotLeaf) continue; - } - infoNews[nNews].cnt++; - infoNews[nNews].size += cellBytes; - infoNews[nNews].oPage = oPage; - infoNews[nNews].oIdx = oIdx; - } - // For internal pages - if (childNotLeaf && oPage < nOlds - 1) { - if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { - nNews++; + ASSERT(infoNews[nNews].size + cellBytes <= TDB_PAGE_USABLE_SIZE(pPage)); + + if (childNotLeaf) { + // for non-child page, this cell is used as the right-most child, + // the divider cell to parent as well + continue; + } } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; - infoNews[nNews].oPage = oPage; + infoNews[nNews].pPage = pPage; infoNews[nNews].oIdx = oIdx; } } - nNews = nNews + 1; + nNews++; // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { + SCell *pLCell, pRCell; + int szLCell, szRCell; + + for (;;) { + if (childNotLeaf) { + pLCell = pRCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); + szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pLCell); + } else { + // For internal page, find the left cell + // TODO: fill pLCell, pRCell, szLCell, szRCell + } + + ASSERT(infoNews[iNew - 1].cnt > 0); + + if (infoNews[iNew].size + szRCell >= infoNews[iNew - 1].size - szRCell) { + break; + } + + // move cell divider left + infoNews[iNew].cnt++; + infoNews[iNew].size += szRCell; + { + // TODO + // infoNews[iNew].oPage = ; + // infoNews[iNew].oIdx = ; + } + + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= szLCell; + { + // TODO + // infoNews[iNew-1].oPage = ; + // infoNews[iNew-1].oIdx = ; + } + } + +#if 0 SCell *pCell; SPage *pPage; int nCells; @@ -685,6 +719,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } +#endif } } @@ -713,146 +748,146 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // TODO: sort the page according to the page number } - { // Do the real cell distribution + // { // Do the real cell distribution - SPage *pTPage[2]; - int tPage, tIdx, iOld; - SCell *pCell; - int szCell; - int nCells; - SCellDecoder cd; - SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; + // SPage *pTPage[2]; + // int tPage, tIdx, iOld; + // SCell *pCell; + // int szCell; + // int nCells; + // SCellDecoder cd; + // SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; - for (int i = 0; i < 2; i++) { - ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); - if (ret < 0) { - ASSERT(0); - } - } + // for (int i = 0; i < 2; i++) { + // ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); + // if (ret < 0) { + // ASSERT(0); + // } + // } - tPage = 0; - tIdx = 0; - iOld = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - if (childNotLeaf) { - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - } + // tPage = 0; + // tIdx = 0; + // iOld = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // if (childNotLeaf) { + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // } - for (int iNew = 0; iNew < nNews; iNew++) { - // fill the iNew page - // TODO: copy current page to tmp space - tdbBtreeZeroPage(pNews[iNew], &iarg); + // for (int iNew = 0; iNew < nNews; iNew++) { + // // fill the iNew page + // // TODO: copy current page to tmp space + // tdbBtreeZeroPage(pNews[iNew], &iarg); - for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { - for (;;) { // loop to find the next available cell - if (tIdx < nCells) { - pCell = tdbPageGetCell(pTPage[tPage], tIdx); - szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - tIdx++; - break; - } else { - if (childNotLeaf) { - if (iOld < nOlds - 1) { - pCell = pDivCell[iOld]; - szCell = szDivCell[iOld]; - ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { + // for (;;) { // loop to find the next available cell + // if (tIdx < nCells) { + // pCell = tdbPageGetCell(pTPage[tPage], tIdx); + // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + // tIdx++; + // break; + // } else { + // if (childNotLeaf) { + // if (iOld < nOlds - 1) { + // pCell = pDivCell[iOld]; + // szCell = szDivCell[iOld]; + // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - break; - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - } - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - } - } - } + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // break; + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // } + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // } + // } + // } - tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); - } + // tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); + // } - // fill right-most child pgno if internal page - if (childNotLeaf) { - if (tIdx < nCells) { - pCell = tdbPageGetCell(pTPage[tPage], tIdx); - szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - tIdx++; - break; - } else { - if (!childNotLeaf) { - if (iOld < nOlds - 1) { - pCell = pDivCell[iOld]; - szCell = szDivCell[iOld]; - ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // // fill right-most child pgno if internal page + // if (childNotLeaf) { + // if (tIdx < nCells) { + // pCell = tdbPageGetCell(pTPage[tPage], tIdx); + // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + // tIdx++; + // break; + // } else { + // if (!childNotLeaf) { + // if (iOld < nOlds - 1) { + // pCell = pDivCell[iOld]; + // szCell = szDivCell[iOld]; + // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - break; - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - } - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - } - } + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // break; + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // } + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // } + // } - ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; - } + // ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + // } - // insert divider cell into the parent page - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - if (iNew == nNews - 1 && pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - } else { - tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); - tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, - &szCell); - // TODO: the cell here may be used by pParent as an overflow cell - tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); - } - } + // // insert divider cell into the parent page + // SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + // if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + // pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + // } else { + // tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); + // tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, + // &szCell); + // // TODO: the cell here may be used by pParent as an overflow cell + // tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + // } + // } - for (int i = 0; i < 2; i++) { - tdbPageDestroy(pTPage[i], NULL, NULL); - } - } + // for (int i = 0; i < 2; i++) { + // tdbPageDestroy(pTPage[i], NULL, NULL); + // } +} - return 0; +return 0; } static int tdbBtreeBalance(SBtCursor *pCur) { diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 5902a6a716..3ebba0a4b5 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -141,8 +141,8 @@ typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, i #define TDB_FLAG_IS(flags, flag) ((flags) == (flag)) #define TDB_FLAG_HAS(flags, flag) (((flags) & (flag)) != 0) #define TDB_FLAG_NO(flags, flag) ((flags) & (flag) == 0) -#define TDB_FLAG_ADD(flags, flag) ((flags) |= (flag)) -#define TDB_FLAG_REMOVE(flags, flag) ((flags) &= (~(flag))) +#define TDB_FLAG_ADD(flags, flag) ((flags) | (flag)) +#define TDB_FLAG_REMOVE(flags, flag) ((flags) & (~(flag))) typedef struct SPager SPager; typedef struct SPCache SPCache; From 9b8a8f12871a85b12d351e4e707c593e3254d770 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:17:26 +0000 Subject: [PATCH 048/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 119 +++++++----------------------- 1 file changed, 25 insertions(+), 94 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 459154e9fb..f649a8e9f6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -603,16 +603,30 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { - SCell *pLCell, pRCell; + SCell *pCell; int szLCell, szRCell; for (;;) { + pCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); + if (childNotLeaf) { - pLCell = pRCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); - szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pLCell); + szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); } else { - // For internal page, find the left cell - // TODO: fill pLCell, pRCell, szLCell, szRCell + szLCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); + + SPage *pPage = infoNews[iNew - 1].pPage; + int oIdx = infoNews[iNew - 1].oIdx + 1; + for (;;) { + if (oIdx < TDB_PAGE_TOTAL_CELLS(pPage)) { + break; + } + + pPage++; + oIdx = 0; + } + + pCell = tdbPageGetCell(pPage, oIdx); + szRCell = tdbBtreeCellSize(pPage, pCell); } ASSERT(infoNews[iNew - 1].cnt > 0); @@ -621,105 +635,22 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { break; } - // move cell divider left - infoNews[iNew].cnt++; - infoNews[iNew].size += szRCell; - { - // TODO - // infoNews[iNew].oPage = ; - // infoNews[iNew].oIdx = ; - } - + // Move a cell right forward infoNews[iNew - 1].cnt--; infoNews[iNew - 1].size -= szLCell; - { - // TODO - // infoNews[iNew-1].oPage = ; - // infoNews[iNew-1].oIdx = ; - } - } - -#if 0 - SCell *pCell; - SPage *pPage; - int nCells; - int cellBytes; - - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - - if (!childNotLeaf) { // child leaf + infoNews[iNew - 1].oIdx--; for (;;) { - pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); - cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - - infoNews[iNew].cnt++; - infoNews[iNew].size += cellBytes; - infoNews[iNew - 1].cnt--; - infoNews[iNew - 1].size -= cellBytes; - if ((infoNews[iNew - 1].oIdx--) == 0) { - infoNews[iNew - 1].oPage--; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); - } - - if (infoNews[iNew].size > infoNews[iNew - 1].size) { + if (infoNews[iNew - 1].oIdx >= 0) { break; } - } - } else { // internal leaf - SCell *pCellRight; - SCell *pCellLeft; - int szCellRight, szCellLeft; - // get pCellLeft, szCellLeft - if (infoNews[iNew - 1].oIdx == nCells) { - ASSERT(infoNews[iNew - 1].oPage < nOlds - 1); - pCellLeft = pDivCell[infoNews[iNew - 1].oPage]; - szCellLeft = szDivCell[infoNews[iNew - 1].oPage]; - } else { - pCellLeft = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); - szCellLeft = tdbBtreeCellSize(pPage, pCellLeft); + infoNews[iNew - 1].pPage--; + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(infoNews[iNew - 1].pPage) - 1; } - // get pCellRight, szCellRight - if (infoNews[iNew - 1].oIdx + 1 < nCells) { - pCellRight = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx + 1); - szCellRight = tdbBtreeCellSize(pPage, pCellRight); - } else { - if (infoNews[iNew - 1].oPage < nOlds - 1) { - pCellRight = pDivCell[infoNews[iNew - 1].oPage]; - szCellRight = szDivCell[infoNews[iNew - 1].oPage]; - } else { - // TODO: what if the next old page is empty? - pCellRight = tdbPageGetCell(pOlds[infoNews[iNew - 1].oPage + 1], 0); - szCellRight = tdbBtreeCellSize(pPage, pCellRight); - } - } - - if (infoNews[iNew - 1].size - szCellLeft - TDB_PAGE_OFFSET_SIZE(pPage) <= - infoNews[iNew].size + szCellRight + TDB_PAGE_OFFSET_SIZE(pPage)) { - break; - } - - // TODO: ASSERT(infoNews[iNew].size < PAGE_MAX_CAPACITY); infoNews[iNew].cnt++; - infoNews[iNew].size += szCellRight; - infoNews[iNew - 1].cnt--; - infoNews[iNew - 1].size -= szCellLeft; - if ((infoNews[iNew - 1].oIdx--) == 0) { - infoNews[iNew - 1].oPage--; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - if (infoNews[iNew - 1].oPage < nOlds - 1) { - infoNews[iNew - 1].oIdx = nCells; - } else { - infoNews[iNew - 1].oIdx = nCells - 1; - } - } + infoNews[iNew].size += szRCell; } -#endif } } From 15fa64ebaab2f3f8237105b83d060cc57c5c3089 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:39:57 +0000 Subject: [PATCH 049/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 177 ++++++++---------------------- 1 file changed, 48 insertions(+), 129 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f649a8e9f6..8912a765fd 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -679,146 +679,65 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // TODO: sort the page according to the page number } - // { // Do the real cell distribution + { // Do the real cell distribution + SPage *pOldsCopy[3]; + SCell *pCell; + int szCell; + SBtreeInitPageArg iarg; + int iNew, nNewCells; - // SPage *pTPage[2]; - // int tPage, tIdx, iOld; - // SCell *pCell; - // int szCell; - // int nCells; - // SCellDecoder cd; - // SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; + iarg.pBt = pBt; + iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); + for (int i = 0; i < nOlds; i++) { + tdbPageCreate(pOlds[0]->pageSize, pOldsCopy + i, NULL, NULL); + tdbBtreeZeroPage(pOldsCopy[i], &iarg); + tdbPageCopy(pOlds[i], pOldsCopy[i]); + } + iNew = 0; + nNewCells = 0; - // for (int i = 0; i < 2; i++) { - // ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); - // if (ret < 0) { - // ASSERT(0); - // } - // } + for (int iOld = 0; iOld < nOlds; iOld++) { + SPage *pPage; - // tPage = 0; - // tIdx = 0; - // iOld = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // if (childNotLeaf) { - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // } + pPage = pOldsCopy[iOld]; - // for (int iNew = 0; iNew < nNews; iNew++) { - // // fill the iNew page - // // TODO: copy current page to tmp space - // tdbBtreeZeroPage(pNews[iNew], &iarg); + for (int oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { + pCell = tdbPageGetCell(pPage, oIdx); + szCell = tdbBtreeCellSize(pPage, pCell); - // for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { - // for (;;) { // loop to find the next available cell - // if (tIdx < nCells) { - // pCell = tdbPageGetCell(pTPage[tPage], tIdx); - // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - // tIdx++; - // break; - // } else { - // if (childNotLeaf) { - // if (iOld < nOlds - 1) { - // pCell = pDivCell[iOld]; - // szCell = szDivCell[iOld]; - // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + ASSERT(nNewCells <= infoNews[iNew].cnt); - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // break; - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // } - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // } - // } - // } + if (nNewCells < infoNews[iNew].cnt) { + tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); + nNewCells++; + } else { + if (childNotLeaf) { + // set current new page right-most child - // tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); - // } + // move to next new page + } else { + // move to next new page - // // fill right-most child pgno if internal page - // if (childNotLeaf) { - // if (tIdx < nCells) { - // pCell = tdbPageGetCell(pTPage[tPage], tIdx); - // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - // tIdx++; - // break; - // } else { - // if (!childNotLeaf) { - // if (iOld < nOlds - 1) { - // pCell = pDivCell[iOld]; - // szCell = szDivCell[iOld]; - // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // insert the cell to the new page + } + } - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // break; - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // } - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // } - // } + if (nNewCells == infoNews[iNew].cnt) { + if (childNotLeaf) { + // TODO + } else { + // TODO + } + } + } + } - // ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; - // } + for (int i = 0; i < nOlds; i++) { + tdbPageDestroy(pOldsCopy[i], NULL, NULL); + } + } - // // insert divider cell into the parent page - // SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - // if (iNew == nNews - 1 && pIntHdr->pgno == 0) { - // pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - // } else { - // tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); - // tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, - // &szCell); - // // TODO: the cell here may be used by pParent as an overflow cell - // tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); - // } - // } - - // for (int i = 0; i < 2; i++) { - // tdbPageDestroy(pTPage[i], NULL, NULL); - // } -} - -return 0; + return 0; } static int tdbBtreeBalance(SBtCursor *pCur) { From f8fc05c76e035de6c0fe2ffa0dbec1eec7b10987 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:48:08 +0000 Subject: [PATCH 050/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8912a765fd..98b9455d13 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -712,13 +712,25 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nNewCells++; } else { if (childNotLeaf) { + // Insert to parent + // set current new page right-most child + ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; // move to next new page + iNew++; + nNewCells = 0; } else { + // TODO: Insert to parent max key and current page number or the right most child + // move to next new page + iNew++; + nNewCells = 0; // insert the cell to the new page + ASSERT(nNewCells < infoNews[iNew].cnt); + tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); + nNewCells++; } } From 7b89447239f1971547d44adfede31d113066784f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:57:15 +0000 Subject: [PATCH 051/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 98b9455d13..c6270d6047 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -733,14 +733,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nNewCells++; } } - - if (nNewCells == infoNews[iNew].cnt) { - if (childNotLeaf) { - // TODO - } else { - // TODO - } - } } } From 09833d2244a0da316a34d3b342513d55fc2a8753 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:21:09 +0000 Subject: [PATCH 052/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 49 ++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c6270d6047..ca50631e90 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -710,28 +710,43 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); nNewCells++; - } else { - if (childNotLeaf) { - // Insert to parent - // set current new page right-most child - ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + // insert parent page + if (!childNotLeaf && nNewCells == infoNews[iNew].cnt) { + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + + if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + } else { + // TODO: + ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + // TODO: pCell here may be inserted as an overflow cell, handle it + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + } // move to next new page iNew++; - nNewCells = 0; - } else { - // TODO: Insert to parent max key and current page number or the right most child - - // move to next new page - iNew++; - nNewCells = 0; - - // insert the cell to the new page - ASSERT(nNewCells < infoNews[iNew].cnt); - tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); - nNewCells++; + nNews = 0; } + } else { + ASSERT(childNotLeaf); + + // set current new page right-most child + ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + + // insert to parent as divider cell + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + + if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + } else { + ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + } + + // move to next new page + iNew++; + nNewCells = 0; } } } From d5a1c7a31f9ea48c7b41a1b7f23f3769b1683123 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:25:57 +0000 Subject: [PATCH 053/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index ca50631e90..a6747e8903 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -685,6 +685,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szCell; SBtreeInitPageArg iarg; int iNew, nNewCells; + SCellDecoder cd; iarg.pBt = pBt; iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); @@ -718,10 +719,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (iNew == nNews - 1 && pIntHdr->pgno == 0) { pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); } else { - // TODO: - ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + tdbBtreeDecodeCell(pPage, pCell, &cd); + // TODO: pCell here may be inserted as an overflow cell, handle it - tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + SCell *pNewCell = malloc(cd.kLen + 9); + int szNewCell; + SPgno pgno; + pgno = TDB_PAGE_PGNO(pNews[iNew]); + tdbBtreeEncodeCell(pPage, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); + tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); } // move to next new page From 72aed8997dfcf56f4a1942b4b58ee651b201bf1a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:39:22 +0000 Subject: [PATCH 054/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a6747e8903..68be6ec8cd 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -493,6 +493,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szDivCell[2]; int sIdx; u8 childNotLeaf; + SPgno rPgno; { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); @@ -547,6 +548,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // the slow defragment process tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } + rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno; } // drop the cells on parent page for (int i = 0; i < nOlds; i++) { @@ -741,14 +743,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; // insert to parent as divider cell - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - - if (iNew == nNews - 1 && pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - } else { - ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); - tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); - } + ASSERT(iNew < nNews - 1); + ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); // move to next new page iNew++; From 795981feae04c4c470768c8de9cff682e10336f4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:54:38 +0000 Subject: [PATCH 055/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 68be6ec8cd..1503d68575 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -737,6 +737,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nNews = 0; } } else { + ASSERT(0); ASSERT(childNotLeaf); // set current new page right-most child From 2bb84eeb2becf4e636d06fff26d9a2443e8903ae Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:59:10 +0000 Subject: [PATCH 056/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 1503d68575..50f585ebd3 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -698,6 +698,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } iNew = 0; nNewCells = 0; + tdbBtreeZeroPage(pNews[iNew], &iarg); for (int iOld = 0; iOld < nOlds; iOld++) { SPage *pPage; @@ -735,6 +736,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; nNews = 0; + tdbBtreeZeroPage(pNews[iNew], &iarg); } } else { ASSERT(0); @@ -751,6 +753,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; nNewCells = 0; + tdbBtreeZeroPage(pNews[iNew], &iarg); } } } From 1fba9d5d3d162085f4dabf76ca79a1d009492c93 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 09:03:01 +0000 Subject: [PATCH 057/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 50f585ebd3..81b8be3588 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -735,8 +735,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; - nNews = 0; - tdbBtreeZeroPage(pNews[iNew], &iarg); + nNewCells = 0; + if (iNew < nNews) { + tdbBtreeZeroPage(pNews[iNew], &iarg); + } } } else { ASSERT(0); @@ -753,7 +755,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; nNewCells = 0; - tdbBtreeZeroPage(pNews[iNew], &iarg); + if (iNew < nNews) { + tdbBtreeZeroPage(pNews[iNew], &iarg); + } } } } From 33e119e9c037c63136bc3729de300e5bc9527e79 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 09:12:21 +0000 Subject: [PATCH 058/105] fix another TDB bug --- source/libs/tdb/src/db/tdbBtree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 81b8be3588..f0055723b6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -729,7 +729,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szNewCell; SPgno pgno; pgno = TDB_PAGE_PGNO(pNews[iNew]); - tdbBtreeEncodeCell(pPage, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); + tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); } From 48306365aed114b632fdf3ce76e7077da130692d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 09:35:25 +0000 Subject: [PATCH 059/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f0055723b6..0f5e6733ed 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -563,10 +563,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nNews = 0; struct { - int cnt; - int size; - SPage *pPage; - int oIdx; + int cnt; + int size; + int iPage; + int oIdx; } infoNews[5] = {0}; { // Get how many new pages are needed and the new distribution @@ -596,7 +596,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; - infoNews[nNews].pPage = pPage; + infoNews[nNews].iPage = oPage; infoNews[nNews].oIdx = oIdx; } } @@ -609,21 +609,23 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szLCell, szRCell; for (;;) { - pCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); + pCell = tdbPageGetCell(pOlds[infoNews[iNew - 1].iPage], infoNews[iNew - 1].oIdx); if (childNotLeaf) { - szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); + szLCell = szRCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); } else { - szLCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); + szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); - SPage *pPage = infoNews[iNew - 1].pPage; + int iPage = infoNews[iNew - 1].iPage; int oIdx = infoNews[iNew - 1].oIdx + 1; + SPage *pPage; for (;;) { + pPage = pOlds[iPage]; if (oIdx < TDB_PAGE_TOTAL_CELLS(pPage)) { break; } - pPage++; + iPage++; oIdx = 0; } @@ -646,8 +648,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { break; } - infoNews[iNew - 1].pPage--; - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(infoNews[iNew - 1].pPage) - 1; + infoNews[iNew - 1].iPage--; + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pOlds[infoNews[iNew - 1].iPage]) - 1; } infoNews[iNew].cnt++; From 27bcf222cec34b67d40b90d33c0d934f6fffb611 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 10:12:43 +0000 Subject: [PATCH 060/105] fix another bug --- source/libs/tdb/src/page/tdbPage.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index ee1a021a1b..5253a1fd67 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -288,8 +288,6 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { } else { TDB_PAGE_FCELL_SET(pPage, pFreeCell - pPage->pData); } - - goto _alloc_finish; } else { if (pPrevFreeCell) { pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, nxFreeCell); @@ -297,6 +295,8 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { TDB_PAGE_FCELL_SET(pPage, nxFreeCell); } } + + goto _alloc_finish; } else { pPrevFreeCell = pFreeCell; szPrevFreeCell = szFreeCell; From b633a96bed37bececfe62551f005a79cedd90644 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 10:17:23 +0000 Subject: [PATCH 061/105] more TDB --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 7bdcc38173..63b53b1732 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -8,7 +8,7 @@ TEST(tdb_test, simple_test) { STDb *pDb; // Open Env - ret = tdbEnvOpen("tdb", 1024, 20, &pEnv); + ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From 2aa6d7e38482a76211f265c6edbc31d52ba7e387 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 10:30:52 +0000 Subject: [PATCH 062/105] start the query --- source/libs/tdb/src/db/tdbDb.c | 5 +++++ source/libs/tdb/src/inc/tdbDb.h | 1 + source/libs/tdb/test/tdbTest.cpp | 35 ++++++++++++++++++++++++-------- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 00f38a19bb..27cbaeaa01 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -85,5 +85,10 @@ int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int v return -1; } + return 0; +} + +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen) { + // TODO return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 06ea74a83e..fdf4c4aca5 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -26,6 +26,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF int tdbDbClose(STDb *pDb); int tdbDbDrop(STDb *pDb); int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 63b53b1732..c5a2787aaf 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,6 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; + int nData = 1000; // Open Env ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); @@ -15,15 +16,33 @@ TEST(tdb_test, simple_test) { ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, NULL, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); - { // Insert some data - char key[64]; - char val[64]; + { + char key[64]; + char val[64]; + void *pVal; + int vLen; - for (int i = 1; i <= 1000; i++) { - sprintf(key, "key%d", i); - sprintf(val, "value%d", i); - ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val)); - GTEST_ASSERT_EQ(ret, 0); + { // Insert some data + + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + sprintf(val, "value%d", i); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val)); + GTEST_ASSERT_EQ(ret, 0); + } + } + + { // Query the data + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + sprintf(val, "value%d", i); + + ret = tdbDbGet(pDb, key, strlen(key), pVal, &vLen); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, strlen(val)); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + } } } From 4383b3b53a45d597b8c2cbcd8a11df46cf923d55 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 02:09:56 +0000 Subject: [PATCH 063/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 28 ++++++++++++++++++++++++++++ source/libs/tdb/src/db/tdbDb.c | 5 ++--- source/libs/tdb/src/inc/tdbBtree.h | 1 + source/libs/tdb/src/inc/tdbDb.h | 2 +- source/libs/tdb/test/tdbTest.cpp | 2 +- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0f5e6733ed..f5715866ed 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -203,6 +203,34 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p return 0; } +int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) { + SBtCursor btc; + SCell *pCell; + int cret; + SCellDecoder cd; + + tdbBtreeCursor(&btc, pBt); + + tdbBtCursorMoveTo(&btc, pKey, kLen, &cret); + + if (cret) { + return cret; + } + + pCell = tdbPageGetCell(btc.pPage, btc.idx); + tdbBtreeDecodeCell(btc.pPage, pCell, &cd); + + *vLen = cd.vLen; + // TODO: here may have memory leak + *ppVal = realloc(*ppVal, *vLen); + if (*ppVal == NULL) { + return -1; + } + + memcpy(*ppVal, cd.pVal, cd.vLen); + return 0; +} + static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { int ret; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 27cbaeaa01..2ba9daaa4a 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -88,7 +88,6 @@ int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int v return 0; } -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen) { - // TODO - return 0; +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { + return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index c1fe77c22e..a0b054a6a8 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -37,6 +37,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SB int tdbBtreeClose(SBTree *pBt); int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt); int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen); +int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index fdf4c4aca5..6a9d653eac 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -26,7 +26,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF int tdbDbClose(STDb *pDb); int tdbDbDrop(STDb *pDb); int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen); +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index c5a2787aaf..3d06d568e4 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -37,7 +37,7 @@ TEST(tdb_test, simple_test) { sprintf(key, "key%d", i); sprintf(val, "value%d", i); - ret = tdbDbGet(pDb, key, strlen(key), pVal, &vLen); + ret = tdbDbGet(pDb, key, strlen(key), &pVal, &vLen); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(vLen, strlen(val)); From 20b1c0d236103f458d356c84d93497dc3858f2bd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 02:27:16 +0000 Subject: [PATCH 064/105] more TDB --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 3d06d568e4..1acd78f188 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,7 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 1000; + int nData = 10000; // Open Env ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); From 820de153a80e5a95e02d91e4dcdf0068103918a9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 05:46:59 +0000 Subject: [PATCH 065/105] fix invalid read --- source/libs/tdb/src/db/tdbBtree.c | 4 ++-- source/libs/tdb/src/page/tdbPage.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f5715866ed..a733c0d981 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -712,7 +712,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } { // Do the real cell distribution - SPage *pOldsCopy[3]; + SPage *pOldsCopy[3] = {0}; SCell *pCell; int szCell; SBtreeInitPageArg iarg; @@ -722,7 +722,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { iarg.pBt = pBt; iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); for (int i = 0; i < nOlds; i++) { - tdbPageCreate(pOlds[0]->pageSize, pOldsCopy + i, NULL, NULL); + tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], NULL, NULL); tdbBtreeZeroPage(pOldsCopy[i], &iarg); tdbPageCopy(pOlds[i], pOldsCopy[i]); } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 5253a1fd67..15b6e218e6 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -52,7 +52,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) } ptr = (u8 *)((*xMalloc)(arg, size)); - if (pPage == NULL) { + if (ptr == NULL) { return -1; } From d59b65fa54d958b2caee0b207687000051e10fc8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 06:00:35 +0000 Subject: [PATCH 066/105] fix memory bug --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 1acd78f188..653efaa19a 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -19,7 +19,7 @@ TEST(tdb_test, simple_test) { { char key[64]; char val[64]; - void *pVal; + void *pVal = NULL; int vLen; { // Insert some data From 08a270777c72cb90c8453a3d734ee085ba1e0e96 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 06:58:52 +0000 Subject: [PATCH 067/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 48 ++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a733c0d981..0bb50fe937 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -517,8 +517,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nOlds; SPage *pOlds[3]; - SCell *pDivCell[2] = {0}; - int szDivCell[2]; + SCell *pDivCell[3] = {0}; + int szDivCell[3]; int sIdx; u8 childNotLeaf; SPgno rPgno; @@ -562,18 +562,18 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // copy the parent key out if child pages are not leaf page childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); if (childNotLeaf) { - for (int i = 0; i < nOlds - 1; i++) { - pCell = tdbPageGetCell(pParent, sIdx + i); + for (int i = 0; i < nOlds; i++) { + if (sIdx + i < TDB_PAGE_TOTAL_CELLS(pParent)) { + pCell = tdbPageGetCell(pParent, sIdx + i); + szDivCell[i] = tdbBtreeCellSize(pParent, pCell); + pDivCell[i] = malloc(szDivCell[i]); + memcpy(pDivCell[i], pCell, szDivCell[i]); + } - szDivCell[i] = tdbBtreeCellSize(pParent, pCell); - pDivCell[i] = malloc(szDivCell[i]); - memcpy(pDivCell, pCell, szDivCell[i]); - - ((SPgno *)pDivCell)[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; - ((SIntHdr *)pOlds[i]->pData)->pgno = 0; - - // here we insert the cell as an overflow cell to avoid - // the slow defragment process + if (i < nOlds - 1) { + ((SPgno *)pDivCell[i])[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; + ((SIntHdr *)pOlds[i]->pData)->pgno = 0; + } tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno; @@ -740,6 +740,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { szCell = tdbBtreeCellSize(pPage, pCell); ASSERT(nNewCells <= infoNews[iNew].cnt); + ASSERT(iNew < nNews - 1); if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); @@ -771,8 +772,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } else { - ASSERT(0); ASSERT(childNotLeaf); + ASSERT(iNew < nNews - 1); // set current new page right-most child ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; @@ -790,6 +791,19 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } + + if (childNotLeaf) { + ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); + ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; + + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[nNews - 1]); + } else { + ((SPgno *)pDivCell[nOlds - 1])[0] = TDB_PAGE_PGNO(pNews[nNews - 1]); + tdbPageInsertCell(pParent, sIdx, pDivCell[nOlds - 1], szDivCell[nOlds - 1], 0); + } + } } for (int i = 0; i < nOlds; i++) { @@ -797,6 +811,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + for (int i = 0; i < 3; i++) { + if (pDivCell[i]) { + free(pDivCell[i]); + } + } + return 0; } From b6d27d3759c55168626212c425b72407211e01c3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 07:43:19 +0000 Subject: [PATCH 068/105] fix another bug --- source/libs/tdb/src/db/tdbBtree.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0bb50fe937..de2ed60ec7 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -516,7 +516,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; int nOlds; - SPage *pOlds[3]; + SPage *pOlds[3] = {0}; SCell *pDivCell[3] = {0}; int szDivCell[3]; int sIdx; @@ -573,8 +573,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (i < nOlds - 1) { ((SPgno *)pDivCell[i])[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; ((SIntHdr *)pOlds[i]->pData)->pgno = 0; + tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } - tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno; } @@ -740,7 +740,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { szCell = tdbBtreeCellSize(pPage, pCell); ASSERT(nNewCells <= infoNews[iNew].cnt); - ASSERT(iNew < nNews - 1); + ASSERT(iNew < nNews); if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); @@ -791,18 +791,18 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } + } - if (childNotLeaf) { - ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); - ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; + if (childNotLeaf) { + ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); + ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - if (pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[nNews - 1]); - } else { - ((SPgno *)pDivCell[nOlds - 1])[0] = TDB_PAGE_PGNO(pNews[nNews - 1]); - tdbPageInsertCell(pParent, sIdx, pDivCell[nOlds - 1], szDivCell[nOlds - 1], 0); - } + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[nNews - 1]); + } else { + ((SPgno *)pDivCell[nOlds - 1])[0] = TDB_PAGE_PGNO(pNews[nNews - 1]); + tdbPageInsertCell(pParent, sIdx, pDivCell[nOlds - 1], szDivCell[nOlds - 1], 0); } } From 493670d2f0ba7c226af5b00b54c8ef4b0447ee0e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 08:11:07 +0000 Subject: [PATCH 069/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index de2ed60ec7..eec75d3a13 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -482,9 +482,11 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { u8 flags; SIntHdr *pIntHdr; SBtreeInitPageArg zArg; + u8 leaf; pPager = pRoot->pPager; flags = TDB_BTREE_PAGE_GET_FLAGS(pRoot); + leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Allocate a new child page zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT); @@ -494,6 +496,10 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } + if (!leaf) { + ((SIntHdr *)pChild->pData)->pgno = ((SIntHdr *)(pRoot->pData))->pgno; + } + // Copy the root page content to the child page tdbPageCopy(pRoot, pChild); From c9fe5301564e14313331b0231b2c344c3961bbad Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 08:16:14 +0000 Subject: [PATCH 070/105] add more test --- source/libs/tdb/test/tdbTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 653efaa19a..5285a5ee94 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 10000; + int nData = 50000; // Open Env - ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); + ret = tdbEnvOpen("tdb", 1024, 2560, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From eb9d3f2196581a1c65534766e582e9dbb7d1e121 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 08:40:37 +0000 Subject: [PATCH 071/105] more TDB --- source/libs/tdb/src/inc/tdbDb.h | 3 ++- source/libs/tdb/test/tdbTest.cpp | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 6a9d653eac..7a65966be3 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -20,7 +20,8 @@ extern "C" { #endif -typedef struct STDb STDb; +typedef struct STDb STDb; +typedef struct STDbC STDbC; int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb); int tdbDbClose(STDb *pDb); diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 5285a5ee94..79532d07b7 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 50000; + int nData = 200000; // Open Env - ret = tdbEnvOpen("tdb", 1024, 2560, &pEnv); + ret = tdbEnvOpen("tdb", 1024, 25600, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -44,6 +44,9 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); } } + + { // Loop to query the data + } } ret = tdbDbDrop(pDb); From 347d1e6f9932ddbdfb2e2d352808cad6552b97df Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 09:03:01 +0000 Subject: [PATCH 072/105] start more fix --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 79532d07b7..cca2082e53 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,7 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 200000; + int nData = 160511; // Open Env ret = tdbEnvOpen("tdb", 1024, 25600, &pEnv); From 61b2f52dca24dd7a7ba66b93194aabfa87704c57 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 09:53:34 +0000 Subject: [PATCH 073/105] now TDB can write 1M records --- source/libs/tdb/src/db/tdbBtree.c | 11 ++++++----- source/libs/tdb/test/tdbTest.cpp | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index eec75d3a13..e244b77f95 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -817,11 +817,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - for (int i = 0; i < 3; i++) { - if (pDivCell[i]) { - free(pDivCell[i]); - } - } + // TODO: here has memory leak + // for (int i = 0; i < 3; i++) { + // if (pDivCell[i]) { + // free(pDivCell[i]); + // } + // } return 0; } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index cca2082e53..0759de54b7 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 160511; + int nData = 1000000; // Open Env - ret = tdbEnvOpen("tdb", 1024, 25600, &pEnv); + ret = tdbEnvOpen("tdb", 4096, 25600, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From e20e13d2dbb9b3234db44b6539a918c6016dcd02 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 10:01:17 +0000 Subject: [PATCH 074/105] more TDB --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 0759de54b7..56284f0271 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,7 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 1000000; + int nData = 2000000; // Open Env ret = tdbEnvOpen("tdb", 4096, 25600, &pEnv); From 488d44a55515152d0982b1a343591343830b4a9c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 10:14:46 +0000 Subject: [PATCH 075/105] w/r 10M K-V pairs --- source/libs/tdb/test/tdbTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 56284f0271..df96a39943 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 2000000; + int nData = 10000000; // Open Env - ret = tdbEnvOpen("tdb", 4096, 25600, &pEnv); + ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From b8ac2d30d5b2658d2dc68560938b07ea55c71b5f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 01:42:03 +0000 Subject: [PATCH 076/105] refact --- source/libs/tdb/src/db/tdbBtree.c | 16 ++++++++-------- source/libs/tdb/src/db/tdbDb.c | 6 +++--- source/libs/tdb/src/inc/tdbBtree.h | 15 ++++++++++----- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e244b77f95..befc021d28 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -67,7 +67,7 @@ typedef struct { u8 *pTmpSpace; } SCellDecoder; -static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst); +static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst); static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); static int tdbBtreeOpenImpl(SBTree *pBt); static int tdbBtreeZeroPage(SPage *pPage, void *arg); @@ -75,7 +75,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg); static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell); static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); -static int tdbBtreeBalance(SBtCursor *pCur); +static int tdbBtreeBalance(SBTC *pCur); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { @@ -131,7 +131,7 @@ int tdbBtreeClose(SBTree *pBt) { return 0; } -int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt) { +int tdbBtreeCursor(SBTC *pCur, SBTree *pBt) { pCur->pBt = pBt; pCur->iPage = -1; pCur->pPage = NULL; @@ -140,7 +140,7 @@ int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt) { return 0; } -int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { +int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { int ret; int idx; SPager *pPager; @@ -204,7 +204,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) { - SBtCursor btc; + SBTC btc; SCell *pCell; int cret; SCellDecoder cd; @@ -231,7 +231,7 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen return 0; } -static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { +static int tdbBtCursorMoveToChild(SBTC *pCur, SPgno pgno) { int ret; pCur->pgStack[pCur->iPage] = pCur->pPage; @@ -248,7 +248,7 @@ static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { return 0; } -static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst) { +static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst) { int ret; SBTree *pBt; SPager *pPager; @@ -827,7 +827,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return 0; } -static int tdbBtreeBalance(SBtCursor *pCur) { +static int tdbBtreeBalance(SBTC *pCur) { int iPage; SPage *pParent; SPage *pPage; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 2ba9daaa4a..9c5fb551f0 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -70,9 +70,9 @@ int tdbDbDrop(STDb *pDb) { } int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { - SBtCursor btc; - SBtCursor *pCur; - int ret; + SBTC btc; + SBTC *pCur; + int ret; pCur = &btc; ret = tdbBtreeCursor(pCur, pDb->pBt); diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index a0b054a6a8..60078f68f4 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -20,10 +20,15 @@ extern "C" { #endif -typedef struct SBTree SBTree; -typedef struct SBtCursor SBtCursor; +typedef struct SBTree SBTree; +typedef struct SBTC SBTC; +typedef struct SBtInfo { + SPgno root; + int nLevel; + int nData; +} SBtInfo; -struct SBtCursor { +struct SBTC { SBTree *pBt; i8 iPage; SPage *pPage; @@ -35,8 +40,8 @@ struct SBtCursor { int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); -int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt); -int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen); +int tdbBtreeCursor(SBTC *pCur, SBTree *pBt); +int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus From 6276f2c33e84e90d5b427a574b444a4413cfacc9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 01:48:35 +0000 Subject: [PATCH 077/105] refact --- source/libs/tdb/src/db/tdbDb.c | 16 ++++++++-------- source/libs/tdb/src/inc/tdbDb.h | 12 ++++++------ source/libs/tdb/src/inc/tdbInt.h | 2 +- source/libs/tdb/test/tdbTest.cpp | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 9c5fb551f0..e889e8772b 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,13 +15,13 @@ #include "tdbInt.h" -struct STDb { +struct STDB { STEnv *pEnv; SBTree *pBt; }; -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb) { - STDb *pDb; +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb) { + STDB *pDb; SPager *pPager; int ret; char fFullName[TDB_FILENAME_LEN]; @@ -30,7 +30,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF *ppDb = NULL; - pDb = (STDb *)calloc(1, sizeof(*pDb)); + pDb = (STDB *)calloc(1, sizeof(*pDb)); if (pDb == NULL) { return -1; } @@ -59,17 +59,17 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF return 0; } -int tdbDbClose(STDb *pDb) { +int tdbDbClose(STDB *pDb) { // TODO return 0; } -int tdbDbDrop(STDb *pDb) { +int tdbDbDrop(STDB *pDb) { // TODO return 0; } -int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { +int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { SBTC btc; SBTC *pCur; int ret; @@ -88,6 +88,6 @@ int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int v return 0; } -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { +int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 7a65966be3..e48de5e8c7 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -20,14 +20,14 @@ extern "C" { #endif -typedef struct STDb STDb; +typedef struct STDB STDB; typedef struct STDbC STDbC; -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb); -int tdbDbClose(STDb *pDb); -int tdbDbDrop(STDb *pDb); -int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb); +int tdbDbClose(STDB *pDb); +int tdbDbDrop(STDB *pDb); +int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); +int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 3ebba0a4b5..98845bb66f 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -95,7 +95,7 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) { // tdb_log #define tdbError(var) -typedef TD_DLIST(STDb) STDbList; +typedef TD_DLIST(STDB) STDbList; typedef TD_DLIST(SPgFile) SPgFileList; typedef TD_DLIST_NODE(SPgFile) SPgFileListNode; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index df96a39943..4f6bde3c96 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -5,7 +5,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; - STDb *pDb; + STDB *pDb; int nData = 10000000; // Open Env From 1cd404151aac734c9fcfcf96e0c04927dde4d0fe Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Mar 2022 11:44:40 +0800 Subject: [PATCH 078/105] minor changes --- source/dnode/mgmt/container/inc/dndInt.h | 1 - source/dnode/mgmt/container/src/dndExec.c | 2 +- source/dnode/mgmt/container/src/dndFile.c | 28 +++++++++---------- source/dnode/mgmt/container/src/dndMsg.c | 11 ++++---- .../dnode/mgmt/container/src/dndTransport.c | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/source/dnode/mgmt/container/inc/dndInt.h b/source/dnode/mgmt/container/inc/dndInt.h index d10835b67f..8ea496b2fb 100644 --- a/source/dnode/mgmt/container/inc/dndInt.h +++ b/source/dnode/mgmt/container/inc/dndInt.h @@ -56,7 +56,6 @@ void dndCleanupServer(SDnode *pDnode); int32_t dndInitClient(SDnode *pDnode); void dndCleanupClient(SDnode *pDnode); int32_t dndInitMsgHandle(SDnode *pDnode); -void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c index 6d07ca1898..1355c4543a 100644 --- a/source/dnode/mgmt/container/src/dndExec.c +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -138,7 +138,7 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t msgLen, void *pCont, int32_t contLen) { dTrace("msg:%p, get from parent queue", pRsp); pRsp->pCont = pCont; - dndSendRpcRsp(pWrapper, pRsp); + dndSendRsp(pWrapper, pRsp); free(pRsp); } diff --git a/source/dnode/mgmt/container/src/dndFile.c b/source/dnode/mgmt/container/src/dndFile.c index a5f8fb7169..ab04040b57 100644 --- a/source/dnode/mgmt/container/src/dndFile.c +++ b/source/dnode/mgmt/container/src/dndFile.c @@ -16,14 +16,16 @@ #define _DEFAULT_SOURCE #include "dndInt.h" +#define MAXLEN 1024 + int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) { - int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - char file[PATH_MAX]; - TdFilePtr pFile = NULL; + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + const int32_t maxLen = MAXLEN; + char content[MAXLEN + 1] = {0}; + cJSON *root = NULL; + char file[PATH_MAX]; + TdFilePtr pFile = NULL; snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); pFile = taosOpenFile(file, TD_FILE_READ); @@ -57,7 +59,6 @@ int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) { dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); _OVER: - if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); @@ -66,7 +67,7 @@ _OVER: } int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) { - char file[PATH_MAX]; + char file[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); @@ -76,9 +77,9 @@ int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) { return -1; } - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); + int32_t len = 0; + const int32_t maxLen = MAXLEN; + char content[MAXLEN + 1] = {0}; len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); @@ -87,9 +88,8 @@ int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) { taosWriteFile(pFile, content, len); taosFsyncFile(pFile); taosCloseFile(&pFile); - free(content); - char realfile[PATH_MAX]; + char realfile[PATH_MAX] = {0}; snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); if (taosRenameFile(file, realfile) != 0) { diff --git a/source/dnode/mgmt/container/src/dndMsg.c b/source/dnode/mgmt/container/src/dndMsg.c index 37ff4ebc05..bf3200ffaf 100644 --- a/source/dnode/mgmt/container/src/dndMsg.c +++ b/source/dnode/mgmt/container/src/dndMsg.c @@ -43,19 +43,18 @@ static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); - return 0; } void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { - if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { - dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); - } - int32_t code = -1; SNodeMsg *pMsg = NULL; NodeMsgFp msgFp = NULL; + if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { + dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); + } + if (dndMarkWrapper(pWrapper) != 0) goto _OVER; if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; @@ -67,6 +66,8 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { } else if (pWrapper->procType == PROC_PARENT) { code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen); } else { + dFatal(" should not be entered in child processes"); + ASSERT(1); } _OVER: diff --git a/source/dnode/mgmt/container/src/dndTransport.c b/source/dnode/mgmt/container/src/dndTransport.c index 4acb1f459e..b7d0cf26c0 100644 --- a/source/dnode/mgmt/container/src/dndTransport.c +++ b/source/dnode/mgmt/container/src/dndTransport.c @@ -348,7 +348,7 @@ int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { } } -void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { +static void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { if (pRsp->code == TSDB_CODE_APP_NOT_READY) { SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE); if (pDnodeWrapper != NULL) { From 212e24dda649b879decfd7bf20c0bb03c7065475 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 04:59:20 +0000 Subject: [PATCH 079/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 1 + source/libs/tdb/test/tdbTest.cpp | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index befc021d28..c3adf7d2b6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -914,6 +914,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, u8 *pPayload, const void *pKey, i return 0; } +// TODO: allow vLen = 0 static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell) { u8 flags; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 4f6bde3c96..05a0f3a6ae 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -17,10 +17,8 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(ret, 0); { - char key[64]; - char val[64]; - void *pVal = NULL; - int vLen; + char key[64]; + char val[64]; { // Insert some data @@ -33,6 +31,9 @@ TEST(tdb_test, simple_test) { } { // Query the data + void *pVal = NULL; + int vLen; + for (int i = 1; i <= nData; i++) { sprintf(key, "key%d", i); sprintf(val, "value%d", i); @@ -46,6 +47,8 @@ TEST(tdb_test, simple_test) { } { // Loop to query the data + // STDbC *pDBC; + // tdbDBNext(pDBC, p) } } From f4359a6fb3c82295ac5b8b752e2804b7dd19ad84 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 06:02:38 +0000 Subject: [PATCH 080/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 7 ++++--- source/libs/tdb/src/inc/tdbUtil.h | 22 ++++++++++++++++++++++ source/libs/tdb/test/tdbTest.cpp | 2 ++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c3adf7d2b6..0ef5d64d30 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -207,6 +207,7 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen SBTC btc; SCell *pCell; int cret; + void *pVal; SCellDecoder cd; tdbBtreeCursor(&btc, pBt); @@ -221,12 +222,12 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen tdbBtreeDecodeCell(btc.pPage, pCell, &cd); *vLen = cd.vLen; - // TODO: here may have memory leak - *ppVal = realloc(*ppVal, *vLen); - if (*ppVal == NULL) { + pVal = TDB_REALLOC(*ppVal, *vLen); + if (pVal == NULL) { return -1; } + *ppVal = pVal; memcpy(*ppVal, cd.pVal, cd.vLen); return 0; } diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 9358aae236..88fc846bf1 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -39,6 +39,28 @@ int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); int tdbPRead(int fd, void *pData, int count, i64 offset); +#define TDB_REALLOC(PTR, SIZE) \ + ({ \ + void *nPtr; \ + if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ + nPtr = realloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \ + if (nPtr) { \ + ((int *)nPtr)[0] = (SIZE); \ + nPtr = (char *)nPtr + sizeof(int); \ + } \ + } else { \ + nPtr = (PTR); \ + } \ + nPtr; \ + }) + +#define TDB_FREE(PTR) \ + do { \ + if (PTR) { \ + free((char *)(PTR) - sizeof(int)); \ + } \ + } while (0) + static inline void *tdbOsMalloc(void *arg, size_t size) { void *ptr; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 05a0f3a6ae..46437392cd 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -44,6 +44,8 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(vLen, strlen(val)); GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); } + + TDB_FREE(pVal); } { // Loop to query the data From 9ffdf09986e1ee511d12b97e4e39531ef63fa9df Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 06:15:21 +0000 Subject: [PATCH 081/105] design dbc api --- source/libs/tdb/src/db/tdbDb.c | 15 +++++++++++++++ source/libs/tdb/src/inc/tdbDb.h | 8 +++++++- source/libs/tdb/test/tdbTest.cpp | 25 ++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index e889e8772b..98f22e4d25 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -90,4 +90,19 @@ int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int v int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); +} + +int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc) { + // TODO + return 0; +} + +int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { + // TODO + return 0; +} + +int tdbDbcClose(STDBC *pDbc) { + // TODO + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index e48de5e8c7..4e69de2bfa 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -21,14 +21,20 @@ extern "C" { #endif typedef struct STDB STDB; -typedef struct STDbC STDbC; +typedef struct STDBC STDBC; +// STDB int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb); int tdbDbClose(STDB *pDb); int tdbDbDrop(STDB *pDb); int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); +// STDBC +int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc); +int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbDbcClose(STDBC *pDbc); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 46437392cd..164b2c0b83 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -48,9 +48,28 @@ TEST(tdb_test, simple_test) { TDB_FREE(pVal); } - { // Loop to query the data - // STDbC *pDBC; - // tdbDBNext(pDBC, p) + { // Iterate to query the DB data + STDBC *pDBC; + void *pKey = NULL; + void *pVal = NULL; + int vLen, kLen; + int count = 0; + + ret = tdbDbcOpen(pDb, &pDBC); + GTEST_ASSERT_EQ(ret, 0); + + for (;;) { + ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + count++; + } + + GTEST_ASSERT_EQ(count, nData); + + tdbDbcClose(pDBC); + + TDB_FREE(pKey); + TDB_FREE(pVal); } } From cc176b1037ebed74221b6f6d99f63775d2f80987 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 06:42:42 +0000 Subject: [PATCH 082/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 29 +++++++++++++++++++++-- source/libs/tdb/src/db/tdbDb.c | 37 +++++++++++++++++++++++++----- source/libs/tdb/src/inc/tdbBtree.h | 10 +++++++- source/libs/tdb/src/inc/tdbDb.h | 2 +- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0ef5d64d30..2b0ba62ed3 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -131,7 +131,7 @@ int tdbBtreeClose(SBTree *pBt) { return 0; } -int tdbBtreeCursor(SBTC *pCur, SBTree *pBt) { +int tdbBtcOpen(SBTC *pCur, SBTree *pBt) { pCur->pBt = pBt; pCur->iPage = -1; pCur->pPage = NULL; @@ -210,7 +210,7 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen void *pVal; SCellDecoder cd; - tdbBtreeCursor(&btc, pBt); + tdbBtcOpen(&btc, pBt); tdbBtCursorMoveTo(&btc, pKey, kLen, &cret); @@ -1072,3 +1072,28 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { } #endif + +int tdbBtcMoveToFirst(SBTC *pBtc) { + // TODO + return 0; +} + +int tdbBtcMoveToLast(SBTC *pBtc) { + // TODO + return 0; +} + +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen) { + // TODO + return 0; +} + +int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { + // TODO + return 0; +} + +int tdbBtcClose(SBTC *pBtc) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 98f22e4d25..4e74dc4cbb 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -20,6 +20,10 @@ struct STDB { SBTree *pBt; }; +struct STDBC { + SBTC btc; +}; + int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb) { STDB *pDb; SPager *pPager; @@ -75,7 +79,7 @@ int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int v int ret; pCur = &btc; - ret = tdbBtreeCursor(pCur, pDb->pBt); + ret = tdbBtcOpen(pCur, pDb->pBt); if (ret < 0) { return -1; } @@ -92,17 +96,38 @@ int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } -int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc) { - // TODO +int tdbDbcOpen(STDB *pDb, STDBC **ppDbc) { + int ret; + STDBC *pDbc = NULL; + + *ppDbc = NULL; + pDbc = malloc(sizeof(*pDbc)); + if (pDbc == NULL) { + return -1; + } + + tdbBtcOpen(&pDbc->btc, pDb->pBt); + + // TODO: move to first now, we can move to any key-value + // and in any direction, design new APIs. + ret = tdbBtcMoveToFirst(&pDbc->btc); + if (ret < 0) { + ASSERT(0); + return -1; + } + + *ppDbc = pDbc; return 0; } int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { - // TODO - return 0; + return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); } int tdbDbcClose(STDBC *pDbc) { - // TODO + if (pDbc) { + free(pDbc); + } + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index 60078f68f4..e46e8bb78b 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -38,12 +38,20 @@ struct SBTC { void *pBuf; }; +// SBTree int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); -int tdbBtreeCursor(SBTC *pCur, SBTree *pBt); int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); +// SBTC +int tdbBtcOpen(SBTC *pCur, SBTree *pBt); +int tdbBtcMoveToFirst(SBTC *pBtc); +int tdbBtcMoveToLast(SBTC *pBtc); +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen); +int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbBtcClose(SBTC *pBtc); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 4e69de2bfa..b96076b826 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -31,7 +31,7 @@ int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int v int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); // STDBC -int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc); +int tdbDbcOpen(STDB *pDb, STDBC **ppDbc); int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbDbcClose(STDBC *pDbc); From 07f6d03a5246aaced36ee3272c6852f703779871 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 07:19:48 +0000 Subject: [PATCH 083/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 107 +++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 2b0ba62ed3..7d925d69f4 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -131,15 +131,6 @@ int tdbBtreeClose(SBTree *pBt) { return 0; } -int tdbBtcOpen(SBTC *pCur, SBTree *pBt) { - pCur->pBt = pBt; - pCur->iPage = -1; - pCur->pPage = NULL; - pCur->idx = -1; - - return 0; -} - int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { int ret; int idx; @@ -1073,13 +1064,107 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { #endif +int tdbBtcOpen(SBTC *pCur, SBTree *pBt) { + pCur->pBt = pBt; + pCur->iPage = -1; + pCur->pPage = NULL; + pCur->idx = -1; + + return 0; +} + int tdbBtcMoveToFirst(SBTC *pBtc) { - // TODO + int ret; + SBTree *pBt; + SPager *pPager; + u8 flags; + SCell *pCell; + SPgno pgno; + + pBt = pBtc->pBt; + pPager = pBt->pPager; + + if (pBtc->iPage < 0) { + // move a clean cursor + ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->iPage = 0; + pBtc->idx = 0; + } else { + // move from a position + ASSERT(0); + } + + // move downward + for (;;) { + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + + if (TDB_BTREE_PAGE_IS_LEAF(flags)) break; + + pCell = tdbPageGetCell(pBtc->pPage, 0); + pgno = *(SPgno *)pCell; + + ret = tdbBtCursorMoveToChild(pBtc, pgno); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->idx = 0; + } + return 0; } int tdbBtcMoveToLast(SBTC *pBtc) { - // TODO + int ret; + SBTree *pBt; + SPager *pPager; + u8 flags; + SPgno pgno; + + pBt = pBtc->pBt; + pPager = pBt->pPager; + + if (pBtc->iPage < 0) { + // move a clean cursor + ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->iPage = 0; + } else { + // move from a position + ASSERT(0); + } + + // move downward + for (;;) { + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + + if (TDB_BTREE_PAGE_IS_LEAF(flags)) { + // TODO: handle empty case + ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0); + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1; + break; + } else { + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; + + ret = tdbBtCursorMoveToChild(pBtc, pgno); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + } + return 0; } From f75cb898663afc37a7c45b964d4b16e462c8d663 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 07:37:41 +0000 Subject: [PATCH 084/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 63 ++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 7d925d69f4..8c86c576c2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -77,6 +77,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); static int tdbBtreeBalance(SBTC *pCur); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); +static int tdbBtcMoveToNext(SBTC *pBtc); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; @@ -1174,7 +1175,67 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen) { } int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { - // TODO + SCell *pCell; + SCellDecoder cd; + void *pKey, *pVal; + int ret; + + if (0) { + // TODO: no valid data current cursor is pointing to + return -1; + } + + pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); + + tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd); + + pKey = TDB_REALLOC(*ppKey, cd.kLen); + if (pKey == NULL) { + return -1; + } + + // TODO: vLen may be zero + pVal = TDB_REALLOC(*ppVal, cd.vLen); + if (pVal == NULL) { + TDB_FREE(pKey); + return -1; + } + + *ppKey = pKey; + *ppVal = pVal; + + *kLen = cd.kLen; + *vLen = cd.vLen; + + memcpy(pKey, cd.pKey, cd.kLen); + memcpy(pVal, cd.pVal, cd.vLen); + + ret = tdbBtcMoveToNext(pBtc); + + return 0; +} + +static int tdbBtcMoveToNext(SBTC *pBtc) { + pBtc->idx++; + + if (pBtc->idx < 0) return -1; + + if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { + return 0; + } + + // Move upward + for (;;) { + // TODO: release the page + + /* code */ + } + + // Move downward + for (;;) { + /* code */ + } + return 0; } From af98aa400509287f1f196754f69186bbd0bf582b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 07:48:55 +0000 Subject: [PATCH 085/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 51 ++++++++++++++++------------- source/libs/tdb/src/db/tdbPCache.c | 47 ++++++++++---------------- source/libs/tdb/src/db/tdbPager.c | 4 +++ source/libs/tdb/src/inc/tdbPCache.h | 25 +++++++------- source/libs/tdb/src/inc/tdbPager.h | 19 ++++++----- 5 files changed, 73 insertions(+), 73 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8c86c576c2..282c5a98a6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -78,6 +78,8 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD static int tdbBtreeBalance(SBTC *pCur); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); static int tdbBtcMoveToNext(SBTC *pBtc); +static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno); +static int tdbBtcMoveUpward(SBTC *pBtc); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; @@ -224,23 +226,6 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen return 0; } -static int tdbBtCursorMoveToChild(SBTC *pCur, SPgno pgno) { - int ret; - - pCur->pgStack[pCur->iPage] = pCur->pPage; - pCur->idxStack[pCur->iPage] = pCur->idx; - pCur->iPage++; - pCur->pPage = NULL; - pCur->idx = -1; - - ret = tdbPagerFetchPage(pCur->pBt->pPager, pgno, &pCur->pPage, tdbBtreeInitPage, pCur->pBt); - if (ret < 0) { - ASSERT(0); - } - - return 0; -} - static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst) { int ret; SBTree *pBt; @@ -318,16 +303,16 @@ static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst) } else { if (c <= 0) { pCur->idx = midx; - tdbBtCursorMoveToChild(pCur, cd.pgno); + tdbBtcMoveDownward(pCur, cd.pgno); } else { pCur->idx = midx + 1; if (midx == nCells - 1) { /* Move to right-most child */ - tdbBtCursorMoveToChild(pCur, ((SIntHdr *)pCur->pPage->pData)->pgno); + tdbBtcMoveDownward(pCur, ((SIntHdr *)pCur->pPage->pData)->pgno); } else { pCell = tdbPageGetCell(pPage, pCur->idx); tdbBtreeDecodeCell(pPage, pCell, &cd); - tdbBtCursorMoveToChild(pCur, cd.pgno); + tdbBtcMoveDownward(pCur, cd.pgno); } } } @@ -1109,7 +1094,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { pCell = tdbPageGetCell(pBtc->pPage, 0); pgno = *(SPgno *)pCell; - ret = tdbBtCursorMoveToChild(pBtc, pgno); + ret = tdbBtcMoveDownward(pBtc, pgno); if (ret < 0) { ASSERT(0); return -1; @@ -1158,7 +1143,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; - ret = tdbBtCursorMoveToChild(pBtc, pgno); + ret = tdbBtcMoveDownward(pBtc, pgno); if (ret < 0) { ASSERT(0); return -1; @@ -1242,4 +1227,26 @@ static int tdbBtcMoveToNext(SBTC *pBtc) { int tdbBtcClose(SBTC *pBtc) { // TODO return 0; +} + +static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno) { + int ret; + + pCur->pgStack[pCur->iPage] = pCur->pPage; + pCur->idxStack[pCur->iPage] = pCur->idx; + pCur->iPage++; + pCur->pPage = NULL; + pCur->idx = -1; + + ret = tdbPagerFetchPage(pCur->pBt->pPager, pgno, &pCur->pPage, tdbBtreeInitPage, pCur->pBt); + if (ret < 0) { + ASSERT(0); + } + + return 0; +} + +static int tdbBtcMoveUpward(SBTC *pBtc) { + // TODO + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 1fc0847b38..3c7d037faa 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -53,10 +53,10 @@ static void tdbPCacheLock(SPCache *pCache); static void tdbPCacheUnlock(SPCache *pCache); static bool tdbPCacheLocked(SPCache *pCache); static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); -static void tdbPCachePinPage(SPage *pPage); -static void tdbPCacheRemovePageFromHash(SPage *pPage); -static void tdbPCacheAddPageToHash(SPage *pPage); -static void tdbPCacheUnpinPage(SPage *pPage); +static void tdbPCachePinPage(SPCache *pCache, SPage *pPage); +static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage); +static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage); +static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage); int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { SPCache *pCache; @@ -100,7 +100,7 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) { return pPage; } -void tdbPCacheRelease(SPage *pPage) { +void tdbPCacheRelease(SPCache *pCache, SPage *pPage) { i32 nRef; nRef = TDB_UNREF_PAGE(pPage); @@ -108,7 +108,7 @@ void tdbPCacheRelease(SPage *pPage) { if (nRef == 0) { if (1 /*TODO: page still clean*/) { - tdbPCacheUnpinPage(pPage); + tdbPCacheUnpinPage(pCache, pPage); } else { // TODO ASSERT(0); @@ -142,7 +142,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe if (pPage || !alcNewPage) { if (pPage) { - tdbPCachePinPage(pPage); + tdbPCachePinPage(pCache, pPage); } return pPage; } @@ -158,8 +158,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe // 3. Try to Recycle a page if (!pPage && !pCache->lru.pLruPrev->isAnchor) { pPage = pCache->lru.pLruPrev; - tdbPCacheRemovePageFromHash(pPage); - tdbPCachePinPage(pPage); + tdbPCacheRemovePageFromHash(pCache, pPage); + tdbPCachePinPage(pCache, pPage); } // 4. Try a stress allocation (TODO) @@ -171,16 +171,13 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid)); pPage->pLruNext = NULL; pPage->pPager = NULL; - tdbPCacheAddPageToHash(pPage); + tdbPCacheAddPageToHash(pCache, pPage); } return pPage; } -static void tdbPCachePinPage(SPage *pPage) { - SPCache *pCache; - - pCache = pPage->pCache; +static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { if (!PAGE_IS_PINNED(pPage)) { pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; @@ -190,11 +187,8 @@ static void tdbPCachePinPage(SPage *pPage) { } } -static void tdbPCacheUnpinPage(SPage *pPage) { - SPCache *pCache; - i32 nRef; - - pCache = pPage->pCache; +static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { + i32 nRef; tdbPCacheLock(pCache); @@ -215,12 +209,10 @@ static void tdbPCacheUnpinPage(SPage *pPage) { tdbPCacheUnlock(pCache); } -static void tdbPCacheRemovePageFromHash(SPage *pPage) { - SPCache *pCache; - SPage **ppPage; - int h; +static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { + SPage **ppPage; + int h; - pCache = pPage->pCache; h = PCACHE_PAGE_HASH(&(pPage->pgid)); for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) ; @@ -230,11 +222,9 @@ static void tdbPCacheRemovePageFromHash(SPage *pPage) { pCache->nPage--; } -static void tdbPCacheAddPageToHash(SPage *pPage) { - SPCache *pCache; - int h; +static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { + int h; - pCache = pPage->pCache; h = PCACHE_PAGE_HASH(&(pPage->pgid)) % pCache->nHash; pPage->pHashNext = pCache->pgHash[h]; @@ -264,7 +254,6 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { // pPage->pgid = 0; pPage->isAnchor = 0; pPage->isLocalPage = 1; - pPage->pCache = pCache; TDB_INIT_PAGE_REF(pPage); pPage->pHashNext = NULL; pPage->pLruNext = NULL; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index ac0bc15e1d..fe4b9aa123 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -255,6 +255,10 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage return 0; } +void tdbPagerReturnPage(SPager *pPager, SPage *pPage) { + tdbPCacheRelease(pPager->pCache, pPage); +} + static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { // TODO: Allocate a page from the free list return 0; diff --git a/source/libs/tdb/src/inc/tdbPCache.h b/source/libs/tdb/src/inc/tdbPCache.h index ff4f1acbb6..c7fa155615 100644 --- a/source/libs/tdb/src/inc/tdbPCache.h +++ b/source/libs/tdb/src/inc/tdbPCache.h @@ -21,23 +21,22 @@ extern "C" { #endif #define TDB_PCACHE_PAGE \ - u8 isAnchor; \ - u8 isLocalPage; \ - u8 isDirty; \ - i32 nRef; \ - SPCache *pCache; \ - SPage *pFreeNext; \ - SPage *pHashNext; \ - SPage *pLruNext; \ - SPage *pLruPrev; \ - SPage *pDirtyNext; \ - SPager *pPager; \ - SPgid pgid; + u8 isAnchor; \ + u8 isLocalPage; \ + u8 isDirty; \ + i32 nRef; \ + SPage *pFreeNext; \ + SPage *pHashNext; \ + SPage *pLruNext; \ + SPage *pLruPrev; \ + SPage *pDirtyNext; \ + SPager *pPager; \ + SPgid pgid; int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache); int tdbPCacheClose(SPCache *pCache); SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); -void tdbPCacheRelease(SPage *pPage); +void tdbPCacheRelease(SPCache *pCache, SPage *pPage); int tdbPCacheGetPageSize(SPCache *pCache); #ifdef __cplusplus diff --git a/source/libs/tdb/src/inc/tdbPager.h b/source/libs/tdb/src/inc/tdbPager.h index e4ed8552fd..f4cc822f27 100644 --- a/source/libs/tdb/src/inc/tdbPager.h +++ b/source/libs/tdb/src/inc/tdbPager.h @@ -20,15 +20,16 @@ extern "C" { #endif -int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); -int tdbPagerClose(SPager *pPager); -int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); -int tdbPagerWrite(SPager *pPager, SPage *pPage); -int tdbPagerBegin(SPager *pPager); -int tdbPagerCommit(SPager *pPager); -int tdbPagerGetPageSize(SPager *pPager); -int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); -int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); +int tdbPagerClose(SPager *pPager); +int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); +int tdbPagerWrite(SPager *pPager, SPage *pPage); +int tdbPagerBegin(SPager *pPager); +int tdbPagerCommit(SPager *pPager); +int tdbPagerGetPageSize(SPager *pPager); +int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +void tdbPagerReturnPage(SPager *pPager, SPage *pPage); #ifdef __cplusplus } From b6c6a11972d01c80c8d57a77b8f5e7610022885f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Mar 2022 16:10:01 +0800 Subject: [PATCH 086/105] minor changes --- source/dnode/mgmt/container/src/dndMsg.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/container/src/dndMsg.c b/source/dnode/mgmt/container/src/dndMsg.c index bf3200ffaf..b72d085861 100644 --- a/source/dnode/mgmt/container/src/dndMsg.c +++ b/source/dnode/mgmt/container/src/dndMsg.c @@ -60,20 +60,23 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; - dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); if (pWrapper->procType == PROC_SINGLE) { + dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); code = (*msgFp)(pWrapper->pMgmt, pMsg); } else if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is created and will put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, + pRpc->ahandle, pMsg->user); code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen); } else { - dFatal(" should not be entered in child processes"); + dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, + pMsg->user); ASSERT(1); } _OVER: if (code == 0) { if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is freed", pMsg); + dTrace("msg:%p, is freed in parent process", pMsg); taosFreeQitem(pMsg); rpcFreeCont(pRpc->pCont); } From cbc8720d293a05ecb3fec048c45bd62d8f470b72 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 08:35:40 +0000 Subject: [PATCH 087/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 51 ++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 282c5a98a6..d135eedb7c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1201,24 +1201,58 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { } static int tdbBtcMoveToNext(SBTC *pBtc) { - pBtc->idx++; + int nCells; + SPgno pgno; + SCell *pCell; + u8 flags; + + ASSERT(TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage))); if (pBtc->idx < 0) return -1; + pBtc->idx++; if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { return 0; } + if (pBtc->iPage == 0) { + pBtc->idx = -1; + return 0; + } + // Move upward for (;;) { - // TODO: release the page + tdbBtcMoveUpward(pBtc); + pBtc->idx++; - /* code */ + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + if (pBtc->idx <= nCells) { + break; + } + + if (pBtc->iPage == 0) { + pBtc->idx = -1; + return 0; + } } // Move downward for (;;) { - /* code */ + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + if (pBtc->idx < nCells) { + pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); + pgno = *(SPgno *)pCell; + } else { + pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; + } + + tdbBtcMoveDownward(pBtc, pgno); + pBtc->idx = 0; + + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + if (TDB_BTREE_PAGE_IS_LEAF(flags)) { + break; + } } return 0; @@ -1247,6 +1281,13 @@ static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno) { } static int tdbBtcMoveUpward(SBTC *pBtc) { - // TODO + if (pBtc->iPage == 0) return -1; + + // tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage); + + pBtc->iPage--; + pBtc->pPage = pBtc->pgStack[pBtc->iPage]; + pBtc->idx = pBtc->idxStack[pBtc->iPage]; + return 0; } \ No newline at end of file From a9acfce8860d6174a734262e010f8108b469b406 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 08:41:00 +0000 Subject: [PATCH 088/105] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index d135eedb7c..de77fb2649 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1165,8 +1165,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { void *pKey, *pVal; int ret; - if (0) { - // TODO: no valid data current cursor is pointing to + if (pBtc->idx < 0) { return -1; } From f47953797845ae9d17548ef457cee4cb40b3e57f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 09:01:09 +0000 Subject: [PATCH 089/105] more --- source/libs/tdb/test/tdbTest.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 164b2c0b83..b9ae2dff7d 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -2,6 +2,8 @@ #include "tdbInt.h" +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); + TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; @@ -61,6 +63,11 @@ TEST(tdb_test, simple_test) { for (;;) { ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; + + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; + count++; } @@ -82,4 +89,19 @@ TEST(tdb_test, simple_test) { // Close Env ret = tdbEnvClose(pEnv); GTEST_ASSERT_EQ(ret, 0); +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + k1 = std::strtol((char *)pKey1 + 3, nullptr, kLen1 - 3); + k2 = std::strtol((char *)pKey2 + 3, nullptr, kLen2 - 3); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } } \ No newline at end of file From 474939ee4976ec4f8ec538d124363fd0e8cec61b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Mar 2022 17:28:38 +0800 Subject: [PATCH 090/105] shm --- source/dnode/mgmt/container/src/dndExec.c | 1 - source/util/src/tprocess.c | 78 +++++++++++------------ 2 files changed, 38 insertions(+), 41 deletions(-) diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c index 1355c4543a..4dcf7e80ec 100644 --- a/source/dnode/mgmt/container/src/dndExec.c +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -178,7 +178,6 @@ static int32_t dndRunInMultiProcess(SDnode *pDnode) { .parentFreeHeadFp = (ProcFreeFp)free, .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .testFlag = 0, .pParent = pWrapper, .name = pWrapper->name}; SProcObj *pProc = taosProcInit(&cfg); diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index f19a17fdb6..24f3f8a6e4 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -56,7 +56,6 @@ typedef struct SProcObj { int32_t pid; bool isChild; bool stopFlag; - bool testFlag; } SProcObj; static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) { @@ -77,7 +76,7 @@ static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) { goto _OVER; } - shmid = shmget(IPC_PRIVATE, sizeof(TdThreadMutex), 0600); + shmid = shmget(IPC_PRIVATE, sizeof(TdThreadMutex), IPC_CREAT | 0600); if (shmid <= 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to init mutex while shmget since %s", terrstr()); @@ -101,8 +100,13 @@ static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) { _OVER: if (code != 0) { - taosThreadMutexDestroy(pMutex); - shmctl(shmid, IPC_RMID, NULL); + if (pMutex != NULL) { + taosThreadMutexDestroy(pMutex); + shmdt(pMutex); + } + if (shmid >= 0) { + shmctl(shmid, IPC_RMID, NULL); + } } else { *ppMutex = pMutex; *pShmid = shmid; @@ -112,12 +116,12 @@ _OVER: return code; } -static void taosProcDestroyMutex(TdThreadMutex *pMutex, int32_t *pShmid) { +static void taosProcDestroyMutex(TdThreadMutex *pMutex, int32_t shmid) { if (pMutex != NULL) { taosThreadMutexDestroy(pMutex); } - if (*pShmid > 0) { - shmctl(*pShmid, IPC_RMID, NULL); + if (shmid >= 0) { + shmctl(shmid, IPC_RMID, NULL); } } @@ -141,9 +145,10 @@ static int32_t taosProcInitBuffer(void **ppBuffer, int32_t size) { return shmid; } -static void taosProcDestroyBuffer(void *pBuffer, int32_t *pShmid) { - if (*pShmid > 0) { - shmctl(*pShmid, IPC_RMID, NULL); +static void taosProcDestroyBuffer(void *pBuffer, int32_t shmid) { + if (shmid > 0) { + shmdt(pBuffer); + shmctl(shmid, IPC_RMID, NULL); } } @@ -155,29 +160,28 @@ static SProcQueue *taosProcQueueInit(int32_t size) { SProcQueue *pQueue = NULL; int32_t shmId = taosProcInitBuffer((void **)&pQueue, bufSize + headSize); - if (shmId <= 0) { + if (shmId < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pQueue->bufferShmid = shmId; if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { - free(pQueue); + taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); return NULL; } if (tsem_init(&pQueue->sem, 1, 0) != 0) { - taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); - free(pQueue); + taosProcDestroyMutex(pQueue->mutex, pQueue->mutexShmid); + taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { - taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); tsem_destroy(&pQueue->sem); - free(pQueue); + taosProcDestroyMutex(pQueue->mutex, pQueue->mutexShmid); + taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); return NULL; } @@ -190,12 +194,12 @@ static SProcQueue *taosProcQueueInit(int32_t size) { return pQueue; } -static void taosProcQueueCleanup(SProcQueue *pQueue) { +static void taosProcCleanupQueue(SProcQueue *pQueue) { if (pQueue != NULL) { uDebug("proc:%s, queue:%p clean up", pQueue->name, pQueue); - taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); tsem_destroy(&pQueue->sem); - free(pQueue); + taosProcDestroyMutex(pQueue->mutex, pQueue->mutexShmid); + taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); } } @@ -344,12 +348,10 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { } pProc->name = pCfg->name; - pProc->testFlag = pCfg->testFlag; - pProc->pChildQueue = taosProcQueueInit(pCfg->childQueueSize); pProc->pParentQueue = taosProcQueueInit(pCfg->parentQueueSize); if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { - taosProcQueueCleanup(pProc->pChildQueue); + taosProcCleanupQueue(pProc->pChildQueue); free(pProc); return NULL; } @@ -369,17 +371,15 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { pProc->pParentQueue->freeBodyFp = pCfg->parentFreeBodyFp; pProc->pParentQueue->consumeFp = pCfg->parentConsumeFp; - uDebug("proc:%s, initialized, child queue:%p parent queue:%p", pProc->name, pProc->pChildQueue, pProc->pParentQueue); + uDebug("proc:%s, is initialized, child queue:%p parent queue:%p", pProc->name, pProc->pChildQueue, pProc->pParentQueue); - if (!pProc->testFlag) { - pProc->pid = fork(); - if (pProc->pid == 0) { - pProc->isChild = 1; - uInfo("this is child process, pid:%d", pProc->pid); - } else { - pProc->isChild = 0; - uInfo("this is parent process, pid:%d", pProc->pid); - } + pProc->pid = fork(); + if (pProc->pid == 0) { + pProc->isChild = 1; + uInfo("this is child process, pid:%d", pProc->pid); + } else { + pProc->isChild = 0; + uInfo("this is parent process, child pid:%d", pProc->pid); } return pProc; @@ -398,7 +398,7 @@ static void taosProcThreadLoop(SProcQueue *pQueue) { if (code < 0) { uDebug("proc:%s, get no message from queue:%p and exiting", pQueue->name, pQueue); break; - } else if (code < 0) { + } else if (code == 0) { uTrace("proc:%s, get no message from queue:%p since %s", pQueue->name, pQueue, terrstr()); taosMsleep(1); continue; @@ -413,16 +413,14 @@ int32_t taosProcRun(SProcObj *pProc) { taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pProc->isChild || pProc->testFlag) { + if (pProc->isChild) { if (taosThreadCreate(&pProc->childThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pChildQueue) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to create thread since %s", terrstr()); return -1; } uDebug("proc:%s, child start to consume queue:%p", pProc->name, pProc->pChildQueue); - } - - if (!pProc->isChild || pProc->testFlag) { + } else { if (taosThreadCreate(&pProc->parentThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pParentQueue) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to create thread since %s", terrstr()); @@ -445,8 +443,8 @@ void taosProcCleanup(SProcObj *pProc) { if (pProc != NULL) { uDebug("proc:%s, clean up", pProc->name); taosProcStop(pProc); - taosProcQueueCleanup(pProc->pChildQueue); - taosProcQueueCleanup(pProc->pParentQueue); + taosProcCleanupQueue(pProc->pChildQueue); + taosProcCleanupQueue(pProc->pParentQueue); free(pProc); } } From 4ea8c88810ed1f578f7817e483fd2a7b0751513a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 09:30:07 +0000 Subject: [PATCH 091/105] more TDB --- source/libs/tdb/test/tdbTest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index b9ae2dff7d..4c7af58b14 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -15,7 +15,7 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(ret, 0); // Create a database - ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, NULL, pEnv, &pDb); + ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, tKeyCmpr, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -94,8 +94,8 @@ TEST(tdb_test, simple_test) { static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { int k1, k2; - k1 = std::strtol((char *)pKey1 + 3, nullptr, kLen1 - 3); - k2 = std::strtol((char *)pKey2 + 3, nullptr, kLen2 - 3); + k1 = std::strtol((char *)pKey1 + 3, nullptr, 10); + k2 = std::strtol((char *)pKey2 + 3, nullptr, 10); if (k1 < k2) { return -1; From d7e13ef739236875e63b368dfabd7afe826d4228 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Mar 2022 18:22:59 +0800 Subject: [PATCH 092/105] shm --- include/util/tprocess.h | 1 + source/dnode/mgmt/container/src/dndExec.c | 25 ++++++++++++++--------- source/dnode/mgmt/container/src/dndObj.c | 2 +- source/util/src/tprocess.c | 10 +++++++-- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/include/util/tprocess.h b/include/util/tprocess.h index 4ce536fd96..a0be38a3ad 100644 --- a/include/util/tprocess.h +++ b/include/util/tprocess.h @@ -51,6 +51,7 @@ void taosProcCleanup(SProcObj *pProc); int32_t taosProcRun(SProcObj *pProc); void taosProcStop(SProcObj *pProc); bool taosProcIsChild(SProcObj *pProc); +int32_t taosProcChildId(SProcObj *pProc); int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c index 4dcf7e80ec..a6c9852546 100644 --- a/source/dnode/mgmt/container/src/dndExec.c +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -20,7 +20,7 @@ static void dndResetLog(SMgmtWrapper *pMgmt) { char logname[24] = {0}; snprintf(logname, sizeof(logname), "%slog", pMgmt->name); - dInfo("node:%s, reset log to %s", pMgmt->name, logname); + dInfo("node:%s, reset log to %s in child process", pMgmt->name, logname); taosCloseLog(); taosInitLog(logname, 1); } @@ -51,6 +51,7 @@ int32_t dndOpenNode(SMgmtWrapper *pWrapper) { void dndCloseNode(SMgmtWrapper *pWrapper) { dDebug("node:%s, start to close", pWrapper->name); + pWrapper->required = false; taosWLockLatch(&pWrapper->latch); if (pWrapper->deployed) { (*pWrapper->fp.closeFp)(pWrapper); @@ -199,7 +200,7 @@ static int32_t dndRunInMultiProcess(SDnode *pDnode) { dInfo("node:%s, will be initialized in child process", pWrapper->name); dndOpenNode(pWrapper); } else { - dInfo("node:%s, will not start in parent process", pWrapper->name); + dInfo("node:%s, will not start in parent process, child pid:%d", pWrapper->name, taosProcChildId(pProc)); pWrapper->procType = PROC_PARENT; } @@ -209,16 +210,20 @@ static int32_t dndRunInMultiProcess(SDnode *pDnode) { } } -#if 0 - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper->procType == PROC_PARENT && dmStart(pWrapper->pMgmt) != 0) { - dndReleaseWrapper(pWrapper); - dError("failed to start dnode worker since %s", terrstr()); - return -1; + dndSetStatus(pDnode, DND_STAT_RUNNING); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + if (!pWrapper->required) continue; + if (pWrapper->fp.startFp == NULL) continue; + if (pWrapper->procType == PROC_PARENT && n != DNODE) continue; + if (pWrapper->procType == PROC_CHILD && n == DNODE) continue; + if ((*pWrapper->fp.startFp)(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } } - dndReleaseWrapper(pWrapper); -#endif return 0; } diff --git a/source/dnode/mgmt/container/src/dndObj.c b/source/dnode/mgmt/container/src/dndObj.c index d618f4e503..2123f5946b 100644 --- a/source/dnode/mgmt/container/src/dndObj.c +++ b/source/dnode/mgmt/container/src/dndObj.c @@ -175,7 +175,7 @@ int32_t dndMarkWrapper(SMgmtWrapper *pWrapper) { int32_t code = 0; taosRLockLatch(&pWrapper->latch); - if (pWrapper->deployed) { + if (pWrapper->deployed || (pWrapper->procType == PROC_PARENT && pWrapper->required)) { int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount); } else { diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index 24f3f8a6e4..f32f578952 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -208,6 +208,11 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHea const int32_t bodyLen = CEIL8(rawBodyLen); const int32_t fullLen = headLen + bodyLen + 8; + if (headLen <= 0 || bodyLen <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + return -1; + } + taosThreadMutexLock(pQueue->mutex); if (fullLen > pQueue->avail) { taosThreadMutexUnlock(pQueue->mutex); @@ -259,7 +264,7 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHea taosThreadMutexUnlock(pQueue->mutex); tsem_post(&pQueue->sem); - uTrace("proc:%s, push msg:%p:%d cont:%p:%d to queue:%p", pQueue->name, pHead, rawHeadLen, pBody, rawBodyLen, pQueue); + uTrace("proc:%s, push msg:%p:%d cont:%p:%d to queue:%p", pQueue->name, pHead, headLen, pBody, bodyLen, pQueue); return 0; } @@ -376,7 +381,6 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { pProc->pid = fork(); if (pProc->pid == 0) { pProc->isChild = 1; - uInfo("this is child process, pid:%d", pProc->pid); } else { pProc->isChild = 0; uInfo("this is parent process, child pid:%d", pProc->pid); @@ -439,6 +443,8 @@ void taosProcStop(SProcObj *pProc) { bool taosProcIsChild(SProcObj *pProc) { return pProc->isChild; } +int32_t taosProcChildId(SProcObj *pProc) { return pProc->pid; } + void taosProcCleanup(SProcObj *pProc) { if (pProc != NULL) { uDebug("proc:%s, clean up", pProc->name); From 72f1719a93254028be4f9d2248557acf7e4c380b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 10:28:23 +0000 Subject: [PATCH 093/105] add btree debug helper function --- source/libs/tdb/src/db/tdbBtree.c | 37 ++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index de77fb2649..45706ae3dc 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1289,4 +1289,39 @@ static int tdbBtcMoveUpward(SBTC *pBtc) { pBtc->idx = pBtc->idxStack[pBtc->iPage]; return 0; -} \ No newline at end of file +} + +#ifndef NODEBUG +typedef struct { + SPgno pgno; + u8 root; + u8 leaf; + SPgno rChild; + int nCells; + int nOvfl; +} SBtPageInfo; + +SBtPageInfo btPageInfos[20]; + +void tdbBtPageInfo(SPage *pPage, int idx) { + u8 flags; + SBtPageInfo *pBtPageInfo; + + pBtPageInfo = btPageInfos + idx; + + pBtPageInfo->pgno = TDB_PAGE_PGNO(pPage); + + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + + pBtPageInfo->root = TDB_BTREE_PAGE_IS_ROOT(flags); + pBtPageInfo->leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + + pBtPageInfo->rChild = 0; + if (!pBtPageInfo->leaf) { + pBtPageInfo->rChild = *(SPgno *)(pPage->pData + 1); + } + + pBtPageInfo->nCells = TDB_PAGE_TOTAL_CELLS(pPage) - pPage->nOverflow; + pBtPageInfo->nOvfl = pPage->nOverflow; +} +#endif \ No newline at end of file From fc64a2c72d00464356d2d4f8b7612a98610568fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 10:54:54 +0000 Subject: [PATCH 094/105] more --- source/libs/tdb/test/tdbTest.cpp | 34 +++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 4c7af58b14..c256f07cf6 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -3,19 +3,23 @@ #include "tdbInt.h" static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); TEST(tdb_test, simple_test) { - int ret; - STEnv *pEnv; - STDB *pDb; - int nData = 10000000; + int ret; + STEnv *pEnv; + STDB *pDb; + FKeyComparator compFunc; + int nData = 10000000; + // int nData = 8508; // Open Env ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database - ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, tKeyCmpr, pEnv, &pDb); + compFunc = tKeyCmpr; + ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -104,4 +108,24 @@ static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) } else { return 0; } +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; } \ No newline at end of file From 0b12117fb263c35179fb85942183e3cad431e313 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 26 Mar 2022 02:53:38 +0000 Subject: [PATCH 095/105] fix a test bug --- source/libs/tdb/test/tdbTest.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index c256f07cf6..52bf26f0ef 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -2,6 +2,8 @@ #include "tdbInt.h" +#include + static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); @@ -11,7 +13,6 @@ TEST(tdb_test, simple_test) { STDB *pDb; FKeyComparator compFunc; int nData = 10000000; - // int nData = 8508; // Open Env ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv); @@ -98,8 +99,10 @@ TEST(tdb_test, simple_test) { static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { int k1, k2; - k1 = std::strtol((char *)pKey1 + 3, nullptr, 10); - k2 = std::strtol((char *)pKey2 + 3, nullptr, 10); + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); if (k1 < k2) { return -1; From dbec5f1fa4044c4198452b6dcf32b0d0e843ba27 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 26 Mar 2022 09:29:49 +0000 Subject: [PATCH 096/105] fix logical bug with memory leak --- source/libs/tdb/src/db/tdbBtree.c | 12 +-- source/libs/tdb/src/page/tdbPage.c | 6 +- source/libs/tdb/test/tdbTest.cpp | 145 +++++++++++++++++++++-------- 3 files changed, 117 insertions(+), 46 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 45706ae3dc..5980c2b531 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -746,6 +746,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { pgno = TDB_PAGE_PGNO(pNews[iNew]); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); + free(pNewCell); } // move to next new page @@ -795,12 +796,11 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - // TODO: here has memory leak - // for (int i = 0; i < 3; i++) { - // if (pDivCell[i]) { - // free(pDivCell[i]); - // } - // } + for (int i = 0; i < 3; i++) { + if (pDivCell[i]) { + free(pDivCell[i]); + } + } return 0; } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 15b6e218e6..516330e4e6 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -143,7 +143,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; } - pPage->apOvfl[iOvfl] = pCell; + // TODO: here has memory leak + pNewCell = (SCell *)malloc(szCell); + memcpy(pNewCell, pCell, szCell); + + pPage->apOvfl[iOvfl] = pNewCell; pPage->aiOvfl[iOvfl] = idx; pPage->nOverflow++; iOvfl++; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 52bf26f0ef..63341e5430 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -4,8 +4,112 @@ #include -static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); -static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)malloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void closePool(SPoolMem *pPool) { + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + free(pMem); + } while (1); + + assert(pPool->size == 0); + + free(pPool); +} + +static void *poolMalloc(void *arg, int size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)malloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + free(pMem); +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} TEST(tdb_test, simple_test) { int ret; @@ -94,41 +198,4 @@ TEST(tdb_test, simple_test) { // Close Env ret = tdbEnvClose(pEnv); GTEST_ASSERT_EQ(ret, 0); -} - -static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { - int k1, k2; - - std::string s1((char *)pKey1 + 3, kLen1 - 3); - std::string s2((char *)pKey2 + 3, kLen2 - 3); - k1 = stoi(s1); - k2 = stoi(s2); - - if (k1 < k2) { - return -1; - } else if (k1 > k2) { - return 1; - } else { - return 0; - } -} - -static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { - int mlen; - int cret; - - ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); - - mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; - cret = memcmp(pKey1, pKey2, mlen); - if (cret == 0) { - if (keyLen1 < keyLen2) { - cret = -1; - } else if (keyLen1 > keyLen2) { - cret = 1; - } else { - cret = 0; - } - } - return cret; } \ No newline at end of file From 9c1f71459ee3352a9e12ff1b48a4d44a2409523c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 26 Mar 2022 23:04:51 +0800 Subject: [PATCH 097/105] minor changes --- source/util/src/tprocess.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index f32f578952..15708f1cec 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -381,6 +381,7 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { pProc->pid = fork(); if (pProc->pid == 0) { pProc->isChild = 1; + prctl(PR_SET_NAME, pProc->name, NULL, NULL, NULL); } else { pProc->isChild = 0; uInfo("this is parent process, child pid:%d", pProc->pid); From d4c636589acdec860c592f49adee20c09f500ea7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 27 Mar 2022 10:42:23 +0800 Subject: [PATCH 098/105] update definition of SSchema/STColumn to support columns larger than 1MB(int16_t to int32_t) --- 2.0/src/client/inc/tsclient.h | 16 +++---- 2.0/src/client/src/tscParseInsert.c | 20 ++++----- 2.0/src/client/src/tscUtil.c | 2 +- include/common/tdataformat.h | 15 ++++--- include/common/tmsg.h | 20 ++++----- include/common/trow.h | 5 ++- include/common/ttypes.h | 1 + include/libs/nodes/querynodes.h | 2 +- include/libs/qcom/query.h | 10 ++--- source/client/src/clientHb.c | 2 +- source/common/src/tdataformat.c | 10 ++--- source/common/src/tmsg.c | 12 +++--- source/dnode/mnode/impl/src/mndStb.c | 12 +++--- source/dnode/vnode/src/meta/metaBDBImpl.c | 8 ++-- source/dnode/vnode/src/tq/tqRead.c | 6 +-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- source/dnode/vnode/src/vnd/vnodeWrite.c | 11 +++++ source/libs/catalog/src/catalog.c | 2 +- source/libs/parser/inc/parInsertData.h | 26 +++++------ source/libs/parser/src/parInsert.c | 43 ++++++++++--------- source/libs/parser/src/parInsertData.c | 8 ++-- source/libs/parser/src/parTranslater.c | 2 +- .../libs/parser/test/mockCatalogService.cpp | 2 +- source/libs/qcom/src/queryUtil.c | 2 +- source/libs/qcom/src/querymsg.c | 2 +- source/libs/qworker/src/qworkerMsg.c | 2 +- 27 files changed, 133 insertions(+), 112 deletions(-) diff --git a/2.0/src/client/inc/tsclient.h b/2.0/src/client/inc/tsclient.h index e14a3123ad..d5021f663d 100644 --- a/2.0/src/client/inc/tsclient.h +++ b/2.0/src/client/inc/tsclient.h @@ -116,7 +116,7 @@ typedef struct SParsedDataColInfo { uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) uint16_t extendedVarLen; uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - int32_t * boundedColumns; // bound column idx according to schema + int32_t *boundColumns; // bound column idx according to schema SBoundColumn * cols; SBoundIdxInfo *colIdxInfo; int8_t orderStatus; // bound columns @@ -125,7 +125,7 @@ typedef struct SParsedDataColInfo { #define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) typedef struct { - uint8_t memRowType; // default is 0, that is SDataRow + uint8_t rowType; // default is 0, that is SDataRow int32_t rowSize; } SMemRowBuilder; @@ -137,17 +137,17 @@ void destroyMemRowBuilder(SMemRowBuilder *pBuilder); /** * @brief * - * @param memRowType + * @param rowType * @param spd * @param idx the absolute bound index of columns * @return FORCE_INLINE */ -static FORCE_INLINE void tscGetMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd, - int32_t idx, int32_t *toffset, int16_t *colId) { +static FORCE_INLINE void tscGetSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, int32_t idx, + int32_t *toffset, int16_t *colId) { int32_t schemaIdx = 0; if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundedColumns[idx]; - if (isDataRowT(memRowType)) { + schemaIdx = spd->boundColumns[idx]; + if (isDataRowT(rowType)) { *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart } else { *toffset = idx * sizeof(SColIdx); // the offset of SColIdx @@ -155,7 +155,7 @@ static FORCE_INLINE void tscGetMemRowAppendInfo(SSchema *pSchema, uint8_t memRow } else { ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; - if (isDataRowT(memRowType)) { + if (isDataRowT(rowType)) { *toffset = (spd->cols + schemaIdx)->toffset; } else { *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx); diff --git a/2.0/src/client/src/tscParseInsert.c b/2.0/src/client/src/tscParseInsert.c index b3f7c076e3..a1f5899cee 100644 --- a/2.0/src/client/src/tscParseInsert.c +++ b/2.0/src/client/src/tscParseInsert.c @@ -428,7 +428,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { // the start position in data block buffer of current value in sql - int32_t colIndex = spd->boundedColumns[i]; + int32_t colIndex = spd->boundColumns[i]; char *start = row + spd->cols[colIndex].offset; @@ -495,7 +495,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); int32_t toffset = -1; int16_t colId = -1; - tscGetMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); + tscGetSTSRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); int32_t ret = tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset, colId); @@ -630,7 +630,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32 pColInfo->numOfCols = numOfCols; pColInfo->numOfBound = numOfCols; pColInfo->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColInfo->boundedColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); + pColInfo->boundColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn)); pColInfo->colIdxInfo = NULL; pColInfo->flen = 0; @@ -656,7 +656,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32 default: break; } - pColInfo->boundedColumns[i] = i; + pColInfo->boundColumns[i] = i; } pColInfo->allNullLen += pColInfo->flen; pColInfo->boundNullLen = pColInfo->allNullLen; // default set allNullLen @@ -991,7 +991,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC } for (int i = 0; i < spd.numOfBound; ++i) { - SSchema* pSchema = &pTagSchema[spd.boundedColumns[i]]; + SSchema *pSchema = &pTagSchema[spd.boundColumns[i]]; index = 0; sToken = tStrGetToken(sql, &index, true); @@ -1158,7 +1158,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat pColInfo->numOfBound = 0; pColInfo->boundNullLen = 0; - memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * nCols); + memset(pColInfo->boundColumns, 0, sizeof(int32_t) * nCols); for (int32_t i = 0; i < nCols; ++i) { pColInfo->cols[i].valStat = VAL_STAT_NONE; } @@ -1205,7 +1205,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } pColInfo->cols[t].valStat = VAL_STAT_HAS; - pColInfo->boundedColumns[pColInfo->numOfBound] = t; + pColInfo->boundColumns[pColInfo->numOfBound] = t; ++pColInfo->numOfBound; switch (pSchema[t].type) { case TSDB_DATA_TYPE_BINARY: @@ -1239,7 +1239,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } pColInfo->cols[t].valStat = VAL_STAT_HAS; - pColInfo->boundedColumns[pColInfo->numOfBound] = t; + pColInfo->boundColumns[pColInfo->numOfBound] = t; ++pColInfo->numOfBound; switch (pSchema[t].type) { case TSDB_DATA_TYPE_BINARY: @@ -1279,7 +1279,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } SBoundIdxInfo *pColIdx = pColInfo->colIdxInfo; for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) { - pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundedColumns[i]; + pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundColumns[i]; pColIdx[i].boundIdx = i; } qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); @@ -1289,7 +1289,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } - memset(&pColInfo->boundedColumns[pColInfo->numOfBound], 0, + memset(&pColInfo->boundColumns[pColInfo->numOfBound], 0, sizeof(int32_t) * (pColInfo->numOfCols - pColInfo->numOfBound)); return TSDB_CODE_SUCCESS; diff --git a/2.0/src/client/src/tscUtil.c b/2.0/src/client/src/tscUtil.c index 3a8c740ea1..449f1ea453 100644 --- a/2.0/src/client/src/tscUtil.c +++ b/2.0/src/client/src/tscUtil.c @@ -1554,7 +1554,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { } void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) { - tfree(pColInfo->boundedColumns); + tfree(pColInfo->boundColumns); tfree(pColInfo->cols); tfree(pColInfo->colIdxInfo); } diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 698352f636..a2899ead8e 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -59,12 +59,15 @@ extern "C" { } while (0); // ----------------- TSDB COLUMN DEFINITION +#pragma pack(push, 1) typedef struct { - int8_t type; // Column type - col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) - int16_t bytes; // column bytes (restore to int16_t in case of misuse) - uint16_t offset; // point offset in STpRow after the header part. + col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) + int32_t type : 8; // column type + int32_t bytes : 24; // column bytes (restore to int32_t in case of misuse) + int32_t sma : 8; // block SMA: 0, no SMA, 1, sum/min/max, 2, ... + int32_t offset : 24; // point offset in STpRow after the header part. } STColumn; +#pragma pack(pop) #define colType(col) ((col)->type) #define colColId(col) ((col)->colId) @@ -136,7 +139,7 @@ typedef struct { int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition @@ -590,7 +593,7 @@ void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); -static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) { +static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 136a576c7c..9c10137654 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -259,10 +259,10 @@ typedef struct { } SSubmitRsp; typedef struct SSchema { - int8_t type; - int32_t colId; - int32_t bytes; - char name[TSDB_COL_NAME_LEN]; + int8_t type; + col_id_t colId; + int32_t bytes; + char name[TSDB_COL_NAME_LEN]; } SSchema; typedef struct { @@ -438,8 +438,8 @@ typedef struct { */ typedef struct { union { - int16_t colId; - int16_t slotId; + col_id_t colId; + int16_t slotId; }; int16_t type; @@ -1902,7 +1902,7 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pSchema->type); tlen += taosEncodeFixedI32(buf, pSchema->bytes); - tlen += taosEncodeFixedI32(buf, pSchema->colId); + tlen += taosEncodeFixedI16(buf, pSchema->colId); tlen += taosEncodeString(buf, pSchema->name); return tlen; } @@ -1910,7 +1910,7 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) { buf = taosDecodeFixedI8(buf, &pSchema->type); buf = taosDecodeFixedI32(buf, &pSchema->bytes); - buf = taosDecodeFixedI32(buf, &pSchema->colId); + buf = taosDecodeFixedI16(buf, &pSchema->colId); buf = taosDecodeStringTo(buf, pSchema->name); return buf; } @@ -1918,7 +1918,7 @@ static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) { static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) { if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1; if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1; - if (tEncodeI32(pEncoder, pSchema->colId) < 0) return -1; + if (tEncodeI16(pEncoder, pSchema->colId) < 0) return -1; if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1; return 0; } @@ -1926,7 +1926,7 @@ static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSch static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) { if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1; if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1; - if (tDecodeI32(pDecoder, &pSchema->colId) < 0) return -1; + if (tDecodeI16(pDecoder, &pSchema->colId) < 0) return -1; if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1; return 0; } diff --git a/include/common/trow.h b/include/common/trow.h index fc99cbc5b2..df28bc9962 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -671,8 +671,9 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa * @param colIdx sorted column index, start from 0 * @return FORCE_INLINE */ -static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, int16_t colId, int8_t colType, TDRowValT valType, - const void *val, bool isCopyVarData, int32_t offset, int16_t colIdx) { +static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, + TDRowValT valType, const void *val, bool isCopyVarData, int32_t offset, + col_id_t colIdx) { STSRow *pRow = pBuilder->pBuf; if (!val) { #ifdef TD_SUPPORT_BITMAP diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 59af14c226..87dc752703 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -29,6 +29,7 @@ typedef uint32_t TDRowLenT; typedef uint8_t TDRowValT; typedef int16_t col_id_t; typedef int8_t col_type_t; +typedef int32_t col_bytes_t; #pragma pack(push, 1) typedef struct { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 616e24d67d..a78a080e73 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -56,7 +56,7 @@ typedef enum EColumnType { typedef struct SColumnNode { SExprNode node; // QUERY_NODE_COLUMN uint64_t tableId; - int16_t colId; + col_id_t colId; EColumnType colType; // column or tag char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 839feb7800..1676f0a267 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -43,10 +43,10 @@ typedef enum { } ETaskType; typedef struct STableComInfo { - uint8_t numOfTags; // the number of tags in schema - uint8_t precision; // the number of precision - int16_t numOfColumns; // the number of columns - int32_t rowSize; // row size of the schema + uint8_t numOfTags; // the number of tags in schema + uint8_t precision; // the number of precision + col_id_t numOfColumns; // the number of columns + int32_t rowSize; // row size of the schema } STableComInfo; typedef struct SIndexMeta { @@ -171,7 +171,7 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); char *jobTaskStatusStr(int32_t status); -SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); +SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 832a415efa..abb6e7fbd1 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -91,7 +91,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } else { tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - tscError("invalid colId[%d] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); + tscError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); tFreeSTableMetaBatchRsp(&batchMetaRsp); return TSDB_CODE_TSC_INVALID_VALUE; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 5b27fd01f4..1b7157c49c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -106,12 +106,12 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { if (tdInitTSchemaBuilder(&schemaBuilder, version) < 0) return NULL; for (int i = 0; i < numOfCols; i++) { - int8_t type = 0; - int16_t colId = 0; - int16_t bytes = 0; + col_type_t type = 0; + col_id_t colId = 0; + col_bytes_t bytes = 0; buf = taosDecodeFixedI8(buf, &type); buf = taosDecodeFixedI16(buf, &colId); - buf = taosDecodeFixedI16(buf, &bytes); + buf = taosDecodeFixedI32(buf, &bytes); if (tdAddColToSchema(&schemaBuilder, type, colId, bytes) < 0) { tdDestroyTSchemaBuilder(&schemaBuilder); return NULL; @@ -148,7 +148,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { pBuilder->version = version; } -int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes) { +int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes) { if (!isValidDataType(type)) return -1; if (pBuilder->nCols >= pBuilder->tCols) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d7df3e115f..d73897eb20 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -299,14 +299,14 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols); for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name); } tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols); for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].colId); + tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); } @@ -333,7 +333,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols); for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); } @@ -374,7 +374,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); - buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].colId)); + buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId)); buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes)); buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); } @@ -382,7 +382,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); - buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].colId); + buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); } @@ -422,7 +422,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); - buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].colId); + buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index acf45a31d9..c23d9fc5c9 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -98,7 +98,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } @@ -106,7 +106,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } @@ -114,7 +114,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfSmas; ++i) { SSchema *pSchema = &pStb->pSmas[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } @@ -185,7 +185,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } @@ -193,7 +193,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } @@ -201,7 +201,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfSmas; ++i) { SSchema *pSchema = &pStb->pSmas[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index fdcc165207..b91c6cd9e3 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -233,7 +233,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { // save sma info int32_t len = tEncodeTSma(NULL, pSmaCfg); - pBuf = taosMemoryCalloc(len, 1); + pBuf = taosMemoryCalloc(1, len); if (pBuf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -285,7 +285,7 @@ static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; tlen += taosEncodeFixedI8(buf, pSchema->type); - tlen += taosEncodeFixedI32(buf, pSchema->colId); + tlen += taosEncodeFixedI16(buf, pSchema->colId); tlen += taosEncodeFixedI32(buf, pSchema->bytes); tlen += taosEncodeString(buf, pSchema->name); } @@ -301,7 +301,7 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; buf = taosDecodeFixedI8(buf, &pSchema->type); - buf = taosDecodeFixedI32(buf, &pSchema->colId); + buf = taosDecodeFixedI16(buf, &pSchema->colId); buf = taosDecodeFixedI32(buf, &pSchema->bytes); buf = taosDecodeStringTo(buf, pSchema->name); } @@ -516,6 +516,7 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { tsize += taosEncodeFixedU64(buf, pTbCfg->ctbCfg.suid); tsize += tdEncodeKVRow(buf, pTbCfg->ctbCfg.pTag); } else if (pTbCfg->type == META_NORMAL_TABLE) { + // TODO } else { ASSERT(0); } @@ -538,6 +539,7 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid)); buf = tdDecodeKVRow(buf, &(pTbCfg->ctbCfg.pTag)); } else if (pTbCfg->type == META_NORMAL_TABLE) { + // TODO } else { ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 37e7ed11ae..8c161e4c8b 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -130,8 +130,8 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int32_t colNeed = 0; while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - int16_t colIdSchema = pColSchema->colId; - int16_t colIdNeed = *(int16_t*)taosArrayGet(pHandle->pColIdList, colNeed); + col_id_t colIdSchema = pColSchema->colId; + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pHandle->pColIdList, colNeed); if (colIdSchema < colIdNeed) { colMeta++; } else if (colIdSchema > colIdNeed) { @@ -159,7 +159,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int j = 0; for (int32_t i = 0; i < colNumNeed; i++) { - int16_t colId = *(int16_t*)taosArrayGet(pHandle->pColIdList, i); + col_id_t colId = *(col_id_t*)taosArrayGet(pHandle->pColIdList, i); while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { j++; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index eca0a9612f..eb8df61051 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1369,7 +1369,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF } } - // Update pBlock membership vairables + // Update pBlock membership variables pBlock->last = isLast; pBlock->offset = offset; pBlock->algorithm = pCfg->compression; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 6eb721fcb1..34dba5d3ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -255,7 +255,7 @@ int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg) { return 0; } -int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { +static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; SSubmitMsgIter msgIter = {0}; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index e0be9ed89a..a0a34b14a4 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -74,6 +74,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: maybe need to clear the request struct taosMemoryFree(vCreateTbReq.stbCfg.pSchema); taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema); + taosMemoryFree(vCreateTbReq.stbCfg.pBSmaCols); + taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam); + taosMemoryFree(vCreateTbReq.dbFName); taosMemoryFree(vCreateTbReq.name); break; } @@ -102,13 +105,18 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name); } taosMemoryFree(pCreateTbReq->name); + taosMemoryFree(pCreateTbReq->dbFName); if (pCreateTbReq->type == TD_SUPER_TABLE) { taosMemoryFree(pCreateTbReq->stbCfg.pSchema); taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema); + taosMemoryFree(pCreateTbReq->stbCfg.pBSmaCols); + taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam); } else if (pCreateTbReq->type == TD_CHILD_TABLE) { taosMemoryFree(pCreateTbReq->ctbCfg.pTag); } else { taosMemoryFree(pCreateTbReq->ntbCfg.pSchema); + taosMemoryFree(pCreateTbReq->ntbCfg.pBSmaCols); + taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam); } } @@ -135,6 +143,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq); taosMemoryFree(vAlterTbReq.stbCfg.pSchema); taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema); + taosMemoryFree(vAlterTbReq.stbCfg.pBSmaCols); + taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam); + taosMemoryFree(vAlterTbReq.dbFName); taosMemoryFree(vAlterTbReq.name); break; } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 7bbff05b4b..4f894e0aa9 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -172,7 +172,7 @@ void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { int32_t colNum = c->numOfColumns + c->numOfTags; for (int32_t i = 0; i < colNum; ++i) { SSchema *s = &p->schema[i]; - ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes); + ctgDebug("[%d] name:%s, type:%d, colId:%" PRIi16 ", bytes:%d", i, s->name, s->type, s->colId, s->bytes); } } diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index acd021572d..a38d64c58c 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -41,26 +41,26 @@ typedef struct SBoundColumn { } SBoundColumn; typedef struct { - uint16_t schemaColIdx; - uint16_t boundIdx; - uint16_t finalIdx; + col_id_t schemaColIdx; + col_id_t boundIdx; + col_id_t finalIdx; } SBoundIdxInfo; typedef struct SParsedDataColInfo { - int16_t numOfCols; - int16_t numOfBound; + col_id_t numOfCols; + col_id_t numOfBound; uint16_t flen; // TODO: get from STSchema uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) uint16_t extendedVarLen; uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - int32_t * boundedColumns; // bound column idx according to schema - SBoundColumn * cols; + col_id_t *boundColumns; // bound column idx according to schema + SBoundColumn *cols; SBoundIdxInfo *colIdxInfo; int8_t orderStatus; // bound columns } SParsedDataColInfo; typedef struct { - uint8_t memRowType; // default is 0, that is SDataRow + uint8_t rowType; // default is 0, that is SDataRow int32_t rowSize; } SMemRowBuilder; @@ -92,11 +92,11 @@ static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) { (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1); } -static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, - int32_t idx, int32_t *toffset, int32_t *colIdx) { - int32_t schemaIdx = 0; +static FORCE_INLINE void getSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, + int32_t *toffset, col_id_t *colIdx) { + col_id_t schemaIdx = 0; if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundedColumns[idx] - PRIMARYKEY_TIMESTAMP_COL_ID; + schemaIdx = spd->boundColumns[idx] - PRIMARYKEY_TIMESTAMP_COL_ID; if (TD_IS_TP_ROW_T(rowType)) { *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart *colIdx = schemaIdx; @@ -132,7 +132,7 @@ static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks* int32_t schemaIdxCompar(const void *lhs, const void *rhs); int32_t boundIdxCompar(const void *lhs, const void *rhs); -void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols); +void setBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); void destroyBoundColumnInfo(SParsedDataColInfo* pColList); void destroyBlockArrayList(SArray* pDataBlockList); void destroyBlockHashmap(SHashObj* pDataBlockHash); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index b2fc39d064..80a83af922 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -600,9 +600,9 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int typedef struct SMemParam { SRowBuilder* rb; - SSchema* schema; - int32_t toffset; - int32_t colIdx; + SSchema* schema; + int32_t toffset; + col_id_t colIdx; } SMemParam; static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* param) { @@ -623,9 +623,11 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); } else { if (value == NULL) { // it is a null data - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx); + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, + pa->colIdx); } else { - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, pa->colIdx); + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, + pa->colIdx); } } return TSDB_CODE_SUCCESS; @@ -633,18 +635,18 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p // pSql -> tag1_name, ...) static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { - int32_t nCols = pColList->numOfCols; + int16_t nCols = pColList->numOfCols; pColList->numOfBound = 0; pColList->boundNullLen = 0; - memset(pColList->boundedColumns, 0, sizeof(int32_t) * nCols); + memset(pColList->boundColumns, 0, sizeof(int16_t) * nCols); for (int32_t i = 0; i < nCols; ++i) { pColList->cols[i].valStat = VAL_STAT_NONE; } SToken sToken; bool isOrdered = true; - int32_t lastColIdx = -1; // last column found + int16_t lastColIdx = -1; // last column found while (1) { NEXT_TOKEN(pCxt->pSql, sToken); @@ -652,8 +654,8 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* break; } - int32_t t = lastColIdx + 1; - int32_t index = findCol(&sToken, t, nCols, pSchema); + int16_t t = lastColIdx + 1; + int16_t index = findCol(&sToken, t, nCols, pSchema); if (index < 0 && t > 0) { index = findCol(&sToken, 0, t, pSchema); isOrdered = false; @@ -666,7 +668,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* } lastColIdx = index; pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundedColumns[pColList->numOfBound] = index + PRIMARYKEY_TIMESTAMP_COL_ID; + pColList->boundColumns[pColList->numOfBound] = index + PRIMARYKEY_TIMESTAMP_COL_ID; ++pColList->numOfBound; switch (pSchema[t].type) { case TSDB_DATA_TYPE_BINARY: @@ -689,18 +691,19 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* return TSDB_CODE_TSC_OUT_OF_MEMORY; } SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (uint16_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = (uint16_t)pColList->boundedColumns[i]; + for (int16_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].schemaColIdx = pColList->boundColumns[i]; pColIdx[i].boundIdx = i; } qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); - for (uint16_t i = 0; i < pColList->numOfBound; ++i) { + for (int16_t i = 0; i < pColList->numOfBound; ++i) { pColIdx[i].finalIdx = i; } qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } - memset(&pColList->boundedColumns[pColList->numOfBound], 0, sizeof(int32_t) * (pColList->numOfCols - pColList->numOfBound)); + memset(&pColList->boundColumns[pColList->numOfBound], 0, + sizeof(int16_t) * (pColList->numOfCols - pColList->numOfBound)); return TSDB_CODE_SUCCESS; } @@ -714,8 +717,8 @@ typedef struct SKvParam { static int32_t KvRowAppend(const void *value, int32_t len, void *param) { SKvParam* pa = (SKvParam*) param; - int32_t type = pa->schema->type; - int32_t colId = pa->schema->colId; + int8_t type = pa->schema->type; + int16_t colId = pa->schema->colId; if (TSDB_DATA_TYPE_BINARY == type) { STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len); @@ -747,7 +750,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema* pSchema = &pTagsSchema[pCxt->tags.boundedColumns[i]]; + SSchema* pSchema = &pTagsSchema[pCxt->tags.boundColumns[i]]; param.schema = pSchema; CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); } @@ -813,9 +816,9 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema *pSchema = &schema[spd->boundedColumns[i] - 1]; + SSchema* pSchema = &schema[spd->boundColumns[i] - 1]; param.schema = pSchema; - getMemRowAppendInfo(schema, pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); + getSTSRowAppendInfo(schema, pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, ¶m, &pCxt->msg)); if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index e516053b1e..f70e514b5a 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -43,11 +43,11 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { } } -void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols) { +void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_id_t numOfCols) { pColList->numOfCols = numOfCols; pColList->numOfBound = numOfCols; pColList->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColList->boundedColumns = taosMemoryCalloc(pColList->numOfCols, sizeof(int32_t)); + pColList->boundColumns = taosMemoryCalloc(pColList->numOfCols, sizeof(col_id_t)); pColList->cols = taosMemoryCalloc(pColList->numOfCols, sizeof(SBoundColumn)); pColList->colIdxInfo = NULL; pColList->flen = 0; @@ -73,7 +73,7 @@ void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t default: break; } - pColList->boundedColumns[i] = pSchema[i].colId; + pColList->boundColumns[i] = pSchema[i].colId; } pColList->allNullLen += pColList->flen; pColList->boundNullLen = pColList->allNullLen; // default set allNullLen @@ -103,7 +103,7 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs) { } void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { - taosMemoryFreeClear(pColList->boundedColumns); + taosMemoryFreeClear(pColList->boundColumns); taosMemoryFreeClear(pColList->cols); taosMemoryFreeClear(pColList->colIdxInfo); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2d1882245d..896fab4fdd 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1959,7 +1959,7 @@ typedef struct SVgroupTablesBatch { char dbName[TSDB_DB_NAME_LEN]; } SVgroupTablesBatch; -static void toSchema(const SColumnDefNode* pCol, int32_t colId, SSchema* pSchema) { +static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) { pSchema->colId = colId; pSchema->type = pCol->dataType.type; pSchema->bytes = pCol->dataType.bytes; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 012af26c17..402caeb252 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -87,7 +87,7 @@ private: return meta_; } - int32_t colId_; + col_id_t colId_; int32_t rowsize_; std::shared_ptr meta_; }; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 79a59a5ecb..b08379a186 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -193,7 +193,7 @@ char *jobTaskStatusStr(int32_t status) { return "UNKNOWN"; } -SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name) { +SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name) { SSchema s = {0}; s.type = type; s.bytes = bytes; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 20eb49ed33..1e91c55dc0 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -188,7 +188,7 @@ static int32_t queryConvertTableMetaMsg(STableMetaRsp *pMetaMsg) { } if (pMetaMsg->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - qError("invalid colId[%d] for the first column in table meta rsp msg", pMetaMsg->pSchemas[0].colId); + qError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", pMetaMsg->pSchemas[0].colId); return TSDB_CODE_TSC_INVALID_VALUE; } diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 56267c462b..ebdc3f287d 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -173,7 +173,7 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { return -1; } - int32_t cols = 0; + col_id_t cols = 0; SSchema *pSchema = showRsp.tableMeta.pSchemas; const SSchema *s = tGetTbnameColumnSchema(); From 583f01d4980a4b315697468a504cd07a76f793b7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 27 Mar 2022 12:24:26 +0800 Subject: [PATCH 099/105] refactor --- source/libs/parser/src/parInsert.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 80a83af922..398e5abaa6 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -635,18 +635,18 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p // pSql -> tag1_name, ...) static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { - int16_t nCols = pColList->numOfCols; + col_id_t nCols = pColList->numOfCols; pColList->numOfBound = 0; pColList->boundNullLen = 0; - memset(pColList->boundColumns, 0, sizeof(int16_t) * nCols); + memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); for (int32_t i = 0; i < nCols; ++i) { pColList->cols[i].valStat = VAL_STAT_NONE; } SToken sToken; bool isOrdered = true; - int16_t lastColIdx = -1; // last column found + col_id_t lastColIdx = -1; // last column found while (1) { NEXT_TOKEN(pCxt->pSql, sToken); @@ -654,8 +654,8 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* break; } - int16_t t = lastColIdx + 1; - int16_t index = findCol(&sToken, t, nCols, pSchema); + col_id_t t = lastColIdx + 1; + col_id_t index = findCol(&sToken, t, nCols, pSchema); if (index < 0 && t > 0) { index = findCol(&sToken, 0, t, pSchema); isOrdered = false; From 4693f795261ee4bd4b63b4a68e7291d37c015525 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 27 Mar 2022 23:26:21 +0800 Subject: [PATCH 100/105] refactor --- include/common/tmsg.h | 2 +- include/util/tdef.h | 2 +- source/libs/parser/src/parInsert.c | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d447f11523..bdb6181884 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -208,11 +208,11 @@ typedef struct { typedef struct SSubmitBlk { int64_t uid; // table unique id int64_t suid; // stable id - int32_t padding; // TODO just for padding here int32_t sversion; // data schema version int32_t dataLen; // data part length, not including the SSubmitBlk head int32_t schemaLen; // schema length, if length is 0, no schema exists int16_t numOfRows; // total number of rows in current submit block + int16_t padding; // TODO just for padding here char data[]; } SSubmitBlk; diff --git a/include/util/tdef.h b/include/util/tdef.h index 655deb4625..6c5208ec00 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -209,7 +209,7 @@ typedef enum ELogicConditionType { #define TSDB_FUNC_TYPE_AGGREGATE 2 #define TSDB_FUNC_MAX_RETRIEVE 1024 -#define TSDB_INDEX_NAME_LEN 33 // 32 + 1 '\0' +#define TSDB_INDEX_NAME_LEN 65 // 64 + 1 '\0' #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 398e5abaa6..ed67de17e0 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -640,7 +640,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList->numOfBound = 0; pColList->boundNullLen = 0; memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); - for (int32_t i = 0; i < nCols; ++i) { + for (col_id_t i = 0; i < nCols; ++i) { pColList->cols[i].valStat = VAL_STAT_NONE; } @@ -691,19 +691,19 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* return TSDB_CODE_TSC_OUT_OF_MEMORY; } SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (int16_t i = 0; i < pColList->numOfBound; ++i) { + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { pColIdx[i].schemaColIdx = pColList->boundColumns[i]; pColIdx[i].boundIdx = i; } qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); - for (int16_t i = 0; i < pColList->numOfBound; ++i) { + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { pColIdx[i].finalIdx = i; } qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(int16_t) * (pColList->numOfCols - pColList->numOfBound)); + sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); return TSDB_CODE_SUCCESS; } From ded8843367436ab2cb68e2a3658de77893e191b3 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 28 Mar 2022 07:35:57 +0800 Subject: [PATCH 101/105] SSDatablock refactor --- source/dnode/vnode/src/tsdb/tsdbSma.c | 102 +++++++++++++++++------- source/dnode/vnode/test/tsdbSmaTest.cpp | 2 +- 2 files changed, 75 insertions(+), 29 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 9d5e132772..07eafd6df0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -38,9 +38,10 @@ typedef enum { } ESmaStorageLevel; typedef struct { - STsdb *pTsdb; - SDBFile dFile; - int32_t interval; // interval with the precision of DB + STsdb *pTsdb; + SDBFile dFile; + SSDataBlock *pData; // sma data + int32_t interval; // interval with the precision of DB } STSmaWriteH; typedef struct { @@ -98,7 +99,8 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ int32_t nMaxResult); // insert data -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, SSDataBlock *pData, int64_t interval, + int8_t intervalUnit); static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); @@ -800,9 +802,10 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p return TSDB_CODE_SUCCESS; } -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData) { +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, SSDataBlock *pData, int64_t interval, int8_t intervalUnit) { pSmaH->pTsdb = pTsdb; - pSmaH->interval = tsdbGetIntervalByPrecision(pData->interval, pData->intervalUnit, REPO_CFG(pTsdb)->precision); + pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision); + pSmaH->pData = pData; return TSDB_CODE_SUCCESS; } @@ -857,10 +860,10 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe * @return int32_t */ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); - int64_t indexUid = SMA_TEST_INDEX_UID; + STsdbCfg *pCfg = REPO_CFG(pTsdb); + SSDataBlock *pData = (SSDataBlock *)msg; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + int64_t indexUid = SMA_TEST_INDEX_UID; if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; @@ -868,15 +871,15 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return terrno; } - if (pData->dataLen <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; + if (pData == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert tSma data failed since pData is NULL", REPO_ID(pTsdb)); + return terrno; } - STSmaWriteH tSmaH = {0}; - - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) { + if (taosArrayGetSize(pData->pDataBlock) <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + tsdbWarn("vgId:%d insert tSma data failed since pDataBlock is empty", REPO_ID(pTsdb)); return TSDB_CODE_FAILED; } @@ -895,6 +898,14 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return TSDB_CODE_FAILED; } + STSma *pSma = pItem->pSma; + + STSmaWriteH tSmaH = {0}; + + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData, pSma->interval, pSma->intervalUnit) != 0) { + return TSDB_CODE_FAILED; + } + char rPath[TSDB_FILENAME_LEN] = {0}; char aPath[TSDB_FILENAME_LEN] = {0}; snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); @@ -907,8 +918,11 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { } // Step 1: Judge the storage level and days - int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); + int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); + + +#if 0 int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file @@ -933,7 +947,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { // Step 3: reset the SSmaStat tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); - +#endif tsdbDestroyTSmaWriteH(&tSmaH); tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_SUCCESS; @@ -999,29 +1013,58 @@ static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, } static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); + STsdbCfg *pCfg = REPO_CFG(pTsdb); + SSDataBlock *pData = (SSDataBlock *)msg; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); + int64_t indexUid = SMA_TEST_INDEX_UID; if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); + tsdbWarn("vgId:%d insert rSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); return terrno; } - if (pData->dataLen <= 0) { - TASSERT(0); + if (pEnv == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert rSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); + return terrno; + } + + if (pData == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert rSma data failed since pData is NULL", REPO_ID(pTsdb)); + return terrno; + } + + if (taosArrayGetSize(pData->pDataBlock) <= 0) { terrno = TSDB_CODE_INVALID_PARA; + tsdbWarn("vgId:%d insert rSma data failed since pDataBlock is empty", REPO_ID(pTsdb)); return TSDB_CODE_FAILED; } + SSmaStat *pStat = SMA_ENV_STAT(pTsdb->pTSmaEnv); + SSmaStatItem *pItem = NULL; + + tsdbRefSmaStat(pTsdb, pStat); + + if (pStat && pStat->smaStatItems) { + pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); + } + + if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL) || tsdbSmaStatIsDropped(pItem)) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } + + STSma *pSma = pItem->pSma; + STSmaWriteH tSmaH = {0}; - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) { + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData, pSma->interval, pSma->intervalUnit) != 0) { return TSDB_CODE_FAILED; } - int64_t indexUid = SMA_TEST_INDEX_UID; char rPath[TSDB_FILENAME_LEN] = {0}; char aPath[TSDB_FILENAME_LEN] = {0}; snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); @@ -1033,8 +1076,9 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { } // Step 1: Judge the storage level and days - int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); + int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); + #if 0 int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file @@ -1057,8 +1101,10 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { // Step 3: reset the SSmaStat tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); +#endif tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 86db3af4dc..29a4b7f552 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -280,7 +280,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { } #endif -#if 1 +#if 0 TEST(testCase, tSma_Data_Insert_Query_Test) { // step 1: prepare meta const char *smaIndexName1 = "sma_index_test_1"; From 53e5bfb820af07bfa9b4dc90716b443a269927db Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Mar 2022 10:09:00 +0800 Subject: [PATCH 102/105] minor changes --- source/dnode/mnode/impl/src/mndUser.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index ff34c26c4a..7d3f755cd7 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -21,7 +21,7 @@ #include "mndTrans.h" #include "tbase64.h" -#define TSDB_USER_VER_NUMBER 1 +#define TSDB_USER_VER_NUMBER 1 #define TSDB_USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); @@ -270,7 +270,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate userObj.updateTime = userObj.createdTime; userObj.superUser = pCreate->superUser; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_CREATE_USER, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to create since %s", pCreate->user, terrstr()); return -1; @@ -350,7 +350,7 @@ CREATE_USER_OVER: } static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNodeMsg *pReq) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER,&pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to update since %s", pOld->user, terrstr()); return -1; @@ -511,7 +511,7 @@ ALTER_USER_OVER: } static int32_t mndDropUser(SMnode *pMnode, SNodeMsg *pReq, SUserObj *pUser) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_DROP_USER, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_USER, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); return -1; From 4b5c05334fb2b336616be1ce7441c2fc5602d4bf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Mar 2022 10:10:06 +0800 Subject: [PATCH 103/105] minor changes --- source/util/src/tprocess.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index f170be0844..f5ce88179b 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -152,7 +152,7 @@ static void taosProcDestroyBuffer(void *pBuffer, int32_t shmid) { } } -static SProcQueue *taosProcQueueInit(int32_t size) { +static SProcQueue *taosProcInitQueue(int32_t size) { if (size <= 0) size = SHM_DEFAULT_SIZE; int32_t bufSize = CEIL8(size); @@ -353,10 +353,10 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { } pProc->name = pCfg->name; - pProc->pChildQueue = taosProcQueueInit(pCfg->childQueueSize); - pProc->pParentQueue = taosProcQueueInit(pCfg->parentQueueSize); + pProc->pChildQueue = taosProcInitQueue(pCfg->childQueueSize); + pProc->pParentQueue = taosProcInitQueue(pCfg->parentQueueSize); if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { - taosProcQueueCleanup(pProc->pChildQueue); + taosProcCleanupQueue(pProc->pChildQueue); taosMemoryFree(pProc); return NULL; } @@ -450,8 +450,8 @@ void taosProcCleanup(SProcObj *pProc) { if (pProc != NULL) { uDebug("proc:%s, clean up", pProc->name); taosProcStop(pProc); - taosProcQueueCleanup(pProc->pChildQueue); - taosProcQueueCleanup(pProc->pParentQueue); + taosProcCleanupQueue(pProc->pChildQueue); + taosProcCleanupQueue(pProc->pParentQueue); taosMemoryFree(pProc); } } From d882edd037f151357270b6486df0b8946cb23da9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Mar 2022 10:13:24 +0800 Subject: [PATCH 104/105] minor changes --- source/dnode/mgmt/container/src/dndExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c index 24c7ad7968..a7b8ca288b 100644 --- a/source/dnode/mgmt/container/src/dndExec.c +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -139,7 +139,7 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t msgLen, void *pCont, int32_t contLen) { dTrace("msg:%p, get from parent queue", pRsp); pRsp->pCont = pCont; - dndSendRpcRsp(pWrapper, pRsp); + dndSendRsp(pWrapper, pRsp); taosMemoryFree(pRsp); } From a942717855cb75931b8617f533ab2a703e6ff326 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 28 Mar 2022 11:56:01 +0800 Subject: [PATCH 105/105] [del walSize check] --- tests/test/c/tmqDemo.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 1690a5fb3e..63a3962d0e 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -691,12 +691,12 @@ int main(int32_t argc, char *argv[]) { float rowsSpeed = totalRows / seconds; float msgsSpeed = totalMsgs / seconds; - walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); - if (walLogSize <= 0) { - printf("vnode2/wal size incorrect!"); - /*exit(-1);*/ - } else { - if (0 == g_stConfInfo.simCase) { + if (0 == g_stConfInfo.simCase) { + walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); + if (walLogSize <= 0) { + printf("%s size incorrect!", g_stConfInfo.vnodeWalPath); + exit(-1); + } else { pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); } }