From d3446a5c7aa1daeabba0bf3959d7ad9bc8da46c9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 09:32:21 +0000 Subject: [PATCH 001/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 2 +- source/libs/tdb/src/db/tdbPage.c | 253 ---------------------------- source/libs/tdb/src/inc/tdbPage.h | 26 +-- source/libs/tdb/src/page/tdbPage.c | 244 +++++++++++++++++---------- source/libs/tdb/src/page/tdbPageL.c | 45 +++-- 5 files changed, 198 insertions(+), 372 deletions(-) delete mode 100644 source/libs/tdb/src/db/tdbPage.c diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c6e8c9dca9..423de0b53c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -414,7 +414,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { } // TODO: need to update the SPage.nFree - pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; + // pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; pPage->nOverflow = 0; return 0; diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c deleted file mode 100644 index df158de756..0000000000 --- a/source/libs/tdb/src/db/tdbPage.c +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -typedef struct __attribute__((__packed__)) { - u8 szCell[2]; - u8 nxOffset[2]; -} SFreeCell; - -typedef struct __attribute__((__packed__)) { - u8 szCell[3]; - u8 nxOffset[3]; -} SFreeCellL; - -/* For small page */ -#define TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCell *)(PCELL))->szCell) -#define TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCell *)(PCELL))->nxOffset) - -#define TDB_SPAGE_FREE_CELL_SIZE(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL))[0] -#define TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) ((u16 *)TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL))[0] - -#define TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) (TDB_SPAGE_FREE_CELL_SIZE(PCELL) = (SIZE)) -#define TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) (TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL) = (OFFSET)) - -/* For large page */ -#define TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) (((SFreeCellL *)(PCELL))->szCell) -#define TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) (((SFreeCellL *)(PCELL))->nxOffset) - -#define TDB_LPAGE_FREE_CELL_SIZE(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL)) -#define TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) TDB_GET_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) - -#define TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL), SIZE) -#define TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET) TDB_PUT_U24(TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL), OFFSET) - -/* For page */ -#define TDB_PAGE_FREE_CELL_SIZE_PTR(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE_PTR(PCELL) : TDB_SPAGE_FREE_CELL_SIZE_PTR(PCELL)) -#define TDB_PAGE_FREE_CELL_NXOFFSET_PTR(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET_PTR(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET_PTR(PCELL)) - -#define TDB_PAGE_FREE_CELL_SIZE(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_SIZE(PCELL) : TDB_SPAGE_FREE_CELL_SIZE(PCELL)) -#define TDB_PAGE_FREE_CELL_NXOFFSET(PPAGE, PCELL) \ - (TDB_IS_LARGE_PAGE(pPage) ? TDB_LPAGE_FREE_CELL_NXOFFSET(PCELL) : TDB_SPAGE_FREE_CELL_NXOFFSET(PCELL)) - -#define TDB_PAGE_FREE_CELL_SIZE_SET(PPAGE, PCELL, SIZE) \ - do { \ - if (TDB_IS_LARGE_PAGE(PPAGE)) { \ - TDB_LPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ - } else { \ - TDB_SPAGE_FREE_CELL_SIZE_SET(PCELL, SIZE); \ - } \ - } while (0) -#define TDB_PAGE_FREE_CELL_NXOFFSET_SET(PPAGE, PCELL, OFFSET) \ - do { \ - if (TDB_IS_LARGE_PAGE(PPAGE)) { \ - TDB_LPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ - } else { \ - TDB_SPAGE_FREE_CELL_NXOFFSET_SET(PCELL, OFFSET); \ - } \ - } while (0) - -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); -static int tdbPageDefragment(SPage *pPage); - -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { - SPage *pPage; - u8 *ptr; - int size; - - ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); - - *ppPage = NULL; - size = pageSize + sizeof(*pPage); - - ptr = (u8 *)((*xMalloc)(arg, size)); - if (pPage == NULL) { - return -1; - } - - memset(ptr, 0, size); - pPage = (SPage *)(ptr + pageSize); - - pPage->pData = ptr; - pPage->pageSize = pageSize; - if (pageSize < 65536) { - pPage->szOffset = 2; - pPage->szPageHdr = sizeof(SPageHdr); - pPage->szFreeCell = sizeof(SFreeCell); - } else { - pPage->szOffset = 3; - pPage->szPageHdr = sizeof(SPageHdrL); - pPage->szFreeCell = sizeof(SFreeCellL); - } - TDB_INIT_PAGE_LOCK(pPage); - - /* TODO */ - - *ppPage = pPage; - return 0; -} - -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { - u8 *ptr; - - ptr = pPage->pData; - (*xFree)(arg, ptr); - - return 0; -} - -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { - int ret; - SCell *pTarget; - u8 *pTmp; - int j; - - if (pPage->nOverflow || szCell + pPage->szOffset > pPage->nFree) { - // TODO: need to figure out if pCell may be used by outside of this function - j = pPage->nOverflow++; - - pPage->apOvfl[j] = pCell; - pPage->aiOvfl[j] = idx; - } else { - ret = tdbPageAllocate(pPage, szCell, &pTarget); - if (ret < 0) { - return -1; - } - - memcpy(pTarget, pCell, szCell); - pTmp = pPage->pCellIdx + idx * pPage->szOffset; - memmove(pTmp + pPage->szOffset, pTmp, pPage->pFreeStart - pTmp - pPage->szOffset); - TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); - TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); - } - - return 0; -} - -int tdbPageDropCell(SPage *pPage, int idx) { - // TODO - return 0; -} - -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { - SCell *pCell; - SFreeCell *pFreeCell; - u8 *pOffset; - int ret; - - ASSERT(pPage->nFree > size + pPage->szOffset); - - pCell = NULL; - *ppCell = NULL; - - // 1. Try to allocate from the free space area - if (pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset) { - pPage->pFreeEnd -= size; - pPage->pFreeStart += pPage->szOffset; - pCell = pPage->pFreeEnd; - } - - // 2. Try to allocate from the page free list - if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= pPage->szOffset) && TDB_PAGE_FCELL(pPage)) { - int szCell; - int nxOffset; - - pCell = pPage->pData + TDB_PAGE_FCELL(pPage); - pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell - : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - - for (;;) { - // Find a cell - if (szCell >= size) { - if (szCell - size >= pPage->szFreeCell) { - SCell *pTmpCell = pCell + size; - - TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); - TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); - // TODO: *pOffset = pTmpCell - pPage->pData; - } else { - TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); - // TODO: *pOffset = nxOffset; - } - break; - } - - // Not find a cell yet - if (nxOffset > 0) { - pCell = pPage->pData + nxOffset; - pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - continue; - } else { - pCell = NULL; - break; - } - } - - if (pCell) { - pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; - } - } - - // 3. Try to dfragment and allocate again - if (pCell == NULL) { - ret = tdbPageDefragment(pPage); - if (ret < 0) { - return -1; - } - - ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + pPage->szOffset); - ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); - - // Allocate from the free space area again - pPage->pFreeEnd -= size; - pPage->pFreeStart += pPage->szOffset; - pCell = pPage->pFreeEnd; - } - - ASSERT(pCell != NULL); - - pPage->nFree = pPage->nFree - size - pPage->szOffset; - *ppCell = pCell; - return 0; -} - -static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { - // TODO - return 0; -} - -static int tdbPageDefragment(SPage *pPage) { - // TODO - ASSERT(0); - return 0; -} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 6479574abe..418a1b6626 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -45,6 +45,9 @@ typedef struct { // cell offset at idx int (*getCellOffset)(SPage *, int); void (*setCellOffset)(SPage *, int, int); + // free cell info + void (*getFreeCellInfo)(SCell *pCell, int *szCell, int *nxOffset); + void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset); } SPageMethods; // Page footer @@ -59,20 +62,19 @@ struct SPage { SPageMethods *pPageMethods; // Fields below used by pager and am u8 szAmHdr; - u8 *pPageHdr; u8 *pAmHdr; + u8 *pPageHdr; u8 *pCellIdx; u8 *pFreeStart; u8 *pFreeEnd; SPageFtr *pPageFtr; - int kLen; // key length of the page, -1 for unknown - int vLen; // value length of the page, -1 for unknown - int nFree; - int maxLocal; - int minLocal; int nOverflow; SCell *apOvfl[4]; int aiOvfl[4]; + int kLen; // key length of the page, -1 for unknown + int vLen; // value length of the page, -1 for unknown + int maxLocal; + int minLocal; // Fields used by SPCache TDB_PCACHE_PAGE }; @@ -92,8 +94,6 @@ struct SPage { #define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) #define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) - #define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) // For page lock @@ -119,10 +119,12 @@ struct SPage { }) // APIs -int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); -int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); -int tdbPageDropCell(SPage *pPage, int idx); +int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); +int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); +void tdbPageZero(SPage *pPage); +void tdbPageInit(SPage *pPage); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); +int tdbPageDropCell(SPage *pPage, int idx); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 4ec3a895e7..e198891338 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -18,10 +18,11 @@ extern SPageMethods pageMethods; extern SPageMethods pageLargeMethods; -typedef struct __attribute__((__packed__)) { - u16 szCell; - u16 nxOffset; -} SFreeCell; +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) +#define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) +#define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) +#define TDB_PAGE_MAX_FREE_BLOCK(pPage) \ + ((pPage)->pageSize - (pPage)->szAmHdr - TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); static int tdbPageDefragment(SPage *pPage); @@ -44,6 +45,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) memset(ptr, 0, size); pPage = (SPage *)(ptr + pageSize); + TDB_INIT_PAGE_LOCK(pPage); pPage->pData = ptr; pPage->pageSize = pageSize; if (pageSize < 65536) { @@ -51,9 +53,6 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) } else { pPage->pPageMethods = &pageLargeMethods; } - TDB_INIT_PAGE_LOCK(pPage); - - /* TODO */ *ppPage = pPage; return 0; @@ -68,29 +67,64 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) return 0; } +void tdbPageZero(SPage *pPage) { + TDB_PAGE_NCELLS_SET(pPage, 0); + TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); + TDB_PAGE_FCELL_SET(pPage, 0); + TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_MAX_FREE_BLOCK(pPage)); + tdbPageInit(pPage); +} + +void tdbPageInit(SPage *pPage) { + pPage->pAmHdr = pPage->pData; + pPage->pPageHdr = pPage->pAmHdr + pPage->szAmHdr; + pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); + pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); + pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); + pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + pPage->nOverflow = 0; + + ASSERT(pPage->pFreeEnd >= pPage->pFreeStart); + ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); +} + int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { + int nFree; int ret; - SCell *pTarget; - u8 *pTmp; - int j; + int nCells; + int lidx; // local idx + SCell *pNewCell; - if (pPage->nOverflow || szCell + TDB_PAGE_OFFSET_SIZE(pPage) > pPage->nFree) { - // TODO: need to figure out if pCell may be used by outside of this function - j = pPage->nOverflow++; + ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage)); - pPage->apOvfl[j] = pCell; - pPage->aiOvfl[j] = idx; - } else { - ret = tdbPageAllocate(pPage, szCell, &pTarget); - if (ret < 0) { - return -1; + nFree = TDB_PAGE_NFREE(pPage); + nCells = TDB_PAGE_NCELLS(pPage); + + if (nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + // page must has enough space to hold the cell locally + ret = tdbPageAllocate(pPage, szCell, &pNewCell); + ASSERT(ret == 0); + + memcpy(pNewCell, pCell, szCell); + + if (pPage->nOverflow == 0) { + lidx = idx; + } else { + // TODO + // lidx = ; } - memcpy(pTarget, pCell, szCell); - pTmp = pPage->pCellIdx + idx * TDB_PAGE_OFFSET_SIZE(pPage); - memmove(pTmp + TDB_PAGE_OFFSET_SIZE(pPage), pTmp, pPage->pFreeStart - pTmp - TDB_PAGE_OFFSET_SIZE(pPage)); - TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pTarget - pPage->pData); - TDB_PAGE_NCELLS_SET(pPage, TDB_PAGE_NCELLS(pPage) + 1); + // no overflow cell exists in this page + u8 *src = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * lidx; + u8 *dest = src + TDB_PAGE_OFFSET_SIZE(pPage); + memmove(dest, src, pPage->pFreeStart - dest); + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, TDB_PAGE_OFFSET_SIZE(pPage) * lidx); + TDB_PAGE_NCELLS_SET(pPage, nCells + 1); + } else { + // TODO: page not has enough space + pPage->apOvfl[pPage->nOverflow] = pCell; + pPage->aiOvfl[pPage->nOverflow] = idx; + pPage->nOverflow++; } return 0; @@ -101,73 +135,80 @@ int tdbPageDropCell(SPage *pPage, int idx) { return 0; } -static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { - SCell *pCell; - SFreeCell *pFreeCell; - u8 *pOffset; - int ret; +static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { + SCell *pFreeCell; + u8 *pOffset; + int nFree; + int ret; + int cellFree; - ASSERT(pPage->nFree > size + TDB_PAGE_OFFSET_SIZE(pPage)); - - pCell = NULL; *ppCell = NULL; + nFree = TDB_PAGE_NFREE(pPage); - // 1. Try to allocate from the free space area - if (pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)) { - pPage->pFreeEnd -= size; + ASSERT(nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)); + ASSERT(TDB_PAGE_CCELLS(pPage) == pPage->pFreeEnd - pPage->pData); + + // 1. Try to allocate from the free space block area + if (pPage->pFreeEnd - pPage->pFreeStart >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); - pCell = pPage->pFreeEnd; + pPage->pFreeEnd -= szCell; + + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); + + *ppCell = pPage->pFreeEnd; + return 0; } // 2. Try to allocate from the page free list - if ((pCell == NULL) && (pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) && - TDB_PAGE_FCELL(pPage)) { -#if 0 - int szCell; - int nxOffset; - - pCell = pPage->pData + TDB_PAGE_FCELL(pPage); - pOffset = TDB_IS_LARGE_PAGE(pPage) ? ((SPageHdrL *)(pPage->pPageHdr))[0].fCell - : (u8 *)&(((SPageHdr *)(pPage->pPageHdr))[0].fCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); + cellFree = TDB_PAGE_FCELL(pPage); + ASSERT(cellFree == 0 || cellFree > pPage->pFreeEnd - pPage->pData); + if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { + SCell *pPrevFreeCell = NULL; + int szPrevFreeCell; + int szFreeCell; + int nxFreeCell; + int newSize; for (;;) { - // Find a cell - if (szCell >= size) { - if (szCell - size >= pPage->szFreeCell) { - SCell *pTmpCell = pCell + size; + if (cellFree == 0) break; - TDB_PAGE_FREE_CELL_SIZE_SET(pPage, pTmpCell, szCell - size); - TDB_PAGE_FREE_CELL_NXOFFSET_SET(pPage, pTmpCell, nxOffset); - // TODO: *pOffset = pTmpCell - pPage->pData; + pFreeCell = pPage->pData + cellFree; + pPage->pPageMethods->getFreeCellInfo(pFreeCell, &szFreeCell, &nxFreeCell); + + if (szFreeCell >= szCell) { + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + TDB_PAGE_NFREE_SET(pPage, nFree - TDB_PAGE_OFFSET_SIZE(pPage) - szCell); + *ppCell = pFreeCell; + + newSize = szFreeCell - szCell; + pFreeCell += szCell; + if (newSize >= TDB_PAGE_FREE_CELL_SIZE(pPage)) { + pPage->pPageMethods->setFreeCellInfo(pFreeCell, newSize, nxFreeCell); + if (pPrevFreeCell) { + pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, pFreeCell - pPage->pData); + } else { + TDB_PAGE_FCELL_SET(pPage, pFreeCell - pPage->pData); + } + + return 0; } else { - TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_NFREE(pPage) + szCell - size); - // TODO: *pOffset = nxOffset; + if (pPrevFreeCell) { + pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, nxFreeCell); + } else { + TDB_PAGE_FCELL_SET(pPage, nxFreeCell); + } } - break; - } - - // Not find a cell yet - if (nxOffset > 0) { - pCell = pPage->pData + nxOffset; - pOffset = TDB_PAGE_FREE_CELL_NXOFFSET_PTR(pPage, pCell); - szCell = TDB_PAGE_FREE_CELL_SIZE(pPage, pCell); - nxOffset = TDB_PAGE_FREE_CELL_NXOFFSET(pPage, pCell); - continue; } else { - pCell = NULL; - break; + pPrevFreeCell = pFreeCell; + szPrevFreeCell = szFreeCell; + cellFree = nxFreeCell; } } - - if (pCell) { - pPage->pFreeStart = pPage->pFreeStart + pPage->szOffset; - } -#endif } - // 3. Try to dfragment and allocate again +// 3. Try to dfragment and allocate again +#if 0 if (pCell == NULL) { ret = tdbPageDefragment(pPage); if (ret < 0) { @@ -175,7 +216,7 @@ static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { } ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)); - ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); + // ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); // Allocate from the free space area again pPage->pFreeEnd -= size; @@ -185,8 +226,9 @@ static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell) { ASSERT(pCell != NULL); - pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); + // pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); *ppCell = pCell; +#endif return 0; } @@ -210,6 +252,11 @@ typedef struct __attribute__((__packed__)) { u16 nFree; } SPageHdr; +typedef struct __attribute__((__packed__)) { + u16 szCell; + u16 nxOffset; +} SFreeCell; + // flags static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } @@ -253,20 +300,35 @@ static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { ((u16 *)pPage->pCellIdx)[idx] = (u16)offset; } +// free cell info +static inline void getPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { + SFreeCell *pFreeCell = (SFreeCell *)pCell; + *szCell = pFreeCell->szCell; + *nxOffset = pFreeCell->nxOffset; +} + +static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { + SFreeCell *pFreeCell = (SFreeCell *)pCell; + pFreeCell->szCell = szCell; + pFreeCell->nxOffset = nxOffset; +} + SPageMethods pageMethods = { - 2, // szOffset - sizeof(SPageHdr), // szPageHdr - sizeof(SFreeCell), // szFreeCell - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp - getPageCellNum, // getCellNum - setPageCellNum, // setCellNum - getPageCellBody, // getCellBody - setPageCellBody, // setCellBody - getPageCellFree, // getCellFree - setPageCellFree, // setCellFree - getPageNFree, // getFreeBytes - setPageNFree, // setFreeBytes - getPageCellOffset, // getCellOffset - setPageCellOffset // setCellOffset + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell + getPageFlags, // getPageFlags + setPageFlags, // setFlagsp + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset, // setCellOffset + getPageFreeCellInfo, // getFreeCellInfo + setPageFreeCellInfo // setFreeCellInfo }; \ No newline at end of file diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index e7c60118d2..e3c1bb8d37 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -66,20 +66,35 @@ static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset); } +// free cell info +static inline void getPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + *szCell = TDB_GET_U24(pFreeCell->szCell); + *nxOffset = TDB_GET_U24(pFreeCell->nxOffset); +} + +static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + TDB_PUT_U24(pFreeCell->szCell, szCell); + TDB_PUT_U24(pFreeCell->nxOffset, nxOffset); +} + SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp - getPageCellNum, // getCellNum - setPageCellNum, // setCellNum - getPageCellBody, // getCellBody - setPageCellBody, // setCellBody - getPageCellFree, // getCellFree - setPageCellFree, // setCellFree - getPageNFree, // getFreeBytes - setPageNFree, // setFreeBytes - getPageCellOffset, // getCellOffset - setPageCellOffset // setCellOffset + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell + getPageFlags, // getPageFlags + setPageFlags, // setFlagsp + getPageCellNum, // getCellNum + setPageCellNum, // setCellNum + getPageCellBody, // getCellBody + setPageCellBody, // setCellBody + getPageCellFree, // getCellFree + setPageCellFree, // setCellFree + getPageNFree, // getFreeBytes + setPageNFree, // setFreeBytes + getPageCellOffset, // getCellOffset + setPageCellOffset, // setCellOffset + getPageFreeCellInfo, // getFreeCellInfo + setPageFreeCellInfo // setFreeCellInfo }; \ No newline at end of file From 57a92869ae28c7ce6f2776e521edcb879f9a4c77 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 09:45:49 +0000 Subject: [PATCH 002/121] more TDB --- source/libs/tdb/src/page/tdbPage.c | 46 +++++++++++------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index e198891338..892d7ecb31 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -141,6 +141,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { int nFree; int ret; int cellFree; + SCell *pCell = NULL; *ppCell = NULL; nFree = TDB_PAGE_NFREE(pPage); @@ -150,14 +151,10 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { // 1. Try to allocate from the free space block area if (pPage->pFreeEnd - pPage->pFreeStart >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { - pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); pPage->pFreeEnd -= szCell; - + pCell = pPage->pFreeEnd; TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); - TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); - - *ppCell = pPage->pFreeEnd; - return 0; + goto _alloc_finish; } // 2. Try to allocate from the page free list @@ -177,9 +174,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { pPage->pPageMethods->getFreeCellInfo(pFreeCell, &szFreeCell, &nxFreeCell); if (szFreeCell >= szCell) { - pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); - TDB_PAGE_NFREE_SET(pPage, nFree - TDB_PAGE_OFFSET_SIZE(pPage) - szCell); - *ppCell = pFreeCell; + pCell = pFreeCell; newSize = szFreeCell - szCell; pFreeCell += szCell; @@ -191,7 +186,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { TDB_PAGE_FCELL_SET(pPage, pFreeCell - pPage->pData); } - return 0; + goto _alloc_finish; } else { if (pPrevFreeCell) { pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, nxFreeCell); @@ -207,28 +202,21 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { } } -// 3. Try to dfragment and allocate again -#if 0 - if (pCell == NULL) { - ret = tdbPageDefragment(pPage); - if (ret < 0) { - return -1; - } + // 3. Try to dfragment and allocate again + tdbPageDefragment(pPage); + ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + ASSERT(nFree == TDB_PAGE_NFREE(pPage)); + ASSERT(pPage->pFreeEnd - pPage->pData == TDB_PAGE_CCELLS(pPage)); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart > size + TDB_PAGE_OFFSET_SIZE(pPage)); - // ASSERT(pPage->nFree == pPage->pFreeEnd - pPage->pFreeStart); + pPage->pFreeEnd -= szCell; + pCell = pPage->pFreeEnd; + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); - // Allocate from the free space area again - pPage->pFreeEnd -= size; - pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); - pCell = pPage->pFreeEnd; - } - - ASSERT(pCell != NULL); - - // pPage->nFree = pPage->nFree - size - TDB_PAGE_OFFSET_SIZE(pPage); +_alloc_finish: + ASSERT(pCell); + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); + TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); *ppCell = pCell; -#endif return 0; } From 0353e590814cfec20350627f07e5e11e3bcf6c67 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 12:02:26 +0000 Subject: [PATCH 003/121] more TDB --- source/libs/tdb/src/page/tdbPage.c | 34 ++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 892d7ecb31..1c9984cd01 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -220,8 +220,38 @@ _alloc_finish: return 0; } -static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int size) { - // TODO +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { + int nFree; + int cellFree; + u8 *dest; + u8 *src; + + ASSERT(pCell >= pPage->pFreeEnd); + ASSERT(pCell + szCell <= (u8 *)(pPage->pPageFtr)); + ASSERT(pCell == TDB_PAGE_CELL_AT(pPage, idx)); + + nFree = TDB_PAGE_NFREE(pPage); + + if (pCell == pPage->pFreeEnd) { + pPage->pFreeEnd += szCell; + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + } else { + if (szCell >= TDB_PAGE_FREE_CELL_SIZE(pPage)) { + cellFree = TDB_PAGE_FCELL(pPage); + pPage->pPageMethods->setFreeCellInfo(pCell, szCell, cellFree); + TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); + } else { + ASSERT(0); + } + } + + dest = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * idx; + src = dest + TDB_PAGE_OFFSET_SIZE(pPage); + memmove(dest, src, pPage->pFreeStart - src); + + pPage->pFreeStart -= TDB_PAGE_OFFSET_SIZE(pPage); + nFree = nFree + szCell + TDB_PAGE_OFFSET_SIZE(pPage); + TDB_PAGE_NFREE_SET(pPage, nFree); return 0; } From dd7344c7800d88c6f444031319965130824cc7b7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 12:41:15 +0000 Subject: [PATCH 004/121] more TDB --- source/libs/tdb/src/inc/tdbPage.h | 1 + source/libs/tdb/src/page/tdbPage.c | 58 ++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 418a1b6626..9c40e9d5c1 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -75,6 +75,7 @@ struct SPage { int vLen; // value length of the page, -1 for unknown int maxLocal; int minLocal; + int (*xCellSize)(SCell *pCell); // Fields used by SPCache TDB_PCACHE_PAGE }; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 1c9984cd01..b10cd7b0ec 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -256,8 +256,62 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { } static int tdbPageDefragment(SPage *pPage) { - // TODO - ASSERT(0); + int nFree; + int nCells; + SCell *pCell; + SCell *pNextCell; + SCell *pTCell; + int szCell; + int idx; + int iCell; + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart < nFree); + + nFree = TDB_PAGE_NFREE(pPage); + nCells = TDB_PAGE_NCELLS(pPage); + + // Loop to compact the page content + // Here we use an O(n^2) algorithm to do the job since + // this is a low frequency job. + pNextCell = (u8 *)pPage->pPageFtr; + pCell = NULL; + for (iCell = 0;; iCell++) { + // compact over + if (iCell == nCells) { + pPage->pFreeEnd = pNextCell; + break; + } + + for (int i = 0; i < nCells; i++) { + if (TDB_PAGE_CELL_OFFSET_AT(pPage, i) < pNextCell - pPage->pData) { + pTCell = TDB_PAGE_CELL_AT(pPage, i); + if (pCell == NULL || pCell < pTCell) { + pCell = pTCell; + idx = i; + } + } else { + continue; + } + } + + ASSERT(pCell != NULL); + + szCell = (*pPage->xCellSize)(pCell); + + ASSERT(pCell + szCell <= pNextCell); + if (pCell + szCell < pNextCell) { + memmove(pNextCell - szCell, pCell, szCell); + } + + pCell = NULL; + pNextCell = pNextCell - szCell; + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, pNextCell - pPage->pData); + } + + ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); + TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); + TDB_PAGE_FCELL_SET(pPage, 0); + return 0; } From f91124ef8edc8f2ae300270b46798a9885f7b81b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 13:00:51 +0000 Subject: [PATCH 005/121] more TDB --- source/libs/tdb/src/page/tdbPage.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index b10cd7b0ec..9d679118e7 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -26,6 +26,7 @@ extern SPageMethods pageLargeMethods; static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); static int tdbPageDefragment(SPage *pPage); +static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg) { SPage *pPage; @@ -131,7 +132,25 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { } int tdbPageDropCell(SPage *pPage, int idx) { - // TODO + int lidx; + SCell *pCell; + int szCell; + int nCells; + + nCells = TDB_PAGE_NCELLS(pPage); + + if (pPage->nOverflow == 0) { + lidx = idx; + } else { + // TODO + } + + pCell = TDB_PAGE_CELL_AT(pPage, lidx); + szCell = (*pPage->xCellSize)(pCell); + tdbPageFree(pPage, lidx, pCell, szCell); + + TDB_PAGE_NCELLS_SET(pPage, nCells - 1); + return 0; } From 6eb2e1b288d1363abf998b1604978a64340ad20a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 13:13:08 +0000 Subject: [PATCH 006/121] more TDB --- source/libs/tdb/src/page/tdbPage.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 9d679118e7..d91a32cb7c 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -136,19 +136,26 @@ int tdbPageDropCell(SPage *pPage, int idx) { SCell *pCell; int szCell; int nCells; + int iOvfl; nCells = TDB_PAGE_NCELLS(pPage); - if (pPage->nOverflow == 0) { - lidx = idx; - } else { - // TODO + ASSERT(idx >= 0 && idx < nCells + pPage->nOverflow); + + iOvfl = 0; + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] == idx) { + // TODO: remove the over flow cell + return 0; + } else if (pPage->aiOvfl[iOvfl] > idx) { + break; + } } + lidx = idx - iOvfl; pCell = TDB_PAGE_CELL_AT(pPage, lidx); szCell = (*pPage->xCellSize)(pCell); tdbPageFree(pPage, lidx, pCell, szCell); - TDB_PAGE_NCELLS_SET(pPage, nCells - 1); return 0; From 7d5030bc7fc2576411698f718f50fbaf7ff697ec Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 15:04:29 +0000 Subject: [PATCH 007/121] more TDB --- source/libs/tdb/src/page/tdbPage.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index d91a32cb7c..2c9563d36d 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -145,7 +145,13 @@ int tdbPageDropCell(SPage *pPage, int idx) { iOvfl = 0; for (; iOvfl < pPage->nOverflow; iOvfl++) { if (pPage->aiOvfl[iOvfl] == idx) { - // TODO: remove the over flow cell + // remove the over flow cell + for (; (++iOvfl) < pPage->nOverflow;) { + pPage->aiOvfl[iOvfl - 1] = pPage->aiOvfl[iOvfl] - 1; + pPage->apOvfl[iOvfl - 1] = pPage->apOvfl[iOvfl]; + } + + pPage->nOverflow--; return 0; } else if (pPage->aiOvfl[iOvfl] > idx) { break; @@ -158,6 +164,11 @@ int tdbPageDropCell(SPage *pPage, int idx) { tdbPageFree(pPage, lidx, pCell, szCell); TDB_PAGE_NCELLS_SET(pPage, nCells - 1); + for (; iOvfl < pPage->nOverflow; iOvfl++) { + pPage->aiOvfl[iOvfl]--; + ASSERT(pPage->aiOvfl[iOvfl] > 0); + } + return 0; } From 7a5985ae46de83a3fbe359aa5d79e21e54b4a8de Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 20 Mar 2022 15:39:51 +0000 Subject: [PATCH 008/121] more TDB --- source/libs/tdb/src/page/tdbPage.c | 39 ++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 2c9563d36d..4673a561e6 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -91,8 +91,8 @@ void tdbPageInit(SPage *pPage) { int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { int nFree; - int ret; int nCells; + int iOvfl; int lidx; // local idx SCell *pNewCell; @@ -100,32 +100,45 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); + iOvfl = 0; + + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] >= idx) { + break; + } + } + + lidx = idx - iOvfl; if (nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { // page must has enough space to hold the cell locally - ret = tdbPageAllocate(pPage, szCell, &pNewCell); - ASSERT(ret == 0); + tdbPageAllocate(pPage, szCell, &pNewCell); memcpy(pNewCell, pCell, szCell); - if (pPage->nOverflow == 0) { - lidx = idx; - } else { - // TODO - // lidx = ; - } - // no overflow cell exists in this page u8 *src = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * lidx; u8 *dest = src + TDB_PAGE_OFFSET_SIZE(pPage); memmove(dest, src, pPage->pFreeStart - dest); TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, TDB_PAGE_OFFSET_SIZE(pPage) * lidx); TDB_PAGE_NCELLS_SET(pPage, nCells + 1); + + ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); } else { - // TODO: page not has enough space - pPage->apOvfl[pPage->nOverflow] = pCell; - pPage->aiOvfl[pPage->nOverflow] = idx; + // add the cell as an overflow cell + for (int i = pPage->nOverflow; i > iOvfl; i--) { + pPage->apOvfl[i] = pPage->apOvfl[i - 1]; + pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; + } + + pPage->apOvfl[iOvfl] = pCell; + pPage->aiOvfl[iOvfl] = idx; pPage->nOverflow++; + iOvfl++; + } + + for (; iOvfl < pPage->nOverflow; iOvfl++) { + pPage->aiOvfl[iOvfl]++; } return 0; From 13b7ba3edb940b226754215ecd17af97025eb727 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 02:27:46 +0000 Subject: [PATCH 009/121] refact TDB --- source/libs/tdb/src/db/tdbBtree.c | 63 +++++++++++++++++------------- source/libs/tdb/src/inc/tdbPage.h | 44 +++++++++++++-------- source/libs/tdb/src/page/tdbPage.c | 20 ++++++++-- 3 files changed, 80 insertions(+), 47 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 423de0b53c..621afc9dfb 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -145,7 +145,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } if (pCur->idx == -1) { - ASSERT(TDB_PAGE_NCELLS(pCur->pPage) == 0); + ASSERT(TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0); idx = 0; } else { if (cret > 0) { @@ -218,7 +218,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p pCur->iPage = 0; - if (TDB_PAGE_NCELLS(pCur->pPage) == 0) { + if (TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0) { // Current page is empty ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); return 0; @@ -231,7 +231,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p SCellDecoder cd = {0}; pPage = pCur->pPage; - nCells = TDB_PAGE_NCELLS(pPage); + nCells = TDB_PAGE_TOTAL_CELLS(pPage); lidx = 0; ridx = nCells - 1; @@ -242,7 +242,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p midx = (lidx + ridx) >> 1; - pCell = TDB_PAGE_CELL_AT(pPage, midx); + pCell = tdbPageGetCell(pPage, midx); ret = tdbBtreeDecodeCell(pPage, pCell, &cd); if (ret < 0) { // TODO: handle error @@ -283,7 +283,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p } else { // TODO: reset cd as uninitialized pCur->idx = midx + 1; - pCell = TDB_PAGE_CELL_AT(pPage, midx + 1); + pCell = tdbPageGetCell(pPage, midx + 1); tdbBtreeDecodeCell(pPage, pCell, &cd); tdbBtCursorMoveToChild(pCur, cd.pgno); } @@ -391,16 +391,12 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { } else { pPage->szAmHdr = sizeof(SBtPageHdr); } - pPage->pPageHdr = pPage->pData; - pPage->pAmHdr = pPage->pPageHdr + pPage->pPageMethods->szPageHdr; - pPage->pCellIdx = pPage->pAmHdr + pPage->szAmHdr; - pPage->pFreeStart = pPage->pCellIdx + pPage->pPageMethods->szOffset * TDB_PAGE_NCELLS(pPage); - pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); - pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + pPage->xCellSize = NULL; // TODO + + tdbPageInit(pPage); TDB_BTREE_ASSERT_FLAG(flags); - // Init other fields if (isLeaf) { pPage->kLen = pBt->keyLen; pPage->vLen = pBt->valLen; @@ -413,30 +409,41 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { pPage->minLocal = pBt->minLocal; } - // TODO: need to update the SPage.nFree - // pPage->nFree = pPage->pFreeEnd - pPage->pFreeStart; - pPage->nOverflow = 0; - return 0; } static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u16 flags; SBTree *pBt; + u8 isLeaf; flags = ((SBtreeZeroPageArg *)arg)->flags; pBt = ((SBtreeZeroPageArg *)arg)->pBt; - - pPage->pPageHdr = pPage->pData; + isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Init the page header TDB_PAGE_FLAGS_SET(pPage, flags); - TDB_PAGE_NCELLS_SET(pPage, 0); - TDB_PAGE_CCELLS_SET(pPage, pBt->pageSize - sizeof(SPageFtr)); - TDB_PAGE_FCELL_SET(pPage, 0); - TDB_PAGE_NFREE_SET(pPage, 0); + // Set szAmHdr + if (isLeaf) { + pPage->szAmHdr = 0; + } else { + pPage->szAmHdr = sizeof(SBtPageHdr); + } + pPage->xCellSize = NULL; // TODO - tdbBtreeInitPage(pPage, (void *)pBt); + tdbPageZero(pPage); + + if (isLeaf) { + pPage->kLen = pBt->keyLen; + pPage->vLen = pBt->valLen; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + } else { + pPage->kLen = pBt->keyLen; + pPage->vLen = sizeof(SPgno); + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + } return 0; } @@ -453,7 +460,8 @@ typedef struct { } SBtreeBalanceHelper; static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { - int nCells = TDB_PAGE_NCELLS(pFrom); +#if 0 + int nCells = TDB_PAGE_TOTAL_CELLS(pFrom); int cCells = TDB_PAGE_CCELLS(pFrom); int fCell = TDB_PAGE_FCELL(pFrom); int nFree = TDB_PAGE_NFREE(pFrom); @@ -469,6 +477,7 @@ static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { TDB_PAGE_NFREE_SET(pTo, nFree); // TODO: update other fields +#endif return 0; } @@ -529,7 +538,7 @@ static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { SBTree *pBt; pParent = pBlh->pParent; - nCells = TDB_PAGE_NCELLS(pParent); + nCells = TDB_PAGE_TOTAL_CELLS(pParent); nChild = nCells + 1; pBt = pBlh->pBt; @@ -556,7 +565,7 @@ static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { if (idxStart + i == nCells) { pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; } else { - pCell = TDB_PAGE_CELL_AT(pParent, idxStart + i); + pCell = tdbPageGetCell(pParent, idxStart + i); // TODO: no need to decode the payload part, and even the kLen, vLen part // we only need the pgno part ret = tdbBtreeDecodeCell(pParent, pCell, &cd); @@ -610,7 +619,7 @@ static int tdbBtreeBalanceStep2(SBtreeBalanceHelper *pBlh) { if (cidx < limit) { // Get local cells - pCell = TDB_PAGE_CELL_AT(pPage, cidx); + pCell = tdbPageGetCell(pPage, cidx); } else if (cidx == limit) { // Get overflow cells pCell = pPage->apOvfl[oidx++]; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 9c40e9d5c1..82287b0921 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -57,8 +57,8 @@ typedef struct __attribute__((__packed__)) { struct SPage { pthread_spinlock_t lock; - u8 *pData; int pageSize; + u8 *pData; SPageMethods *pPageMethods; // Fields below used by pager and am u8 szAmHdr; @@ -81,21 +81,8 @@ struct SPage { }; /* For page */ -#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) -#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) -#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage) -#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage) -#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) -#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx) - -#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) -#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS) -#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS) -#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL) -#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) -#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) - -#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) +#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) +#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) // For page lock #define P_LOCK_SUCC 0 @@ -120,6 +107,8 @@ struct SPage { }) // APIs +#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) + int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); void tdbPageZero(SPage *pPage); @@ -127,6 +116,29 @@ void tdbPageInit(SPage *pPage); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); +static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { + SCell *pCell; + int iOvfl; + int lidx; + + ASSERT(idx >= 0 && idx < pPage->nOverflow + pPage->pPageMethods->getCellNum(pPage)); + + iOvfl = 0; + for (; iOvfl < pPage->nOverflow; iOvfl++) { + if (pPage->aiOvfl[iOvfl] == idx) { + pCell = pPage->apOvfl[iOvfl]; + return pCell; + } else if (pPage->aiOvfl[iOvfl] > idx) { + break; + } + } + + lidx = idx - iOvfl; + pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); + + return pCell; +} + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 4673a561e6..2b0e573aae 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -18,9 +18,20 @@ extern SPageMethods pageMethods; extern SPageMethods pageLargeMethods; -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) -#define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) -#define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) +#define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) +#define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) +#define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) +#define TDB_PAGE_CCELLS(pPage) (*(pPage)->pPageMethods->getCellBody)(pPage) +#define TDB_PAGE_FCELL(pPage) (*(pPage)->pPageMethods->getCellFree)(pPage) +#define TDB_PAGE_NFREE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage) +#define TDB_PAGE_CELL_OFFSET_AT(pPage, idx) (*(pPage)->pPageMethods->getCellOffset)(pPage, idx) +#define TDB_PAGE_NCELLS_SET(pPage, NCELLS) (*(pPage)->pPageMethods->setCellNum)(pPage, NCELLS) +#define TDB_PAGE_CCELLS_SET(pPage, CCELLS) (*(pPage)->pPageMethods->setCellBody)(pPage, CCELLS) +#define TDB_PAGE_FCELL_SET(pPage, FCELL) (*(pPage)->pPageMethods->setCellFree)(pPage, FCELL) +#define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) +#define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) +#define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) #define TDB_PAGE_MAX_FREE_BLOCK(pPage) \ ((pPage)->pageSize - (pPage)->szAmHdr - TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) @@ -47,8 +58,8 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) pPage = (SPage *)(ptr + pageSize); TDB_INIT_PAGE_LOCK(pPage); - pPage->pData = ptr; pPage->pageSize = pageSize; + pPage->pData = ptr; if (pageSize < 65536) { pPage->pPageMethods = &pageMethods; } else { @@ -125,6 +136,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); } else { + // TODO: make it extensible // add the cell as an overflow cell for (int i = pPage->nOverflow; i > iOvfl; i--) { pPage->apOvfl[i] = pPage->apOvfl[i - 1]; From cde7a7c6a1558a8bb76ad3d7bd83fef44fc89172 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 02:54:00 +0000 Subject: [PATCH 010/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 14 ++++++++------ source/libs/tdb/src/inc/tdbPage.h | 6 +++--- source/libs/tdb/src/page/tdbPage.c | 24 ++++++++++++++++-------- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 621afc9dfb..0c83dd609c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -381,19 +381,20 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { SBTree *pBt; u16 flags; u8 isLeaf; + u8 szAmHdr; pBt = (SBTree *)arg; flags = TDB_PAGE_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { - pPage->szAmHdr = 0; + szAmHdr = 0; } else { - pPage->szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SBtPageHdr); } pPage->xCellSize = NULL; // TODO - tdbPageInit(pPage); + tdbPageInit(pPage, szAmHdr); TDB_BTREE_ASSERT_FLAG(flags); @@ -416,6 +417,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u16 flags; SBTree *pBt; u8 isLeaf; + u8 szAmHdr; flags = ((SBtreeZeroPageArg *)arg)->flags; pBt = ((SBtreeZeroPageArg *)arg)->pBt; @@ -425,13 +427,13 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { TDB_PAGE_FLAGS_SET(pPage, flags); // Set szAmHdr if (isLeaf) { - pPage->szAmHdr = 0; + szAmHdr = 0; } else { - pPage->szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SBtPageHdr); } pPage->xCellSize = NULL; // TODO - tdbPageZero(pPage); + tdbPageZero(pPage, szAmHdr); if (isLeaf) { pPage->kLen = pBt->keyLen; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 82287b0921..58ae1f3963 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -61,7 +61,6 @@ struct SPage { u8 *pData; SPageMethods *pPageMethods; // Fields below used by pager and am - u8 szAmHdr; u8 *pAmHdr; u8 *pPageHdr; u8 *pCellIdx; @@ -111,8 +110,8 @@ struct SPage { int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -void tdbPageZero(SPage *pPage); -void tdbPageInit(SPage *pPage); +void tdbPageZero(SPage *pPage, u8 szAmHdr); +void tdbPageInit(SPage *pPage, u8 szAmHdr); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); @@ -134,6 +133,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { } lidx = idx - iOvfl; + ASSERT(lidx >= 0 && lidx < pPage->pPageMethods->getCellNum(pPage)); pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx); return pCell; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 2b0e573aae..11c9a9247e 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -32,8 +32,8 @@ extern SPageMethods pageLargeMethods; #define TDB_PAGE_NFREE_SET(pPage, NFREE) (*(pPage)->pPageMethods->setFreeBytes)(pPage, NFREE) #define TDB_PAGE_CELL_OFFSET_AT_SET(pPage, idx, OFFSET) (*(pPage)->pPageMethods->setCellOffset)(pPage, idx, OFFSET) #define TDB_PAGE_CELL_AT(pPage, idx) ((pPage)->pData + TDB_PAGE_CELL_OFFSET_AT(pPage, idx)) -#define TDB_PAGE_MAX_FREE_BLOCK(pPage) \ - ((pPage)->pageSize - (pPage)->szAmHdr - TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) +#define TDB_PAGE_MAX_FREE_BLOCK(pPage, szAmHdr) \ + ((pPage)->pageSize - (szAmHdr)-TDB_PAGE_HDR_SIZE(pPage) - sizeof(SPageFtr)) static int tdbPageAllocate(SPage *pPage, int size, SCell **ppCell); static int tdbPageDefragment(SPage *pPage); @@ -79,17 +79,25 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) return 0; } -void tdbPageZero(SPage *pPage) { +void tdbPageZero(SPage *pPage, u8 szAmHdr) { + pPage->pAmHdr = pPage->pData; + pPage->pPageHdr = pPage->pAmHdr + szAmHdr; TDB_PAGE_NCELLS_SET(pPage, 0); TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); TDB_PAGE_FCELL_SET(pPage, 0); - TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_MAX_FREE_BLOCK(pPage)); - tdbPageInit(pPage); + TDB_PAGE_NFREE_SET(pPage, TDB_PAGE_MAX_FREE_BLOCK(pPage, szAmHdr)); + pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); + pPage->pFreeStart = pPage->pCellIdx; + pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); + pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); + pPage->nOverflow = 0; + + ASSERT((u8 *)pPage->pPageFtr == pPage->pFreeEnd); } -void tdbPageInit(SPage *pPage) { +void tdbPageInit(SPage *pPage, u8 szAmHdr) { pPage->pAmHdr = pPage->pData; - pPage->pPageHdr = pPage->pAmHdr + pPage->szAmHdr; + pPage->pPageHdr = pPage->pAmHdr + szAmHdr; pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); @@ -107,7 +115,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { int lidx; // local idx SCell *pNewCell; - ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage)); + ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pAmHdr)); nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); From 50aa291ff60089a8270f3b85f3e4bd84d0a2aae9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 03:29:39 +0000 Subject: [PATCH 011/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0c83dd609c..59c132ecb2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -423,9 +423,6 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { pBt = ((SBtreeZeroPageArg *)arg)->pBt; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); - // Init the page header - TDB_PAGE_FLAGS_SET(pPage, flags); - // Set szAmHdr if (isLeaf) { szAmHdr = 0; } else { @@ -433,6 +430,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { } pPage->xCellSize = NULL; // TODO + TDB_PAGE_FLAGS_SET(pPage, flags); tdbPageZero(pPage, szAmHdr); if (isLeaf) { From 03eaa648d68c2b9f11a54f780d88e13630809d99 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 03:42:18 +0000 Subject: [PATCH 012/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 59c132ecb2..52f0c7d466 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -385,6 +385,9 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { pBt = (SBTree *)arg; + ASSERT(0); + + // TODO: here has problem flags = TDB_PAGE_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { @@ -430,8 +433,8 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { } pPage->xCellSize = NULL; // TODO - TDB_PAGE_FLAGS_SET(pPage, flags); tdbPageZero(pPage, szAmHdr); + TDB_PAGE_FLAGS_SET(pPage, flags); if (isLeaf) { pPage->kLen = pBt->keyLen; From 4e8f8445de7f719dbef9a75321253df3d4b1fff5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 05:47:06 +0000 Subject: [PATCH 013/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 41 +++++++++++++++++------------ source/libs/tdb/src/inc/tdbPage.h | 4 +++ source/libs/tdb/src/page/tdbPage.c | 10 ++++--- source/libs/tdb/src/page/tdbPageL.c | 10 ++++--- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 52f0c7d466..4b2394f861 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -39,9 +39,16 @@ struct SBTree { u8 *pTmp; }; +#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; + typedef struct __attribute__((__packed__)) { - SPgno rChild; -} SBtPageHdr; + TDB_BTREE_PAGE_COMMON_HDR +} SLeafHdr; + +typedef struct __attribute__((__packed__)) { + TDB_BTREE_PAGE_COMMON_HDR; + SPgno pgno; // right-most child +} SIntHdr; typedef struct { u16 flags; @@ -220,7 +227,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p if (TDB_PAGE_TOTAL_CELLS(pCur->pPage) == 0) { // Current page is empty - ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); + // ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pCur->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); return 0; } @@ -265,7 +272,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p } // Move downward or break - u16 flags = TDB_PAGE_FLAGS(pPage); + u16 flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (leaf) { pCur->idx = midx; @@ -279,7 +286,7 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p if (midx == nCells - 1) { /* Move to right-most child */ pCur->idx = midx + 1; - tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); + // tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); } else { // TODO: reset cd as uninitialized pCur->idx = midx + 1; @@ -388,12 +395,12 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { ASSERT(0); // TODO: here has problem - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { - szAmHdr = 0; + szAmHdr = sizeof(SLeafHdr); } else { - szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SIntHdr); } pPage->xCellSize = NULL; // TODO @@ -427,14 +434,14 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { - szAmHdr = 0; + szAmHdr = sizeof(SLeafHdr); } else { - szAmHdr = sizeof(SBtPageHdr); + szAmHdr = sizeof(SIntHdr); } pPage->xCellSize = NULL; // TODO tdbPageZero(pPage, szAmHdr); - TDB_PAGE_FLAGS_SET(pPage, flags); + // TDB_PAGE_FLAGS_SET(pPage, flags); if (isLeaf) { pPage->kLen = pBt->keyLen; @@ -522,7 +529,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } - ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; + // TODO: ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; *ppChild = pChild; return 0; @@ -566,7 +573,7 @@ static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { i = pBlh->nOld - 1; if (idxStart + i == nCells) { - pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; + // pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; } else { pCell = tdbPageGetCell(pParent, idxStart + i); // TODO: no need to decode the payload part, and even the kLen, vLen part @@ -676,7 +683,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; SBtreeBalanceHelper blh; - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); + // ASSERT(!TDB_BTREE_PGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); blh.pBt = pBt; blh.pParent = pParent; @@ -748,7 +755,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { for (;;) { iPage = pCur->iPage; pPage = pCur->pPage; - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); root = TDB_BTREE_PAGE_IS_ROOT(flags); @@ -842,7 +849,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo nPayload = 0; nHeader = 0; - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // 1. Encode Header part @@ -911,7 +918,7 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD int ret; nHeader = 0; - flags = TDB_PAGE_FLAGS(pPage); + flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Clear the state of decoder diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 58ae1f3963..155e7b2e37 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -27,9 +27,11 @@ typedef struct { int szOffset; int szPageHdr; int szFreeCell; +#if 0 // flags u16 (*getFlags)(SPage *); void (*setFlags)(SPage *, u16); +#endif // cell number int (*getCellNum)(SPage *); void (*setCellNum)(SPage *, int); @@ -79,9 +81,11 @@ struct SPage { TDB_PCACHE_PAGE }; +#if 0 /* For page */ #define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) #define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) +#endif // For page lock #define P_LOCK_SUCC 0 diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 11c9a9247e..61793b6025 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -399,9 +399,11 @@ typedef struct __attribute__((__packed__)) { u16 nxOffset; } SFreeCell; +#if 0 // flags static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } +#endif // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } @@ -456,11 +458,13 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageMethods = { - 2, // szOffset - sizeof(SPageHdr), // szPageHdr - sizeof(SFreeCell), // szFreeCell + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell +#if 0 getPageFlags, // getPageFlags setPageFlags, // setFlagsp +#endif getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index e3c1bb8d37..f4bcc62494 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -28,9 +28,11 @@ typedef struct __attribute__((__packed__)) { u8 nxOffset[3]; } SFreeCellL; +#if 0 // flags static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdrL *)(pPage->pPageHdr))[0].flags; } static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdrL *)(pPage->pPageHdr))[0].flags = flags; } +#endif // cellNum static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } @@ -80,11 +82,13 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell +#if 0 getPageFlags, // getPageFlags setPageFlags, // setFlagsp +#endif getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody From df306127a644ebec5d8dc6ff37264ed8c4d9fed3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 05:56:33 +0000 Subject: [PATCH 014/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 3 ++- source/libs/tdb/src/inc/tdbPage.h | 11 ----------- source/libs/tdb/src/page/tdbPage.c | 16 +++------------- source/libs/tdb/src/page/tdbPageL.c | 16 +++------------- 4 files changed, 8 insertions(+), 38 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4b2394f861..68319da3b2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -39,7 +39,8 @@ struct SBTree { u8 *pTmp; }; -#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; +#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; +#define TDB_BTREE_PAGE_FLAGS(PAGE) (PAGE)->pAmHdr[0] typedef struct __attribute__((__packed__)) { TDB_BTREE_PAGE_COMMON_HDR diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 155e7b2e37..dc3b803708 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -27,11 +27,6 @@ typedef struct { int szOffset; int szPageHdr; int szFreeCell; -#if 0 - // flags - u16 (*getFlags)(SPage *); - void (*setFlags)(SPage *, u16); -#endif // cell number int (*getCellNum)(SPage *); void (*setCellNum)(SPage *, int); @@ -81,12 +76,6 @@ struct SPage { TDB_PCACHE_PAGE }; -#if 0 -/* For page */ -#define TDB_PAGE_FLAGS(pPage) (*(pPage)->pPageMethods->getFlags)(pPage) -#define TDB_PAGE_FLAGS_SET(pPage, FLAGS) (*(pPage)->pPageMethods->setFlags)(pPage, FLAGS) -#endif - // For page lock #define P_LOCK_SUCC 0 #define P_LOCK_BUSY 1 diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 61793b6025..b3a3bdbdcc 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -399,12 +399,6 @@ typedef struct __attribute__((__packed__)) { u16 nxOffset; } SFreeCell; -#if 0 -// flags -static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].flags; } -static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdr *)(pPage->pPageHdr))[0].flags = flags; } -#endif - // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } static inline void setPageCellNum(SPage *pPage, int cellNum) { @@ -458,13 +452,9 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageMethods = { - 2, // szOffset - sizeof(SPageHdr), // szPageHdr - sizeof(SFreeCell), // szFreeCell -#if 0 - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp -#endif + 2, // szOffset + sizeof(SPageHdr), // szPageHdr + sizeof(SFreeCell), // szFreeCell getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index f4bcc62494..cfd8c919df 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -28,12 +28,6 @@ typedef struct __attribute__((__packed__)) { u8 nxOffset[3]; } SFreeCellL; -#if 0 -// flags -static inline u16 getPageFlags(SPage *pPage) { return ((SPageHdrL *)(pPage->pPageHdr))[0].flags; } -static inline void setPageFlags(SPage *pPage, u16 flags) { ((SPageHdrL *)(pPage->pPageHdr))[0].flags = flags; } -#endif - // cellNum static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } static inline void setPageCellNum(SPage *pPage, int cellNum) { @@ -82,13 +76,9 @@ static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { } SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell -#if 0 - getPageFlags, // getPageFlags - setPageFlags, // setFlagsp -#endif + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell getPageCellNum, // getCellNum setPageCellNum, // setCellNum getPageCellBody, // getCellBody From 2b7da2a79b1972c35853419cb564e9e429f40609 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:05:32 +0000 Subject: [PATCH 015/121] more TDB --- source/libs/tdb/src/btree/tdbBtreeBalance.c | 14 +++++++++++ source/libs/tdb/src/btree/tdbBtreeCommon.c | 14 +++++++++++ source/libs/tdb/src/btree/tdbBtreeInt.h | 27 +++++++++++++++++++++ source/libs/tdb/src/btree/tdbBtreeOpen.c | 14 +++++++++++ 4 files changed, 69 insertions(+) create mode 100644 source/libs/tdb/src/btree/tdbBtreeBalance.c create mode 100644 source/libs/tdb/src/btree/tdbBtreeCommon.c create mode 100644 source/libs/tdb/src/btree/tdbBtreeInt.h create mode 100644 source/libs/tdb/src/btree/tdbBtreeOpen.c diff --git a/source/libs/tdb/src/btree/tdbBtreeBalance.c b/source/libs/tdb/src/btree/tdbBtreeBalance.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeBalance.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeCommon.c b/source/libs/tdb/src/btree/tdbBtreeCommon.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeCommon.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeInt.h b/source/libs/tdb/src/btree/tdbBtreeInt.h new file mode 100644 index 0000000000..b8a935a614 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeInt.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TDB_BTREE_INT_H_ +#define _TDB_BTREE_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TDB_BTREE_INT_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeOpen.c b/source/libs/tdb/src/btree/tdbBtreeOpen.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeOpen.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file From d9c4bcab6ccb19c9494587472c7ee6598e7cdd94 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:14:47 +0000 Subject: [PATCH 016/121] more TDB --- source/libs/tdb/CMakeLists.txt | 1 - source/libs/tdb/src/btree/tdbBtreeDelete.c | 14 ++++++++++++++ source/libs/tdb/src/btree/tdbBtreeInsert.c | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 source/libs/tdb/src/btree/tdbBtreeDelete.c create mode 100644 source/libs/tdb/src/btree/tdbBtreeInsert.c diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 978649499a..a9b56d42b8 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -8,7 +8,6 @@ target_sources(tdb "src/db/tdbBtree.c" "src/db/tdbDb.c" "src/db/tdbEnv.c" - # "src/db/tdbPage.c" "src/page/tdbPage.c" "src/page/tdbPageL.c" ) diff --git a/source/libs/tdb/src/btree/tdbBtreeDelete.c b/source/libs/tdb/src/btree/tdbBtreeDelete.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeDelete.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/btree/tdbBtreeInsert.c b/source/libs/tdb/src/btree/tdbBtreeInsert.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/btree/tdbBtreeInsert.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file From a84155b951b24d87b204a2a4009c9c19cbe6f2cc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:23:32 +0000 Subject: [PATCH 017/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 21 ++++++++++++++------- source/libs/tdb/src/inc/tdbPage.h | 2 +- source/libs/tdb/src/page/tdbPage.c | 4 ++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 68319da3b2..2a2d630cd8 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -52,9 +52,9 @@ typedef struct __attribute__((__packed__)) { } SIntHdr; typedef struct { - u16 flags; + u8 flags; SBTree *pBt; -} SBtreeZeroPageArg; +} SBtreeInitPageArg; typedef struct { int kLen; @@ -74,6 +74,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo int *szCell); static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); static int tdbBtreeBalance(SBtCursor *pCur); +static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; @@ -371,7 +372,7 @@ static int tdbBtreeOpenImpl(SBTree *pBt) { } // Try to create a new database - SBtreeZeroPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; + SBtreeInitPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt}; ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg); if (ret < 0) { return -1; @@ -430,8 +431,8 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u8 isLeaf; u8 szAmHdr; - flags = ((SBtreeZeroPageArg *)arg)->flags; - pBt = ((SBtreeZeroPageArg *)arg)->pBt; + flags = ((SBtreeInitPageArg *)arg)->flags; + pBt = ((SBtreeInitPageArg *)arg)->pBt; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (isLeaf) { @@ -498,7 +499,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPage *pChild; SPgno pgnoChild; int ret; - SBtreeZeroPageArg zArg; + SBtreeInitPageArg zArg; pPager = pRoot->pPager; @@ -959,4 +960,10 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD return 0; } -#endif \ No newline at end of file +#endif + +static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { + // TODO + ASSERT(0); + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index dc3b803708..0960d88c33 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -71,7 +71,7 @@ struct SPage { int vLen; // value length of the page, -1 for unknown int maxLocal; int minLocal; - int (*xCellSize)(SCell *pCell); + int (*xCellSize)(const SPage *, SCell *); // Fields used by SPCache TDB_PCACHE_PAGE }; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index b3a3bdbdcc..7088904285 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -193,7 +193,7 @@ int tdbPageDropCell(SPage *pPage, int idx) { lidx = idx - iOvfl; pCell = TDB_PAGE_CELL_AT(pPage, lidx); - szCell = (*pPage->xCellSize)(pCell); + szCell = (*pPage->xCellSize)(pPage, pCell); tdbPageFree(pPage, lidx, pCell, szCell); TDB_PAGE_NCELLS_SET(pPage, nCells - 1); @@ -366,7 +366,7 @@ static int tdbPageDefragment(SPage *pPage) { ASSERT(pCell != NULL); - szCell = (*pPage->xCellSize)(pCell); + szCell = (*pPage->xCellSize)(pPage, pCell); ASSERT(pCell + szCell <= pNextCell); if (pCell + szCell < pNextCell) { From 0a6be10ff1d1f5b35c0ffceb095485347f774194 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:34:34 +0000 Subject: [PATCH 018/121] more tdb --- source/libs/tdb/src/db/tdbBtree.c | 44 ++++++++++++++---------------- source/libs/tdb/src/inc/tdbPage.h | 4 +-- source/libs/tdb/src/page/tdbPage.c | 6 ++-- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 2a2d630cd8..06f96e5498 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -39,8 +39,10 @@ struct SBTree { u8 *pTmp; }; -#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; -#define TDB_BTREE_PAGE_FLAGS(PAGE) (PAGE)->pAmHdr[0] +#define TDB_BTREE_PAGE_COMMON_HDR u8 flags; + +#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pAmHdr[0] +#define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pAmHdr[0] = (flags)) typedef struct __attribute__((__packed__)) { TDB_BTREE_PAGE_COMMON_HDR @@ -274,8 +276,8 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p } // Move downward or break - u16 flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); - u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + u8 flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); if (leaf) { pCur->idx = midx; *pCRst = c; @@ -388,25 +390,23 @@ static int tdbBtreeOpenImpl(SBTree *pBt) { static int tdbBtreeInitPage(SPage *pPage, void *arg) { SBTree *pBt; - u16 flags; + u8 flags; u8 isLeaf; u8 szAmHdr; - pBt = (SBTree *)arg; - - ASSERT(0); - - // TODO: here has problem - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + pBt = ((SBtreeInitPageArg *)arg)->pBt; + flags = ((SBtreeInitPageArg *)arg)->flags; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + + ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); + if (isLeaf) { szAmHdr = sizeof(SLeafHdr); } else { szAmHdr = sizeof(SIntHdr); } - pPage->xCellSize = NULL; // TODO - tdbPageInit(pPage, szAmHdr); + tdbPageInit(pPage, szAmHdr, tdbBtreeCellSize); TDB_BTREE_ASSERT_FLAG(flags); @@ -426,7 +426,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { } static int tdbBtreeZeroPage(SPage *pPage, void *arg) { - u16 flags; + u8 flags; SBTree *pBt; u8 isLeaf; u8 szAmHdr; @@ -440,10 +440,8 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { } else { szAmHdr = sizeof(SIntHdr); } - pPage->xCellSize = NULL; // TODO - tdbPageZero(pPage, szAmHdr); - // TDB_PAGE_FLAGS_SET(pPage, flags); + tdbPageZero(pPage, szAmHdr, tdbBtreeCellSize); if (isLeaf) { pPage->kLen = pBt->keyLen; @@ -749,7 +747,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { SPage *pParent; SPage *pPage; int ret; - u16 flags; + u8 flags; u8 leaf; u8 root; @@ -757,7 +755,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { for (;;) { iPage = pCur->iPage; pPage = pCur->pPage; - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); root = TDB_BTREE_PAGE_IS_ROOT(flags); @@ -840,7 +838,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, u8 *pPayload, const void *pKey, i static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell) { - u16 flags; + u8 flags; u8 leaf; int nHeader; int nPayload; @@ -851,7 +849,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo nPayload = 0; nHeader = 0; - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // 1. Encode Header part @@ -914,13 +912,13 @@ static int tdbBtreeDecodePayload(SPage *pPage, const u8 *pPayload, SCellDecoder } static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder) { - u16 flags; + u8 flags; u8 leaf; int nHeader; int ret; nHeader = 0; - flags = 0; // TODO: TDB_PAGE_FLAGS(pPage); + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Clear the state of decoder diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 0960d88c33..b9007b99e0 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -103,8 +103,8 @@ struct SPage { int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); -void tdbPageZero(SPage *pPage, u8 szAmHdr); -void tdbPageInit(SPage *pPage, u8 szAmHdr); +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 7088904285..97ed9bb703 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -79,7 +79,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) return 0; } -void tdbPageZero(SPage *pPage, u8 szAmHdr) { +void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { pPage->pAmHdr = pPage->pData; pPage->pPageHdr = pPage->pAmHdr + szAmHdr; TDB_PAGE_NCELLS_SET(pPage, 0); @@ -91,11 +91,12 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr) { pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); pPage->nOverflow = 0; + pPage->xCellSize = xCellSize; ASSERT((u8 *)pPage->pPageFtr == pPage->pFreeEnd); } -void tdbPageInit(SPage *pPage, u8 szAmHdr) { +void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { pPage->pAmHdr = pPage->pData; pPage->pPageHdr = pPage->pAmHdr + szAmHdr; pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); @@ -103,6 +104,7 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr) { pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); pPage->pPageFtr = (SPageFtr *)(pPage->pData + pPage->pageSize - sizeof(SPageFtr)); pPage->nOverflow = 0; + pPage->xCellSize = xCellSize; ASSERT(pPage->pFreeEnd >= pPage->pFreeStart); ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); From ad0e3a007bba08799cf7b193db4b2f97323edf27 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:45:08 +0000 Subject: [PATCH 019/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 06f96e5498..8b4bef50a5 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -392,7 +392,6 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { SBTree *pBt; u8 flags; u8 isLeaf; - u8 szAmHdr; pBt = ((SBtreeInitPageArg *)arg)->pBt; flags = ((SBtreeInitPageArg *)arg)->flags; @@ -400,13 +399,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); - if (isLeaf) { - szAmHdr = sizeof(SLeafHdr); - } else { - szAmHdr = sizeof(SIntHdr); - } - - tdbPageInit(pPage, szAmHdr, tdbBtreeCellSize); + tdbPageInit(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); TDB_BTREE_ASSERT_FLAG(flags); @@ -429,26 +422,26 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u8 flags; SBTree *pBt; u8 isLeaf; - u8 szAmHdr; flags = ((SBtreeInitPageArg *)arg)->flags; pBt = ((SBtreeInitPageArg *)arg)->pBt; isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); - if (isLeaf) { - szAmHdr = sizeof(SLeafHdr); - } else { - szAmHdr = sizeof(SIntHdr); - } - - tdbPageZero(pPage, szAmHdr, tdbBtreeCellSize); + tdbPageZero(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); if (isLeaf) { + SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pPageHdr); + pLeafHdr->flags = flags; + pPage->kLen = pBt->keyLen; pPage->vLen = pBt->valLen; pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; } else { + SIntHdr *pIntHdr = (SIntHdr *)(pPage->pPageHdr); + pIntHdr->flags = flags; + pIntHdr->pgno = 0; + pPage->kLen = pBt->keyLen; pPage->vLen = sizeof(SPgno); pPage->maxLocal = pBt->maxLocal; From 9c8102027fc921e0c9d35f50b16e4579a5df6569 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:46:37 +0000 Subject: [PATCH 020/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8b4bef50a5..f72ce41355 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -430,7 +430,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { tdbPageZero(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); if (isLeaf) { - SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pPageHdr); + SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pAmHdr); pLeafHdr->flags = flags; pPage->kLen = pBt->keyLen; @@ -438,7 +438,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; } else { - SIntHdr *pIntHdr = (SIntHdr *)(pPage->pPageHdr); + SIntHdr *pIntHdr = (SIntHdr *)(pPage->pAmHdr); pIntHdr->flags = flags; pIntHdr->pgno = 0; From 0cd8cca66dd1aa03b16079de53514c4545ec9d31 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 06:55:17 +0000 Subject: [PATCH 021/121] more TDB --- source/libs/tdb/src/page/tdbPage.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 97ed9bb703..4a067bb978 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -141,7 +141,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { u8 *src = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * lidx; u8 *dest = src + TDB_PAGE_OFFSET_SIZE(pPage); memmove(dest, src, pPage->pFreeStart - dest); - TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, TDB_PAGE_OFFSET_SIZE(pPage) * lidx); + TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, pNewCell - pPage->pData); TDB_PAGE_NCELLS_SET(pPage, nCells + 1); ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); From 8becb5421110425ce073ce2e8fb70176087862b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 07:30:34 +0000 Subject: [PATCH 022/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 7 ------- source/libs/tdb/src/inc/tdbPage.h | 1 + source/libs/tdb/src/page/tdbPage.c | 10 ++++++++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f72ce41355..4c7cddf311 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -752,13 +752,6 @@ static int tdbBtreeBalance(SBtCursor *pCur) { leaf = TDB_BTREE_PAGE_IS_LEAF(flags); root = TDB_BTREE_PAGE_IS_ROOT(flags); - // TODO: Get the page free space if not get yet - // if (pPage->nFree < 0) { - // if (tdbBtreeComputeFreeSpace(pPage) < 0) { - // return -1; - // } - // } - // when the page is not overflow and not too empty, the balance work // is finished. Just break out the balance loop. if (pPage->nOverflow == 0 /* TODO: && pPage->nFree <= */) { diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index b9007b99e0..00322067df 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -107,6 +107,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); int tdbPageDropCell(SPage *pPage, int idx); +void tdbPageCopy(SPage *pFromPage, SPage *pToPage); static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { SCell *pCell; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 4a067bb978..c3fec25d81 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -207,6 +207,16 @@ int tdbPageDropCell(SPage *pPage, int idx) { return 0; } +void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { + pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); + pToPage->pFreeEnd = (u8 *)(pToPage->pPageFtr) - ((u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); + + ASSERT(pToPage->pFreeEnd >= pToPage->pFreeStart); + + memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); + memcpy(pToPage->pFreeEnd, pFromPage->pFreeEnd, (u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); +} + static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { SCell *pFreeCell; u8 *pOffset; From b8954ae1c5036a91293b385c80fc69501a8e5e82 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 07:55:56 +0000 Subject: [PATCH 023/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 28 +--------------------------- source/libs/tdb/src/page/tdbPage.c | 12 ++++++++++++ 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4c7cddf311..9e6ea91a4c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -462,29 +462,6 @@ typedef struct { SPage *pNewPages[5]; } SBtreeBalanceHelper; -static int tdbBtreeCopyPageContent(SPage *pFrom, SPage *pTo) { -#if 0 - int nCells = TDB_PAGE_TOTAL_CELLS(pFrom); - int cCells = TDB_PAGE_CCELLS(pFrom); - int fCell = TDB_PAGE_FCELL(pFrom); - int nFree = TDB_PAGE_NFREE(pFrom); - - pTo->pFreeStart = pTo->pCellIdx + nCells * pFrom->pPageMethods->szOffset; - memcpy(pTo->pCellIdx, pFrom->pCellIdx, nCells * pFrom->pPageMethods->szOffset); - pTo->pFreeEnd = (u8 *)pTo->pPageFtr - (u8 *)(pFrom->pPageFtr) + pFrom->pFreeEnd; - memcpy(pTo->pFreeEnd, pFrom->pFreeEnd, (u8 *)pFrom->pPageFtr - pFrom->pFreeEnd); - - TDB_PAGE_NCELLS_SET(pTo, nCells); - TDB_PAGE_CCELLS_SET(pTo, cCells); - TDB_PAGE_FCELL_SET(pTo, fCell); - TDB_PAGE_NFREE_SET(pTo, nFree); - - // TODO: update other fields -#endif - - return 0; -} - static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPager *pPager; SPage *pChild; @@ -503,10 +480,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { } // Copy the root page content to the child page - ret = tdbBtreeCopyPageContent(pRoot, pChild); - if (ret < 0) { - return -1; - } + tdbPageCopy(pRoot, pChild); pChild->nOverflow = pRoot->nOverflow; for (int i = 0; i < pChild->nOverflow; i++) { diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index c3fec25d81..803bf168cf 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -208,6 +208,8 @@ int tdbPageDropCell(SPage *pPage, int idx) { } void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { + int delta, nFree; + pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); pToPage->pFreeEnd = (u8 *)(pToPage->pPageFtr) - ((u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); @@ -215,6 +217,16 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); memcpy(pToPage->pFreeEnd, pFromPage->pFreeEnd, (u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); + + ASSERT(TDB_PAGE_CCELLS(pToPage) == pToPage->pFreeEnd - pToPage->pData); + + delta = (pToPage->pPageHdr - pToPage->pAmHdr) - (pFromPage->pPageHdr - pFromPage->pAmHdr); + if (delta != 0) { + nFree = TDB_PAGE_NFREE(pFromPage); + TDB_PAGE_NFREE_SET(pToPage, nFree - delta); + } + + // TODO: do we need to copy the overflow part ??? } static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { From 2e6f7f15d5934ab67f6ee2cb1f2e7bb435bccb9e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 07:59:37 +0000 Subject: [PATCH 024/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 6 ------ source/libs/tdb/src/page/tdbPage.c | 7 ++++++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 9e6ea91a4c..5f56b2f1e7 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -482,12 +482,6 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { // Copy the root page content to the child page tdbPageCopy(pRoot, pChild); - pChild->nOverflow = pRoot->nOverflow; - for (int i = 0; i < pChild->nOverflow; i++) { - pChild->apOvfl[i] = pRoot->apOvfl[i]; - pChild->aiOvfl[i] = pRoot->aiOvfl[i]; - } - // Reinitialize the root page zArg.flags = TDB_BTREE_ROOT; zArg.pBt = pBt; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 803bf168cf..2f6d4101a5 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -226,7 +226,12 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { TDB_PAGE_NFREE_SET(pToPage, nFree - delta); } - // TODO: do we need to copy the overflow part ??? + // Copy the overflow cells + for (int iOvfl = 0; iOvfl < pFromPage->nOverflow; iOvfl++) { + pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl]; + pToPage->apOvfl[iOvfl] = pFromPage->apOvfl[iOvfl]; + } + pToPage->nOverflow = pFromPage->nOverflow; } static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { From 5ca3e29cf19ed454ec9a0a4e7c00c3d474d01c99 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 08:17:58 +0000 Subject: [PATCH 025/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 6 ++++-- source/libs/tdb/src/page/tdbPage.c | 1 - source/libs/tdb/src/page/tdbPageL.c | 9 ++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 5f56b2f1e7..4e4b2275f8 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -467,6 +467,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPage *pChild; SPgno pgnoChild; int ret; + SIntHdr *pIntHdr; SBtreeInitPageArg zArg; pPager = pRoot->pPager; @@ -490,7 +491,8 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } - // TODO: ((SBtPageHdr *)pRoot->pAmHdr)[0].rChild = pgnoChild; + pIntHdr = (SIntHdr *)(pRoot->pAmHdr); + pIntHdr->pgno = pgnoChild; *ppChild = pChild; return 0; @@ -731,7 +733,7 @@ static int tdbBtreeBalance(SBtCursor *pCur) { // ignore the case of empty if (pPage->nOverflow == 0) break; - ret = tdbBtreeBalanceDeeper(pCur->pBt, pCur->pPage, &(pCur->pgStack[1])); + ret = tdbBtreeBalanceDeeper(pCur->pBt, pPage, &(pCur->pgStack[1])); if (ret < 0) { return -1; } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 2f6d4101a5..bc00da9381 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -416,7 +416,6 @@ static int tdbPageDefragment(SPage *pPage) { /* ---------------------------------------------------------------------------------------------------------- */ typedef struct __attribute__((__packed__)) { - u16 flags; u16 cellNum; u16 cellBody; u16 cellFree; diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c index cfd8c919df..c5d4a6047f 100644 --- a/source/libs/tdb/src/page/tdbPageL.c +++ b/source/libs/tdb/src/page/tdbPageL.c @@ -16,11 +16,10 @@ #include "tdbInt.h" typedef struct __attribute__((__packed__)) { - u16 flags; - u8 cellNum[3]; - u8 cellBody[3]; - u8 cellFree[3]; - u8 nFree[3]; + u8 cellNum[3]; + u8 cellBody[3]; + u8 cellFree[3]; + u8 nFree[3]; } SPageHdrL; typedef struct __attribute__((__packed__)) { From db870af8b7afb105417d4d3d7e6acaaf1bc90235 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 09:33:10 +0000 Subject: [PATCH 026/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 65 +++++++++++++++++++++++++----- source/libs/tdb/src/inc/tdbPage.h | 3 +- source/libs/tdb/src/page/tdbPage.c | 10 ++--- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4e4b2275f8..b801002822 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -41,8 +41,8 @@ struct SBTree { #define TDB_BTREE_PAGE_COMMON_HDR u8 flags; -#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pAmHdr[0] -#define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pAmHdr[0] = (flags)) +#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pData[0] +#define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pData[0] = (flags)) typedef struct __attribute__((__packed__)) { TDB_BTREE_PAGE_COMMON_HDR @@ -393,8 +393,8 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { u8 flags; u8 isLeaf; - pBt = ((SBtreeInitPageArg *)arg)->pBt; - flags = ((SBtreeInitPageArg *)arg)->flags; + pBt = (SBTree *)arg; + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); @@ -430,7 +430,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { tdbPageZero(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); if (isLeaf) { - SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pAmHdr); + SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pData); pLeafHdr->flags = flags; pPage->kLen = pBt->keyLen; @@ -438,7 +438,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) { pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; } else { - SIntHdr *pIntHdr = (SIntHdr *)(pPage->pAmHdr); + SIntHdr *pIntHdr = (SIntHdr *)(pPage->pData); pIntHdr->flags = flags; pIntHdr->pgno = 0; @@ -491,7 +491,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } - pIntHdr = (SIntHdr *)(pRoot->pAmHdr); + pIntHdr = (SIntHdr *)(pRoot->pData); pIntHdr->pgno = pgnoChild; *ppChild = pChild; @@ -643,15 +643,59 @@ static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { } static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { - int ret; + int ret; +#if 0 SBtreeBalanceHelper blh; - // ASSERT(!TDB_BTREE_PGE_IS_LEAF(TDB_PAGE_FLAGS(pParent))); - blh.pBt = pBt; blh.pParent = pParent; blh.idx = idx; +#endif + int nOlds; + SPage *pOlds[3]; + + { + // Find 3 child pages at most to do balance + int nCells = TDB_PAGE_TOTAL_CELLS(pParent); + int sIdx; + if (nCells <= 2) { + sIdx = 0; + nOlds = nCells + 1; + } else { + // has more than three child pages + if (idx == 0) { + sIdx = 0; + } else if (idx == nCells) { + sIdx = idx - 2; + } else { + sIdx = idx - 1; + } + nOlds = 3; + } + for (int i = 0; i < nOlds; i++, sIdx++) { + ASSERT(sIdx <= nCells); + + SPgno pgno; + if (sIdx == nCells) { + ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pParent))); + pgno = ((SIntHdr *)(pParent->pData))->pgno; + } else { + SCell *pCell; + + pCell = tdbPageGetCell(pParent, sIdx); + pgno = *(SPgno *)pCell; + } + + ret = tdbPagerFetchPage(pBt->pPager, pgno, pOlds + i, tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + } + +#if 0 // Step 1: find two sibling pages and get engough info about the old pages ret = tdbBtreeBalanceStep1(&blh); if (ret < 0) { @@ -693,6 +737,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { ASSERT(0); return -1; } +#endif { // TODO: Reset states diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 00322067df..e513a63fe4 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -58,7 +58,6 @@ struct SPage { u8 *pData; SPageMethods *pPageMethods; // Fields below used by pager and am - u8 *pAmHdr; u8 *pPageHdr; u8 *pCellIdx; u8 *pFreeStart; @@ -114,7 +113,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { int iOvfl; int lidx; - ASSERT(idx >= 0 && idx < pPage->nOverflow + pPage->pPageMethods->getCellNum(pPage)); + ASSERT(idx >= 0 && idx < TDB_PAGE_TOTAL_CELLS(pPage)); iOvfl = 0; for (; iOvfl < pPage->nOverflow; iOvfl++) { diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index bc00da9381..0dc7853940 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -80,8 +80,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) } void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { - pPage->pAmHdr = pPage->pData; - pPage->pPageHdr = pPage->pAmHdr + szAmHdr; + pPage->pPageHdr = pPage->pData + szAmHdr; TDB_PAGE_NCELLS_SET(pPage, 0); TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); TDB_PAGE_FCELL_SET(pPage, 0); @@ -97,8 +96,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell } void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)) { - pPage->pAmHdr = pPage->pData; - pPage->pPageHdr = pPage->pAmHdr + szAmHdr; + pPage->pPageHdr = pPage->pData + szAmHdr; pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); @@ -117,7 +115,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { int lidx; // local idx SCell *pNewCell; - ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pAmHdr)); + ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); @@ -220,7 +218,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { ASSERT(TDB_PAGE_CCELLS(pToPage) == pToPage->pFreeEnd - pToPage->pData); - delta = (pToPage->pPageHdr - pToPage->pAmHdr) - (pFromPage->pPageHdr - pFromPage->pAmHdr); + delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); if (delta != 0) { nFree = TDB_PAGE_NFREE(pFromPage); TDB_PAGE_NFREE_SET(pToPage, nFree - delta); From b3b5298510d28f1a75ec917dcf8cec1978538697 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 09:47:46 +0000 Subject: [PATCH 027/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 34 ++++++++++++++++--------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index b801002822..52ea45578b 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -695,6 +695,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + // >>>>>>>>>>>>>>>>>>> + #if 0 // Step 1: find two sibling pages and get engough info about the old pages ret = tdbBtreeBalanceStep1(&blh); @@ -854,6 +856,14 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // 1. Encode Header part + /* Encode SPgno if interior page */ + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; + nHeader = nHeader + sizeof(SPgno); + } + /* Encode kLen if need */ if (pPage->kLen == TDB_VARIANT_LEN) { nHeader += tdbPutVarInt(pCell + nHeader, kLen); @@ -864,14 +874,6 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo nHeader += tdbPutVarInt(pCell + nHeader, vLen); } - /* Encode SPgno if interior page */ - if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); - - ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; - nHeader = nHeader + sizeof(SPgno); - } - // 2. Encode payload part if (leaf) { ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, pVal, vLen, &nPayload); @@ -930,6 +932,14 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD pDecoder->pgno = 0; // 1. Decode header part + if (!leaf) { + ASSERT(pPage->vLen == sizeof(SPgno)); + + pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; + pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); + nHeader = nHeader + sizeof(SPgno); + } + if (pPage->kLen == TDB_VARIANT_LEN) { nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->kLen)); } else { @@ -942,14 +952,6 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD pDecoder->vLen = pPage->vLen; } - if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); - - pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; - pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); - nHeader = nHeader + sizeof(SPgno); - } - // 2. Decode payload part ret = tdbBtreeDecodePayload(pPage, pCell + nHeader, pDecoder); if (ret < 0) { From 1075c230ed0615f80449000afe09feafd9a63723 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:18:04 +0000 Subject: [PATCH 028/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 37 ++++++++++++++++++++++--------- source/libs/tdb/src/inc/tdbPage.h | 3 ++- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 52ea45578b..93fc75269c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -644,19 +644,11 @@ static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; -#if 0 - SBtreeBalanceHelper blh; - - blh.pBt = pBt; - blh.pParent = pParent; - blh.idx = idx; -#endif int nOlds; SPage *pOlds[3]; - { - // Find 3 child pages at most to do balance + { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); int sIdx; if (nCells <= 2) { @@ -695,7 +687,32 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - // >>>>>>>>>>>>>>>>>>> + int nNews = 0; + int cntNews[5] = {0}; // TODO: maybe 5 is not enough + int szNews[5] = {0}; + int maxPageCapacity; // TODO + + { // Get how many new pages are needed and the new distribution + + // loop to find number of pages needed + for (int i = 0; i < nOlds; i++) { + SPage *pPage = pOlds[i]; + SCell *pCell; + int cellBytes; + for (int cIdx = 0; cIdx < TDB_PAGE_TOTAL_CELLS(pPage); cIdx++) { + pCell = tdbPageGetCell(pPage, cIdx); + cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); + + if (szNews[nNews] + cellBytes > maxPageCapacity) { + nNews++; + } + cntNews[nNews]++; + szNews[nNews] = szNews[nNews] + cellBytes; + } + } + + // Loop to make the distribution even + } #if 0 // Step 1: find two sibling pages and get engough info about the old pages diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index e513a63fe4..479c7603e1 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -98,7 +98,8 @@ struct SPage { }) // APIs -#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); From b5b522d66643663ceeabbf5b8a8548153431073a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:23:34 +0000 Subject: [PATCH 029/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 4 ++-- source/libs/tdb/src/inc/tdbPage.h | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 93fc75269c..aa3e4c6b90 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -690,7 +690,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nNews = 0; int cntNews[5] = {0}; // TODO: maybe 5 is not enough int szNews[5] = {0}; - int maxPageCapacity; // TODO { // Get how many new pages are needed and the new distribution @@ -699,11 +698,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SPage *pPage = pOlds[i]; SCell *pCell; int cellBytes; + for (int cIdx = 0; cIdx < TDB_PAGE_TOTAL_CELLS(pPage); cIdx++) { pCell = tdbPageGetCell(pPage, cIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - if (szNews[nNews] + cellBytes > maxPageCapacity) { + if (szNews[nNews] + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { nNews++; } cntNews[nNews]++; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 479c7603e1..258eafcf63 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -99,6 +99,7 @@ struct SPage { // APIs #define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) +#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); From bd8c171be1d79dfefd53288739110cb0536ef2a5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:44:18 +0000 Subject: [PATCH 030/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 39 ++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index aa3e4c6b90..4169509280 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -978,10 +978,37 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD return 0; } -#endif - static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { - // TODO - ASSERT(0); - return 0; -} \ No newline at end of file + u8 flags; + u8 isLeaf; + int szCell; + int kLen = 0, vLen = 0; + + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + szCell = 0; + + if (!isLeaf) { + szCell += sizeof(SPgno); + } + + if (pPage->kLen == TDB_VARIANT_LEN) { + szCell += tdbGetVarInt(pCell + szCell, &kLen); + } else { + kLen = pPage->kLen; + } + + if (isLeaf) { + if (pPage->vLen == TDB_VARIANT_LEN) { + szCell += tdbGetVarInt(pCell + szCell, &vLen); + } else { + vLen = pPage->vLen; + } + } + + szCell = szCell + kLen + vLen; + + return szCell; +} + +#endif From d123388a1962630ead69fa7d13e4b92c5f7f4be5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 10:53:49 +0000 Subject: [PATCH 031/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4169509280..670cdc002a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -711,7 +711,16 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - // Loop to make the distribution even + nNews = nNews + 1; + + // back loop to make the distribution even + for (int iNew = nNews - 1; iNew > 0; iNew--) { + /* code */ + } + } + + SPage *pNews[5]; + { // Allocate the new pages } #if 0 From 925004770202e76b8de27e4a8f4cfbf48c9d67dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 21 Mar 2022 14:56:07 +0000 Subject: [PATCH 032/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 670cdc002a..98a5cf158c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -715,7 +715,34 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { - /* code */ + // TODO: find the last cell of page (iNew-1) + int cIdx; + int iPage; + int cellBytes; + SPage *pPage; + SCell *pCell; + + for (;;) { + pCell = tdbPageGetCell(pPage, cIdx); + cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); + + if (szNews[iNew] + cellBytes >= szNews[iNew - 1] + cellBytes) { + break; + } + + // Move the cell right + szNews[iNew] += cellBytes; + cntNews[iNew]++; + + szNews[iNew - 1] -= cellBytes; + cntNews[iNew - 1]++; + + cIdx--; + if (cIdx < 0) { + pPage = pOlds[--iPage]; + cIdx = TDB_PAGE_TOTAL_CELLS(pPage) - 1; + } + } } } From e2f1434bf2c9d2b5cbe13408f4fbbf876a0ec417 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:08:56 +0000 Subject: [PATCH 033/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 108 ++++++++++-------------------- 1 file changed, 34 insertions(+), 74 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 98a5cf158c..cbb389140f 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -688,26 +688,33 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } int nNews = 0; - int cntNews[5] = {0}; // TODO: maybe 5 is not enough - int szNews[5] = {0}; + struct { + int cnt; + int size; + int oPage; + int oIdx; + } infoNews[5] = {0}; { // Get how many new pages are needed and the new distribution - // loop to find number of pages needed - for (int i = 0; i < nOlds; i++) { - SPage *pPage = pOlds[i]; + // first loop to find minimum number of pages needed + for (int oPage = 0; oPage < nOlds; oPage++) { + SPage *pPage = pOlds[oPage]; SCell *pCell; int cellBytes; - for (int cIdx = 0; cIdx < TDB_PAGE_TOTAL_CELLS(pPage); cIdx++) { - pCell = tdbPageGetCell(pPage, cIdx); + for (int oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { + pCell = tdbPageGetCell(pPage, oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - if (szNews[nNews] + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { + if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { + // page is full, use a new page nNews++; } - cntNews[nNews]++; - szNews[nNews] = szNews[nNews] + cellBytes; + infoNews[nNews].cnt++; + infoNews[nNews].size += cellBytes; + infoNews[nNews].oPage = oPage; + infoNews[nNews].oIdx = oIdx; } } @@ -715,85 +722,38 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { - // TODO: find the last cell of page (iNew-1) - int cIdx; - int iPage; - int cellBytes; - SPage *pPage; SCell *pCell; + SPage *pPage; + int cellBytes; for (;;) { - pCell = tdbPageGetCell(pPage, cIdx); + pPage = pOlds[infoNews[iNew - 1].oPage]; + pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - if (szNews[iNew] + cellBytes >= szNews[iNew - 1] + cellBytes) { - break; + infoNews[iNew].cnt++; + infoNews[iNew].size += cellBytes; + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= cellBytes; + if ((infoNews[iNew - 1].oIdx--) == 0) { + infoNews[iNew - 1].oPage--; + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pOlds[infoNews[iNew - 1].oPage]); } - // Move the cell right - szNews[iNew] += cellBytes; - cntNews[iNew]++; - - szNews[iNew - 1] -= cellBytes; - cntNews[iNew - 1]++; - - cIdx--; - if (cIdx < 0) { - pPage = pOlds[--iPage]; - cIdx = TDB_PAGE_TOTAL_CELLS(pPage) - 1; + if (infoNews[iNew].size > infoNews[iNew - 1].size) { + break; } } } + + int k = 0; } SPage *pNews[5]; - { // Allocate the new pages + { + // Allocate the new pages } -#if 0 - // Step 1: find two sibling pages and get engough info about the old pages - ret = tdbBtreeBalanceStep1(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 2: Load all cells on the old page and the divider cells - ret = tdbBtreeBalanceStep2(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 3: Get the number of pages needed to hold all cells - ret = tdbBtreeBalanceStep3(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 4: Allocate enough new pages. Reuse old pages as much as possible - ret = tdbBtreeBalanceStep4(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 5: Insert new divider cells into pParent - ret = tdbBtreeBalanceStep5(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Step 6: Update the sibling pages - ret = tdbBtreeBalanceStep6(&blh); - if (ret < 0) { - ASSERT(0); - return -1; - } -#endif - { // TODO: Reset states } From d5afc7240a92968e07027c464d1db6caee590084 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:39:12 +0000 Subject: [PATCH 034/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index cbb389140f..e969f15775 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -745,13 +745,29 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } - - int k = 0; } - SPage *pNews[5]; - { - // Allocate the new pages + SPage *pNews[5] = {0}; + { // Allocate new pages, reuse the old page when possible + + SPgno pgno; + SBtreeInitPageArg iarg; + u8 flags; + + flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); + + for (int iNew = 0; iNew < nNews; iNew++) { + if (iNew < nOlds) { + pNews[iNew] = pOlds[iNew]; + } else { + iarg.pBt = pBt; + iarg.flags = flags; + ret = tdbPagerNewPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeZeroPage, &iarg); + if (ret < 0) { + ASSERT(0); + } + } + } } { From 6cd9d2703428e5647fdd64a0c40c94d3cd68724f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:45:59 +0000 Subject: [PATCH 035/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 150 +----------------------------- 1 file changed, 1 insertion(+), 149 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e969f15775..af0f331373 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -498,150 +498,6 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return 0; } -static int tdbBtreeBalanceStep1(SBtreeBalanceHelper *pBlh) { - int nCells; - int i; - int idxStart; - int nChild; - int ret; - SPage *pParent; - SPgno pgno; - SCell *pCell; - SCellDecoder cd; - SBTree *pBt; - - pParent = pBlh->pParent; - nCells = TDB_PAGE_TOTAL_CELLS(pParent); - nChild = nCells + 1; - pBt = pBlh->pBt; - - // TODO: ASSERT below needs to be removed - ASSERT(pParent->nOverflow == 0); - ASSERT(pBlh->idx <= nCells); - - if (nChild < 3) { - idxStart = 0; - pBlh->nOld = nChild; - } else { - if (pBlh->idx == 0) { - idxStart = 0; - } else if (pBlh->idx == nCells) { - idxStart = pBlh->idx - 2; - } else { - idxStart = pBlh->idx - 1; - } - pBlh->nOld = 3; - } - - i = pBlh->nOld - 1; - - if (idxStart + i == nCells) { - // pgno = ((SBtPageHdr *)(pParent->pAmHdr))[0].rChild; - } else { - pCell = tdbPageGetCell(pParent, idxStart + i); - // TODO: no need to decode the payload part, and even the kLen, vLen part - // we only need the pgno part - ret = tdbBtreeDecodeCell(pParent, pCell, &cd); - if (ret < 0) { - ASSERT(0); - return -1; - } - pgno = cd.pgno; - } - for (;;) { - ret = tdbPagerFetchPage(pBt->pPager, pgno, &(pBlh->pOldPages[i]), tdbBtreeInitPage, pBt); - if (ret < 0) { - ASSERT(0); - return -1; - } - - // Loop over - if ((i--) == 0) break; - - { - // TODO - // ASSERT(0); - } - } - - return 0; -} - -static int tdbBtreeBalanceStep2(SBtreeBalanceHelper *pBlh) { -#if 0 - SPage *pPage; - int oidx; - int cidx; - int limit; - SCell *pCell; - - for (int i = 0; i < pBlh->nOld; i++) { - pPage = pBlh->pOldPages[i]; - oidx = 0; - cidx = 0; - - if (oidx < pPage->nOverflow) { - limit = pPage->aiOvfl[oidx]; - } else { - limit = pPage->pPageHdr->nCells; - } - - // Loop to copy each cell pointer out - for (;;) { - if (oidx >= pPage->nOverflow && cidx >= pPage->pPageHdr->nCells) break; - - if (cidx < limit) { - // Get local cells - pCell = tdbPageGetCell(pPage, cidx); - } else if (cidx == limit) { - // Get overflow cells - pCell = pPage->apOvfl[oidx++]; - - if (oidx < pPage->nOverflow) { - limit = pPage->aiOvfl[oidx]; - } else { - limit = pPage->pPageHdr->nCells; - } - } else { - ASSERT(0); - } - } - - { - // TODO: Copy divider cells here - } - } - - /* TODO */ - -#endif - return 0; -} - -static int tdbBtreeBalanceStep3(SBtreeBalanceHelper *pBlh) { - // Figure out number of pages needed after balance - for (int i = 0; i < pBlh->nOld; i++) { - /* TODO */ - } - - return 0; -} - -static int tdbBtreeBalanceStep4(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - -static int tdbBtreeBalanceStep5(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - -static int tdbBtreeBalanceStep6(SBtreeBalanceHelper *pBlh) { - // TODO - return 0; -} - static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; @@ -771,11 +627,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } { - // TODO: Reset states - } - - { - // TODO: Clear resources + // Do the actual redistribute } return 0; From f3c10f886c589766bda0ade1f4c9a88b24ef3fd4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 02:48:32 +0000 Subject: [PATCH 036/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index af0f331373..71a9e70064 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -310,32 +310,6 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p return 0; } -static int tdbBtCursorMoveToRoot(SBtCursor *pCur) { - SBTree *pBt; - SPager *pPager; - SPage *pPage; - int ret; - - pBt = pCur->pBt; - pPager = pBt->pPager; - - // pPage = tdbPagerGet(pPager, pBt->root, true); - // if (pPage == NULL) { - // // TODO: handle error - // } - - // ret = tdbInitBtPage(pPage, &pBtPage); - // if (ret < 0) { - // // TODO - // return 0; - // } - - // pCur->pPage = pBtPage; - // pCur->iPage = 0; - - return 0; -} - static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { int mlen; int cret; From df12ae0d79e73725e54cf4c7ca44743dd21eda71 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 05:28:54 +0000 Subject: [PATCH 037/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 50 ++++++++++++++++++++++++++++-- source/libs/tdb/src/db/tdbPCache.c | 14 +-------- source/libs/tdb/src/inc/tdbUtil.h | 10 ++++++ source/libs/tdb/src/page/tdbPage.c | 7 +++++ 4 files changed, 66 insertions(+), 15 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 71a9e70064..196c4593c5 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -598,10 +598,56 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } + + // TODO: sort the page according to the page number } - { - // Do the actual redistribute + { // Do the actual cell distribution + + SPage *pTPage[2]; + int tPage, tIdx, iOld; + SCell *pCell; + int szCell; + SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; + + for (int i = 0; i < 2; i++) { + ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); + if (ret < 0) { + ASSERT(0); + } + } + + tPage = 0; + tIdx = 0; + iOld = 0; + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + + for (int iNew = 0; iNew < nNews; iNew++) { + // fill the iNew page + tdbBtreeZeroPage(pNews[iNew], &iarg); + + for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { + while (tIdx >= TDB_PAGE_TOTAL_CELLS(pTPage[tPage])) { + tPage = (tPage + 1) % 2; + tIdx = 0; + + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + } + + pCell = tdbPageGetCell(pTPage[tPage], tIdx); + szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + + tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell); + + tIdx++; + } + } + + for (int i = 0; i < 2; i++) { + tdbPageDestroy(pTPage[i], NULL, NULL); + } } return 0; diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 29f7395cbc..1fc0847b38 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -57,8 +57,6 @@ static void tdbPCachePinPage(SPage *pPage); static void tdbPCacheRemovePageFromHash(SPage *pPage); static void tdbPCacheAddPageToHash(SPage *pPage); static void tdbPCacheUnpinPage(SPage *pPage); -static void *tdbOsMalloc(void *arg, size_t size); -static void tdbOsFree(void *arg, void *ptr); int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { SPCache *pCache; @@ -257,7 +255,7 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { pCache->nFree = 0; pCache->pFree = NULL; for (int i = 0; i < pCache->cacheSize; i++) { - ret = tdbPageCreate(pCache->pageSize, &pPage, tdbOsMalloc, NULL); + ret = tdbPageCreate(pCache->pageSize, &pPage, NULL, NULL); if (ret < 0) { // TODO: handle error return -1; @@ -297,13 +295,3 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { } int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; } - -static void *tdbOsMalloc(void *arg, size_t size) { - void *ptr; - - ptr = malloc(size); - - return ptr; -} - -static void tdbOsFree(void *arg, void *ptr) { free(ptr); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 9576ee6d15..9358aae236 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -39,6 +39,16 @@ int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); int tdbPRead(int fd, void *pData, int count, i64 offset); +static inline void *tdbOsMalloc(void *arg, size_t size) { + void *ptr; + + ptr = malloc(size); + + return ptr; +} + +static inline void tdbOsFree(void *arg, void *ptr) { free(ptr); } + static inline int tdbPutVarInt(u8 *p, int v) { int n = 0; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 0dc7853940..16130ba13f 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -48,6 +48,9 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = NULL; size = pageSize + sizeof(*pPage); + if (xMalloc == NULL) { + xMalloc = tdbOsMalloc; + } ptr = (u8 *)((*xMalloc)(arg, size)); if (pPage == NULL) { @@ -73,6 +76,10 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { u8 *ptr; + if (!xFree) { + xFree = tdbOsFree; + } + ptr = pPage->pData; (*xFree)(arg, ptr); From 3abd4b2e672f5af1e1de3652427dbe840b71d9ce Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 06:08:28 +0000 Subject: [PATCH 038/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 16 +++++++++++++++- source/libs/tdb/src/inc/tdbPage.h | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 196c4593c5..a05202f456 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -602,7 +602,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // TODO: sort the page according to the page number } - { // Do the actual cell distribution + { // Do the real cell distribution SPage *pTPage[2]; int tPage, tIdx, iOld; @@ -650,6 +650,20 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + { // Insert records in parent page + int cIdx; + int szCell; + SCell pCell[128]; // TODO + SCellDecoder cd; + + for (int iNew = 0; iNew < nNews; iNew++) { + tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); + + tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); + tdbPageInsertCell(pParent, cIdx, pCell, szCell); + } + } + return 0; } diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 258eafcf63..61927fabf6 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -100,6 +100,7 @@ struct SPage { // APIs #define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) #define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) +#define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno) #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); From 0b752392aa482ab45cd38d6d78a4112c84782042 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 06:24:16 +0000 Subject: [PATCH 039/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a05202f456..da11eb099a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -495,17 +495,17 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } nOlds = 3; } - for (int i = 0; i < nOlds; i++, sIdx++) { - ASSERT(sIdx <= nCells); + for (int i = 0; i < nOlds; i++) { + ASSERT(sIdx + i <= nCells); SPgno pgno; - if (sIdx == nCells) { + if (sIdx + i == nCells) { ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pParent))); pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { SCell *pCell; - pCell = tdbPageGetCell(pParent, sIdx); + pCell = tdbPageGetCell(pParent, sIdx + i); pgno = *(SPgno *)pCell; } @@ -515,6 +515,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return -1; } } + // drop the cells + for (int i = 0; i < nOlds; i++) { + nCells = TDB_PAGE_TOTAL_CELLS(pParent); + if (sIdx < nCells) { + tdbPageDropCell(pParent, sIdx); + } else { + ((SIntHdr *)pParent->pData)->pgno = 0; + } + } } int nNews = 0; @@ -657,6 +666,16 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SCellDecoder cd; for (int iNew = 0; iNew < nNews; iNew++) { + if (iNew == nNews - 1) { + // The last new page + + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + break; + } + } + tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); @@ -664,6 +683,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + int k = 0; + return 0; } From 2d1e356cc214aa99aed731b17352635f95728f41 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 06:56:29 +0000 Subject: [PATCH 040/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index da11eb099a..e7abbc7161 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -477,10 +477,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nOlds; SPage *pOlds[3]; + int sIdx; { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); - int sIdx; if (nCells <= 2) { sIdx = 0; nOlds = nCells + 1; @@ -515,7 +515,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return -1; } } - // drop the cells + // drop the cells on parent page for (int i = 0; i < nOlds; i++) { nCells = TDB_PAGE_TOTAL_CELLS(pParent); if (sIdx < nCells) { @@ -660,7 +660,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } { // Insert records in parent page - int cIdx; + int cIdx = sIdx; int szCell; SCell pCell[128]; // TODO SCellDecoder cd; @@ -679,12 +679,11 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); + // TODO: the cell here may be used by pParent as an overflow cell tdbPageInsertCell(pParent, cIdx, pCell, szCell); } } - int k = 0; - return 0; } From 238550b0f4d0ceeb987ad41eb7992afba994dacd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 07:50:34 +0000 Subject: [PATCH 041/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 36 ++++++++++++++++++++++++++---- source/libs/tdb/src/inc/tdbPage.h | 1 + source/libs/tdb/src/page/tdbPage.c | 1 - 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e7abbc7161..d1188ee5b3 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -477,10 +477,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nOlds; SPage *pOlds[3]; + SCell *pDivCell[2] = {0}; + int szDivCell[2]; int sIdx; + u8 childLeaf; { // Find 3 child pages at most to do balance - int nCells = TDB_PAGE_TOTAL_CELLS(pParent); + int nCells = TDB_PAGE_TOTAL_CELLS(pParent); + SCell *pCell; + if (nCells <= 2) { sIdx = 0; nOlds = nCells + 1; @@ -503,8 +508,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pParent))); pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { - SCell *pCell; - pCell = tdbPageGetCell(pParent, sIdx + i); pgno = *(SPgno *)pCell; } @@ -515,6 +518,17 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return -1; } } + // copy the parent key out if child pages are not leaf page + childLeaf = TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); + if (childLeaf) { + for (int i = 0; i < nOlds - 1; i++) { + pCell = tdbPageGetCell(pParent, sIdx + i); + + szDivCell[i] = tdbBtreeCellSize(pParent, pCell); + pDivCell[i] = malloc(szDivCell[i]); + memcpy(pDivCell, pCell, szDivCell[i]); + } + } // drop the cells on parent page for (int i = 0; i < nOlds; i++) { nCells = TDB_PAGE_TOTAL_CELLS(pParent); @@ -541,14 +555,28 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SPage *pPage = pOlds[oPage]; SCell *pCell; int cellBytes; + int oIdx; - for (int oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { + for (oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { pCell = tdbPageGetCell(pPage, oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; + // for a child leaf case, this cell is used as the new divider cell + if (childLeaf) continue; + } + infoNews[nNews].cnt++; + infoNews[nNews].size += cellBytes; + infoNews[nNews].oPage = oPage; + infoNews[nNews].oIdx = oIdx; + } + + // For child leaf pages + if (childLeaf && oPage < nOlds - 1) { + if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { + nNews++; } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 61927fabf6..0ba7cf236b 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -102,6 +102,7 @@ struct SPage { #define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx) #define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno) #define TDB_BYTES_CELL_TAKEN(pPage, pCell) ((*(pPage)->xCellSize)(pPage, pCell) + (pPage)->pPageMethods->szOffset) +#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg); int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 16130ba13f..14960de45d 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -18,7 +18,6 @@ extern SPageMethods pageMethods; extern SPageMethods pageLargeMethods; -#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset) #define TDB_PAGE_HDR_SIZE(pPage) ((pPage)->pPageMethods->szPageHdr) #define TDB_PAGE_FREE_CELL_SIZE(pPage) ((pPage)->pPageMethods->szFreeCell) #define TDB_PAGE_NCELLS(pPage) (*(pPage)->pPageMethods->getCellNum)(pPage) From aee8b603242f0db0facc04cb7b3793815395c521 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 08:13:32 +0000 Subject: [PATCH 042/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index d1188ee5b3..6daecb7908 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -564,7 +564,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for a child leaf case, this cell is used as the new divider cell + // for a child leaf case, this cell is used as the new divider cell to parent if (childLeaf) continue; } infoNews[nNews].cnt++; @@ -591,10 +591,13 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { for (int iNew = nNews - 1; iNew > 0; iNew--) { SCell *pCell; SPage *pPage; + int nCells; int cellBytes; + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + for (;;) { - pPage = pOlds[infoNews[iNew - 1].oPage]; pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); @@ -604,7 +607,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { infoNews[iNew - 1].size -= cellBytes; if ((infoNews[iNew - 1].oIdx--) == 0) { infoNews[iNew - 1].oPage--; - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pOlds[infoNews[iNew - 1].oPage]); + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); } if (infoNews[iNew].size > infoNews[iNew - 1].size) { From d7b54201bcab335b39a881d32f06af88bb987d72 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 08:26:43 +0000 Subject: [PATCH 043/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 50 +++++++++++++++++-------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 6daecb7908..cda958c578 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -520,7 +520,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } // copy the parent key out if child pages are not leaf page childLeaf = TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); - if (childLeaf) { + if (!childLeaf) { for (int i = 0; i < nOlds - 1; i++) { pCell = tdbPageGetCell(pParent, sIdx + i); @@ -564,8 +564,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for a child leaf case, this cell is used as the new divider cell to parent - if (childLeaf) continue; + // for a internal leaf case, this cell is used as the new divider cell to parent + if (!childLeaf) continue; } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; @@ -573,8 +573,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { infoNews[nNews].oIdx = oIdx; } - // For child leaf pages - if (childLeaf && oPage < nOlds - 1) { + // For internal pages + if (!childLeaf && oPage < nOlds - 1) { if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { nNews++; } @@ -594,27 +594,31 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nCells; int cellBytes; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); + if (childLeaf) { // child leaf + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); - for (;;) { - pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); - cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); + for (;;) { + pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); + cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - infoNews[iNew].cnt++; - infoNews[iNew].size += cellBytes; - infoNews[iNew - 1].cnt--; - infoNews[iNew - 1].size -= cellBytes; - if ((infoNews[iNew - 1].oIdx--) == 0) { - infoNews[iNew - 1].oPage--; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); - } - - if (infoNews[iNew].size > infoNews[iNew - 1].size) { - break; + infoNews[iNew].cnt++; + infoNews[iNew].size += cellBytes; + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= cellBytes; + if ((infoNews[iNew - 1].oIdx--) == 0) { + infoNews[iNew - 1].oPage--; + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); + } + + if (infoNews[iNew].size > infoNews[iNew - 1].size) { + break; + } } + } else { // internal leaf + // TODO } } } From 3b49a863ea18e13f224088a212dc3a522089f770 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 10:43:51 +0000 Subject: [PATCH 044/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 194 ++++++++++++++++++++++++------ 1 file changed, 154 insertions(+), 40 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index cda958c578..4c6d85bc0a 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -564,7 +564,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for a internal leaf case, this cell is used as the new divider cell to parent + // for an internal leaf case, this cell is used as the new divider cell to parent if (!childLeaf) continue; } infoNews[nNews].cnt++; @@ -594,10 +594,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nCells; int cellBytes; - if (childLeaf) { // child leaf - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + if (childLeaf) { // child leaf for (;;) { pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); @@ -618,7 +618,55 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } else { // internal leaf - // TODO + SCell *pCellRight; + SCell *pCellLeft; + int szCellRight, szCellLeft; + + // get pCellLeft, szCellLeft + if (infoNews[iNew - 1].oIdx == nCells) { + ASSERT(infoNews[iNew - 1].oPage < nOlds - 1); + pCellLeft = pDivCell[infoNews[iNew - 1].oPage]; + szCellLeft = szDivCell[infoNews[iNew - 1].oPage]; + } else { + pCellLeft = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); + szCellLeft = tdbBtreeCellSize(pPage, pCellLeft); + } + + // get pCellRight, szCellRight + if (infoNews[iNew - 1].oIdx + 1 < nCells) { + pCellRight = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx + 1); + szCellRight = tdbBtreeCellSize(pPage, pCellRight); + } else { + if (infoNews[iNew - 1].oPage < nOlds - 1) { + pCellRight = pDivCell[infoNews[iNew - 1].oPage]; + szCellRight = szDivCell[infoNews[iNew - 1].oPage]; + } else { + // TODO: what if the next old page is empty? + pCellRight = tdbPageGetCell(pOlds[infoNews[iNew - 1].oPage + 1], 0); + szCellRight = tdbBtreeCellSize(pPage, pCellRight); + } + } + + if (infoNews[iNew - 1].size - szCellLeft - TDB_PAGE_OFFSET_SIZE(pPage) <= + infoNews[iNew].size + szCellRight + TDB_PAGE_OFFSET_SIZE(pPage)) { + break; + } + + // TODO: ASSERT(infoNews[iNew].size < PAGE_MAX_CAPACITY); + infoNews[iNew].cnt++; + infoNews[iNew].size += szCellRight; + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= szCellLeft; + if ((infoNews[iNew - 1].oIdx--) == 0) { + infoNews[iNew - 1].oPage--; + pPage = pOlds[infoNews[iNew - 1].oPage]; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + if (infoNews[iNew - 1].oPage < nOlds - 1) { + infoNews[iNew - 1].oIdx = nCells; + } else { + infoNews[iNew - 1].oIdx = nCells - 1; + } + } } } } @@ -654,6 +702,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int tPage, tIdx, iOld; SCell *pCell; int szCell; + int nCells; + SCellDecoder cd; SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; for (int i = 0; i < 2; i++) { @@ -666,28 +716,117 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tPage = 0; tIdx = 0; iOld = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + if (!childLeaf) { + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + } for (int iNew = 0; iNew < nNews; iNew++) { // fill the iNew page + // TODO: copy current page to tmp space tdbBtreeZeroPage(pNews[iNew], &iarg); for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { - while (tIdx >= TDB_PAGE_TOTAL_CELLS(pTPage[tPage])) { - tPage = (tPage + 1) % 2; - tIdx = 0; + for (;;) { // loop to find the next available cell + if (tIdx < nCells) { + pCell = tdbPageGetCell(pTPage[tPage], tIdx); + szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + tIdx++; + break; + } else { + if (!childLeaf) { + if (iOld < nOlds - 1) { + pCell = pDivCell[iOld]; + szCell = szDivCell[iOld]; + ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld++], pTPage[tPage]); + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + break; + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + } + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + } + } } - pCell = tdbPageGetCell(pTPage[tPage], tIdx); - szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell); + } - tIdx++; + // fill right-most child pgno if internal page + if (!childLeaf) { + if (tIdx < nCells) { + pCell = tdbPageGetCell(pTPage[tPage], tIdx); + szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + tIdx++; + break; + } else { + if (!childLeaf) { + if (iOld < nOlds - 1) { + pCell = pDivCell[iOld]; + szCell = szDivCell[iOld]; + ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + break; + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + } + } else { + iOld++; + tPage = (tPage + 1) % 2; + tIdx = 0; + nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + tdbBtreeZeroPage(pTPage[tPage], &iarg); + tdbPageCopy(pOlds[iOld], pTPage[tPage]); + } + } + + ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + } + + // insert divider cell into the parent page + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + } else { + tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); + tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, + &szCell); + // TODO: the cell here may be used by pParent as an overflow cell + tdbPageInsertCell(pParent, sIdx++, pCell, szCell); } } @@ -696,31 +835,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - { // Insert records in parent page - int cIdx = sIdx; - int szCell; - SCell pCell[128]; // TODO - SCellDecoder cd; - - for (int iNew = 0; iNew < nNews; iNew++) { - if (iNew == nNews - 1) { - // The last new page - - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - if (pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - break; - } - } - - tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); - - tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); - // TODO: the cell here may be used by pParent as an overflow cell - tdbPageInsertCell(pParent, cIdx, pCell, szCell); - } - } - return 0; } From 3a173f466069f6cf59f2e9e281eaf9298886766d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Mar 2022 11:47:25 +0000 Subject: [PATCH 045/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4c6d85bc0a..67b11f8ef4 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -204,7 +204,19 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { - // TODO + int ret; + + pCur->pgStack[pCur->iPage] = pCur->pPage; + pCur->idxStack[pCur->iPage] = pCur->idx; + pCur->iPage++; + pCur->pPage = NULL; + pCur->idx = -1; + + ret = tdbPagerFetchPage(pCur->pBt->pPager, pgno, &pCur->pPage, tdbBtreeInitPage, pCur->pBt); + if (ret < 0) { + ASSERT(0); + } + return 0; } @@ -287,14 +299,12 @@ static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *p pCur->idx = midx; tdbBtCursorMoveToChild(pCur, cd.pgno); } else { + pCur->idx = midx + 1; if (midx == nCells - 1) { /* Move to right-most child */ - pCur->idx = midx + 1; - // tdbBtCursorMoveToChild(pCur, ((SBtPageHdr *)(pPage->pAmHdr))->rChild); + tdbBtCursorMoveToChild(pCur, ((SIntHdr *)pCur->pPage->pData)->pgno); } else { - // TODO: reset cd as uninitialized - pCur->idx = midx + 1; - pCell = tdbPageGetCell(pPage, midx + 1); + pCell = tdbPageGetCell(pPage, pCur->idx); tdbBtreeDecodeCell(pPage, pCell, &cd); tdbBtCursorMoveToChild(pCur, cd.pgno); } From 44daa77d8db957e3d9e7ca150bfce61f26f8764a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 05:32:39 +0000 Subject: [PATCH 046/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 33 ++++++++++++++++++------------ source/libs/tdb/src/inc/tdbPage.h | 2 +- source/libs/tdb/src/page/tdbPage.c | 28 ++++++++++++------------- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 67b11f8ef4..a1f879c729 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -187,7 +187,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } // Insert the cell to the index - ret = tdbPageInsertCell(pCur->pPage, idx, pCell, szCell); + ret = tdbPageInsertCell(pCur->pPage, idx, pCell, szCell, 0); if (ret < 0) { return -1; } @@ -490,7 +490,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SCell *pDivCell[2] = {0}; int szDivCell[2]; int sIdx; - u8 childLeaf; + u8 childNotLeaf; { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); @@ -529,14 +529,21 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } // copy the parent key out if child pages are not leaf page - childLeaf = TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); - if (!childLeaf) { + childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); + if (childNotLeaf) { for (int i = 0; i < nOlds - 1; i++) { pCell = tdbPageGetCell(pParent, sIdx + i); szDivCell[i] = tdbBtreeCellSize(pParent, pCell); pDivCell[i] = malloc(szDivCell[i]); memcpy(pDivCell, pCell, szDivCell[i]); + + ((SPgno *)pDivCell)[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; + ((SIntHdr *)pOlds[i]->pData)->pgno = 0; + + // here we insert the cell as an overflow cell to avoid + // the slow defragment process + tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } } // drop the cells on parent page @@ -575,7 +582,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // page is full, use a new page nNews++; // for an internal leaf case, this cell is used as the new divider cell to parent - if (!childLeaf) continue; + if (childNotLeaf) continue; } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; @@ -584,7 +591,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } // For internal pages - if (!childLeaf && oPage < nOlds - 1) { + if (childNotLeaf && oPage < nOlds - 1) { if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { nNews++; } @@ -607,7 +614,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { pPage = pOlds[infoNews[iNew - 1].oPage]; nCells = TDB_PAGE_TOTAL_CELLS(pPage); - if (childLeaf) { // child leaf + if (!childNotLeaf) { // child leaf for (;;) { pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); @@ -729,7 +736,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); tdbBtreeZeroPage(pTPage[tPage], &iarg); tdbPageCopy(pOlds[iOld], pTPage[tPage]); - if (!childLeaf) { + if (childNotLeaf) { ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; } @@ -746,7 +753,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tIdx++; break; } else { - if (!childLeaf) { + if (childNotLeaf) { if (iOld < nOlds - 1) { pCell = pDivCell[iOld]; szCell = szDivCell[iOld]; @@ -780,18 +787,18 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell); + tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); } // fill right-most child pgno if internal page - if (!childLeaf) { + if (childNotLeaf) { if (tIdx < nCells) { pCell = tdbPageGetCell(pTPage[tPage], tIdx); szCell = tdbBtreeCellSize(pTPage[tPage], pCell); tIdx++; break; } else { - if (!childLeaf) { + if (!childNotLeaf) { if (iOld < nOlds - 1) { pCell = pDivCell[iOld]; szCell = szDivCell[iOld]; @@ -836,7 +843,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, &szCell); // TODO: the cell here may be used by pParent as an overflow cell - tdbPageInsertCell(pParent, sIdx++, pCell, szCell); + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); } } diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 0ba7cf236b..a6f9fbf615 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -108,7 +108,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg); void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *)); -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell); +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); int tdbPageDropCell(SPage *pPage, int idx); void tdbPageCopy(SPage *pFromPage, SPage *pToPage); diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 14960de45d..ee1a021a1b 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -114,7 +114,7 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); } -int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { +int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl) { int nFree; int nCells; int iOvfl; @@ -135,7 +135,19 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { lidx = idx - iOvfl; - if (nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + if (asOvfl || nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + // TODO: make it extensible + // add the cell as an overflow cell + for (int i = pPage->nOverflow; i > iOvfl; i--) { + pPage->apOvfl[i] = pPage->apOvfl[i - 1]; + pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; + } + + pPage->apOvfl[iOvfl] = pCell; + pPage->aiOvfl[iOvfl] = idx; + pPage->nOverflow++; + iOvfl++; + } else { // page must has enough space to hold the cell locally tdbPageAllocate(pPage, szCell, &pNewCell); @@ -149,18 +161,6 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell) { TDB_PAGE_NCELLS_SET(pPage, nCells + 1); ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); - } else { - // TODO: make it extensible - // add the cell as an overflow cell - for (int i = pPage->nOverflow; i > iOvfl; i--) { - pPage->apOvfl[i] = pPage->apOvfl[i - 1]; - pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; - } - - pPage->apOvfl[iOvfl] = pCell; - pPage->aiOvfl[iOvfl] = idx; - pPage->nOverflow++; - iOvfl++; } for (; iOvfl < pPage->nOverflow; iOvfl++) { From 5c9e63e5018f42af709ac3a75fa831fb7bacff86 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 06:56:53 +0000 Subject: [PATCH 047/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 327 +++++++++++++++++------------- source/libs/tdb/src/inc/tdbInt.h | 4 +- 2 files changed, 183 insertions(+), 148 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a1f879c729..459154e9fb 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -451,13 +451,15 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { SPage *pChild; SPgno pgnoChild; int ret; + u8 flags; SIntHdr *pIntHdr; SBtreeInitPageArg zArg; pPager = pRoot->pPager; + flags = TDB_BTREE_PAGE_GET_FLAGS(pRoot); // Allocate a new child page - zArg.flags = TDB_BTREE_LEAF; + zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT); zArg.pBt = pBt; ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg); if (ret < 0) { @@ -559,10 +561,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nNews = 0; struct { - int cnt; - int size; - int oPage; - int oIdx; + int cnt; + int size; + SPage *pPage; + int oIdx; } infoNews[5] = {0}; { // Get how many new pages are needed and the new distribution @@ -581,31 +583,63 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (infoNews[nNews].size + cellBytes > TDB_PAGE_USABLE_SIZE(pPage)) { // page is full, use a new page nNews++; - // for an internal leaf case, this cell is used as the new divider cell to parent - if (childNotLeaf) continue; - } - infoNews[nNews].cnt++; - infoNews[nNews].size += cellBytes; - infoNews[nNews].oPage = oPage; - infoNews[nNews].oIdx = oIdx; - } - // For internal pages - if (childNotLeaf && oPage < nOlds - 1) { - if (infoNews[nNews].size + szDivCell[oPage] + TDB_PAGE_OFFSET_SIZE(pPage) > TDB_PAGE_USABLE_SIZE(pPage)) { - nNews++; + ASSERT(infoNews[nNews].size + cellBytes <= TDB_PAGE_USABLE_SIZE(pPage)); + + if (childNotLeaf) { + // for non-child page, this cell is used as the right-most child, + // the divider cell to parent as well + continue; + } } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; - infoNews[nNews].oPage = oPage; + infoNews[nNews].pPage = pPage; infoNews[nNews].oIdx = oIdx; } } - nNews = nNews + 1; + nNews++; // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { + SCell *pLCell, pRCell; + int szLCell, szRCell; + + for (;;) { + if (childNotLeaf) { + pLCell = pRCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); + szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pLCell); + } else { + // For internal page, find the left cell + // TODO: fill pLCell, pRCell, szLCell, szRCell + } + + ASSERT(infoNews[iNew - 1].cnt > 0); + + if (infoNews[iNew].size + szRCell >= infoNews[iNew - 1].size - szRCell) { + break; + } + + // move cell divider left + infoNews[iNew].cnt++; + infoNews[iNew].size += szRCell; + { + // TODO + // infoNews[iNew].oPage = ; + // infoNews[iNew].oIdx = ; + } + + infoNews[iNew - 1].cnt--; + infoNews[iNew - 1].size -= szLCell; + { + // TODO + // infoNews[iNew-1].oPage = ; + // infoNews[iNew-1].oIdx = ; + } + } + +#if 0 SCell *pCell; SPage *pPage; int nCells; @@ -685,6 +719,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } +#endif } } @@ -713,146 +748,146 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // TODO: sort the page according to the page number } - { // Do the real cell distribution + // { // Do the real cell distribution - SPage *pTPage[2]; - int tPage, tIdx, iOld; - SCell *pCell; - int szCell; - int nCells; - SCellDecoder cd; - SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; + // SPage *pTPage[2]; + // int tPage, tIdx, iOld; + // SCell *pCell; + // int szCell; + // int nCells; + // SCellDecoder cd; + // SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; - for (int i = 0; i < 2; i++) { - ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); - if (ret < 0) { - ASSERT(0); - } - } + // for (int i = 0; i < 2; i++) { + // ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); + // if (ret < 0) { + // ASSERT(0); + // } + // } - tPage = 0; - tIdx = 0; - iOld = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - if (childNotLeaf) { - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - } + // tPage = 0; + // tIdx = 0; + // iOld = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // if (childNotLeaf) { + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // } - for (int iNew = 0; iNew < nNews; iNew++) { - // fill the iNew page - // TODO: copy current page to tmp space - tdbBtreeZeroPage(pNews[iNew], &iarg); + // for (int iNew = 0; iNew < nNews; iNew++) { + // // fill the iNew page + // // TODO: copy current page to tmp space + // tdbBtreeZeroPage(pNews[iNew], &iarg); - for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { - for (;;) { // loop to find the next available cell - if (tIdx < nCells) { - pCell = tdbPageGetCell(pTPage[tPage], tIdx); - szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - tIdx++; - break; - } else { - if (childNotLeaf) { - if (iOld < nOlds - 1) { - pCell = pDivCell[iOld]; - szCell = szDivCell[iOld]; - ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { + // for (;;) { // loop to find the next available cell + // if (tIdx < nCells) { + // pCell = tdbPageGetCell(pTPage[tPage], tIdx); + // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + // tIdx++; + // break; + // } else { + // if (childNotLeaf) { + // if (iOld < nOlds - 1) { + // pCell = pDivCell[iOld]; + // szCell = szDivCell[iOld]; + // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - break; - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - } - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - } - } - } + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // break; + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // } + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // } + // } + // } - tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); - } + // tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); + // } - // fill right-most child pgno if internal page - if (childNotLeaf) { - if (tIdx < nCells) { - pCell = tdbPageGetCell(pTPage[tPage], tIdx); - szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - tIdx++; - break; - } else { - if (!childNotLeaf) { - if (iOld < nOlds - 1) { - pCell = pDivCell[iOld]; - szCell = szDivCell[iOld]; - ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // // fill right-most child pgno if internal page + // if (childNotLeaf) { + // if (tIdx < nCells) { + // pCell = tdbPageGetCell(pTPage[tPage], tIdx); + // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); + // tIdx++; + // break; + // } else { + // if (!childNotLeaf) { + // if (iOld < nOlds - 1) { + // pCell = pDivCell[iOld]; + // szCell = szDivCell[iOld]; + // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - break; - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - } - } else { - iOld++; - tPage = (tPage + 1) % 2; - tIdx = 0; - nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - tdbBtreeZeroPage(pTPage[tPage], &iarg); - tdbPageCopy(pOlds[iOld], pTPage[tPage]); - } - } + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // break; + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // } + // } else { + // iOld++; + // tPage = (tPage + 1) % 2; + // tIdx = 0; + // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); + // tdbBtreeZeroPage(pTPage[tPage], &iarg); + // tdbPageCopy(pOlds[iOld], pTPage[tPage]); + // } + // } - ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; - } + // ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + // } - // insert divider cell into the parent page - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - if (iNew == nNews - 1 && pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - } else { - tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); - tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, - &szCell); - // TODO: the cell here may be used by pParent as an overflow cell - tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); - } - } + // // insert divider cell into the parent page + // SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + // if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + // pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + // } else { + // tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); + // tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, + // &szCell); + // // TODO: the cell here may be used by pParent as an overflow cell + // tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + // } + // } - for (int i = 0; i < 2; i++) { - tdbPageDestroy(pTPage[i], NULL, NULL); - } - } + // for (int i = 0; i < 2; i++) { + // tdbPageDestroy(pTPage[i], NULL, NULL); + // } +} - return 0; +return 0; } static int tdbBtreeBalance(SBtCursor *pCur) { diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 5902a6a716..3ebba0a4b5 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -141,8 +141,8 @@ typedef int (*FKeyComparator)(const void *pKey1, int kLen1, const void *pKey2, i #define TDB_FLAG_IS(flags, flag) ((flags) == (flag)) #define TDB_FLAG_HAS(flags, flag) (((flags) & (flag)) != 0) #define TDB_FLAG_NO(flags, flag) ((flags) & (flag) == 0) -#define TDB_FLAG_ADD(flags, flag) ((flags) |= (flag)) -#define TDB_FLAG_REMOVE(flags, flag) ((flags) &= (~(flag))) +#define TDB_FLAG_ADD(flags, flag) ((flags) | (flag)) +#define TDB_FLAG_REMOVE(flags, flag) ((flags) & (~(flag))) typedef struct SPager SPager; typedef struct SPCache SPCache; From 9b8a8f12871a85b12d351e4e707c593e3254d770 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:17:26 +0000 Subject: [PATCH 048/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 119 +++++++----------------------- 1 file changed, 25 insertions(+), 94 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 459154e9fb..f649a8e9f6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -603,16 +603,30 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // back loop to make the distribution even for (int iNew = nNews - 1; iNew > 0; iNew--) { - SCell *pLCell, pRCell; + SCell *pCell; int szLCell, szRCell; for (;;) { + pCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); + if (childNotLeaf) { - pLCell = pRCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); - szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pLCell); + szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); } else { - // For internal page, find the left cell - // TODO: fill pLCell, pRCell, szLCell, szRCell + szLCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); + + SPage *pPage = infoNews[iNew - 1].pPage; + int oIdx = infoNews[iNew - 1].oIdx + 1; + for (;;) { + if (oIdx < TDB_PAGE_TOTAL_CELLS(pPage)) { + break; + } + + pPage++; + oIdx = 0; + } + + pCell = tdbPageGetCell(pPage, oIdx); + szRCell = tdbBtreeCellSize(pPage, pCell); } ASSERT(infoNews[iNew - 1].cnt > 0); @@ -621,105 +635,22 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { break; } - // move cell divider left - infoNews[iNew].cnt++; - infoNews[iNew].size += szRCell; - { - // TODO - // infoNews[iNew].oPage = ; - // infoNews[iNew].oIdx = ; - } - + // Move a cell right forward infoNews[iNew - 1].cnt--; infoNews[iNew - 1].size -= szLCell; - { - // TODO - // infoNews[iNew-1].oPage = ; - // infoNews[iNew-1].oIdx = ; - } - } - -#if 0 - SCell *pCell; - SPage *pPage; - int nCells; - int cellBytes; - - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - - if (!childNotLeaf) { // child leaf + infoNews[iNew - 1].oIdx--; for (;;) { - pCell = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); - cellBytes = TDB_BYTES_CELL_TAKEN(pPage, pCell); - - infoNews[iNew].cnt++; - infoNews[iNew].size += cellBytes; - infoNews[iNew - 1].cnt--; - infoNews[iNew - 1].size -= cellBytes; - if ((infoNews[iNew - 1].oIdx--) == 0) { - infoNews[iNew - 1].oPage--; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pPage); - } - - if (infoNews[iNew].size > infoNews[iNew - 1].size) { + if (infoNews[iNew - 1].oIdx >= 0) { break; } - } - } else { // internal leaf - SCell *pCellRight; - SCell *pCellLeft; - int szCellRight, szCellLeft; - // get pCellLeft, szCellLeft - if (infoNews[iNew - 1].oIdx == nCells) { - ASSERT(infoNews[iNew - 1].oPage < nOlds - 1); - pCellLeft = pDivCell[infoNews[iNew - 1].oPage]; - szCellLeft = szDivCell[infoNews[iNew - 1].oPage]; - } else { - pCellLeft = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx); - szCellLeft = tdbBtreeCellSize(pPage, pCellLeft); + infoNews[iNew - 1].pPage--; + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(infoNews[iNew - 1].pPage) - 1; } - // get pCellRight, szCellRight - if (infoNews[iNew - 1].oIdx + 1 < nCells) { - pCellRight = tdbPageGetCell(pPage, infoNews[iNew - 1].oIdx + 1); - szCellRight = tdbBtreeCellSize(pPage, pCellRight); - } else { - if (infoNews[iNew - 1].oPage < nOlds - 1) { - pCellRight = pDivCell[infoNews[iNew - 1].oPage]; - szCellRight = szDivCell[infoNews[iNew - 1].oPage]; - } else { - // TODO: what if the next old page is empty? - pCellRight = tdbPageGetCell(pOlds[infoNews[iNew - 1].oPage + 1], 0); - szCellRight = tdbBtreeCellSize(pPage, pCellRight); - } - } - - if (infoNews[iNew - 1].size - szCellLeft - TDB_PAGE_OFFSET_SIZE(pPage) <= - infoNews[iNew].size + szCellRight + TDB_PAGE_OFFSET_SIZE(pPage)) { - break; - } - - // TODO: ASSERT(infoNews[iNew].size < PAGE_MAX_CAPACITY); infoNews[iNew].cnt++; - infoNews[iNew].size += szCellRight; - infoNews[iNew - 1].cnt--; - infoNews[iNew - 1].size -= szCellLeft; - if ((infoNews[iNew - 1].oIdx--) == 0) { - infoNews[iNew - 1].oPage--; - pPage = pOlds[infoNews[iNew - 1].oPage]; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - if (infoNews[iNew - 1].oPage < nOlds - 1) { - infoNews[iNew - 1].oIdx = nCells; - } else { - infoNews[iNew - 1].oIdx = nCells - 1; - } - } + infoNews[iNew].size += szRCell; } -#endif } } From 15fa64ebaab2f3f8237105b83d060cc57c5c3089 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:39:57 +0000 Subject: [PATCH 049/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 177 ++++++++---------------------- 1 file changed, 48 insertions(+), 129 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f649a8e9f6..8912a765fd 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -679,146 +679,65 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // TODO: sort the page according to the page number } - // { // Do the real cell distribution + { // Do the real cell distribution + SPage *pOldsCopy[3]; + SCell *pCell; + int szCell; + SBtreeInitPageArg iarg; + int iNew, nNewCells; - // SPage *pTPage[2]; - // int tPage, tIdx, iOld; - // SCell *pCell; - // int szCell; - // int nCells; - // SCellDecoder cd; - // SBtreeInitPageArg iarg = {.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]), .pBt = pBt}; + iarg.pBt = pBt; + iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); + for (int i = 0; i < nOlds; i++) { + tdbPageCreate(pOlds[0]->pageSize, pOldsCopy + i, NULL, NULL); + tdbBtreeZeroPage(pOldsCopy[i], &iarg); + tdbPageCopy(pOlds[i], pOldsCopy[i]); + } + iNew = 0; + nNewCells = 0; - // for (int i = 0; i < 2; i++) { - // ret = tdbPageCreate(pOlds[0]->pageSize, &pTPage[i], NULL, NULL); - // if (ret < 0) { - // ASSERT(0); - // } - // } + for (int iOld = 0; iOld < nOlds; iOld++) { + SPage *pPage; - // tPage = 0; - // tIdx = 0; - // iOld = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // if (childNotLeaf) { - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // } + pPage = pOldsCopy[iOld]; - // for (int iNew = 0; iNew < nNews; iNew++) { - // // fill the iNew page - // // TODO: copy current page to tmp space - // tdbBtreeZeroPage(pNews[iNew], &iarg); + for (int oIdx = 0; oIdx < TDB_PAGE_TOTAL_CELLS(pPage); oIdx++) { + pCell = tdbPageGetCell(pPage, oIdx); + szCell = tdbBtreeCellSize(pPage, pCell); - // for (int iCell = 0; iCell < infoNews[iNew].cnt; iCell++) { - // for (;;) { // loop to find the next available cell - // if (tIdx < nCells) { - // pCell = tdbPageGetCell(pTPage[tPage], tIdx); - // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - // tIdx++; - // break; - // } else { - // if (childNotLeaf) { - // if (iOld < nOlds - 1) { - // pCell = pDivCell[iOld]; - // szCell = szDivCell[iOld]; - // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + ASSERT(nNewCells <= infoNews[iNew].cnt); - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // break; - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // } - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // } - // } - // } + if (nNewCells < infoNews[iNew].cnt) { + tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); + nNewCells++; + } else { + if (childNotLeaf) { + // set current new page right-most child - // tdbPageInsertCell(pNews[iNew], iCell, pCell, szCell, 0); - // } + // move to next new page + } else { + // move to next new page - // // fill right-most child pgno if internal page - // if (childNotLeaf) { - // if (tIdx < nCells) { - // pCell = tdbPageGetCell(pTPage[tPage], tIdx); - // szCell = tdbBtreeCellSize(pTPage[tPage], pCell); - // tIdx++; - // break; - // } else { - // if (!childNotLeaf) { - // if (iOld < nOlds - 1) { - // pCell = pDivCell[iOld]; - // szCell = szDivCell[iOld]; - // ((SPgno *)pCell)[0] = ((SIntHdr *)pOlds[iOld]->pData)->pgno; + // insert the cell to the new page + } + } - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // break; - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // ((SIntHdr *)pTPage[tPage]->pData)->pgno = ((SIntHdr *)pOlds[iOld]->pData)->pgno; - // } - // } else { - // iOld++; - // tPage = (tPage + 1) % 2; - // tIdx = 0; - // nCells = TDB_PAGE_TOTAL_CELLS(pOlds[iOld]); - // tdbBtreeZeroPage(pTPage[tPage], &iarg); - // tdbPageCopy(pOlds[iOld], pTPage[tPage]); - // } - // } + if (nNewCells == infoNews[iNew].cnt) { + if (childNotLeaf) { + // TODO + } else { + // TODO + } + } + } + } - // ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; - // } + for (int i = 0; i < nOlds; i++) { + tdbPageDestroy(pOldsCopy[i], NULL, NULL); + } + } - // // insert divider cell into the parent page - // SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - // if (iNew == nNews - 1 && pIntHdr->pgno == 0) { - // pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - // } else { - // tdbBtreeDecodeCell(pNews[iNew], tdbPageGetCell(pNews[iNew], TDB_PAGE_TOTAL_CELLS(pNews[iNew]) - 1), &cd); - // tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&TDB_PAGE_PGNO(pNews[iNew]), sizeof(SPgno), pCell, - // &szCell); - // // TODO: the cell here may be used by pParent as an overflow cell - // tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); - // } - // } - - // for (int i = 0; i < 2; i++) { - // tdbPageDestroy(pTPage[i], NULL, NULL); - // } -} - -return 0; + return 0; } static int tdbBtreeBalance(SBtCursor *pCur) { From f8fc05c76e035de6c0fe2ffa0dbec1eec7b10987 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:48:08 +0000 Subject: [PATCH 050/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8912a765fd..98b9455d13 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -712,13 +712,25 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nNewCells++; } else { if (childNotLeaf) { + // Insert to parent + // set current new page right-most child + ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; // move to next new page + iNew++; + nNewCells = 0; } else { + // TODO: Insert to parent max key and current page number or the right most child + // move to next new page + iNew++; + nNewCells = 0; // insert the cell to the new page + ASSERT(nNewCells < infoNews[iNew].cnt); + tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); + nNewCells++; } } From 7b89447239f1971547d44adfede31d113066784f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 07:57:15 +0000 Subject: [PATCH 051/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 98b9455d13..c6270d6047 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -733,14 +733,6 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nNewCells++; } } - - if (nNewCells == infoNews[iNew].cnt) { - if (childNotLeaf) { - // TODO - } else { - // TODO - } - } } } From 09833d2244a0da316a34d3b342513d55fc2a8753 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:21:09 +0000 Subject: [PATCH 052/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 49 ++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c6270d6047..ca50631e90 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -710,28 +710,43 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); nNewCells++; - } else { - if (childNotLeaf) { - // Insert to parent - // set current new page right-most child - ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + // insert parent page + if (!childNotLeaf && nNewCells == infoNews[iNew].cnt) { + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + + if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + } else { + // TODO: + ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + // TODO: pCell here may be inserted as an overflow cell, handle it + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + } // move to next new page iNew++; - nNewCells = 0; - } else { - // TODO: Insert to parent max key and current page number or the right most child - - // move to next new page - iNew++; - nNewCells = 0; - - // insert the cell to the new page - ASSERT(nNewCells < infoNews[iNew].cnt); - tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); - nNewCells++; + nNews = 0; } + } else { + ASSERT(childNotLeaf); + + // set current new page right-most child + ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; + + // insert to parent as divider cell + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + + if (iNew == nNews - 1 && pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); + } else { + ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + } + + // move to next new page + iNew++; + nNewCells = 0; } } } From d5a1c7a31f9ea48c7b41a1b7f23f3769b1683123 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:25:57 +0000 Subject: [PATCH 053/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index ca50631e90..a6747e8903 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -685,6 +685,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szCell; SBtreeInitPageArg iarg; int iNew, nNewCells; + SCellDecoder cd; iarg.pBt = pBt; iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); @@ -718,10 +719,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (iNew == nNews - 1 && pIntHdr->pgno == 0) { pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); } else { - // TODO: - ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + tdbBtreeDecodeCell(pPage, pCell, &cd); + // TODO: pCell here may be inserted as an overflow cell, handle it - tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); + SCell *pNewCell = malloc(cd.kLen + 9); + int szNewCell; + SPgno pgno; + pgno = TDB_PAGE_PGNO(pNews[iNew]); + tdbBtreeEncodeCell(pPage, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); + tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); } // move to next new page From 72aed8997dfcf56f4a1942b4b58ee651b201bf1a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:39:22 +0000 Subject: [PATCH 054/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a6747e8903..68be6ec8cd 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -493,6 +493,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szDivCell[2]; int sIdx; u8 childNotLeaf; + SPgno rPgno; { // Find 3 child pages at most to do balance int nCells = TDB_PAGE_TOTAL_CELLS(pParent); @@ -547,6 +548,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // the slow defragment process tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } + rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno; } // drop the cells on parent page for (int i = 0; i < nOlds; i++) { @@ -741,14 +743,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; // insert to parent as divider cell - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - - if (iNew == nNews - 1 && pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[iNew]); - } else { - ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); - tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); - } + ASSERT(iNew < nNews - 1); + ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); + tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); // move to next new page iNew++; From 795981feae04c4c470768c8de9cff682e10336f4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:54:38 +0000 Subject: [PATCH 055/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 68be6ec8cd..1503d68575 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -737,6 +737,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { nNews = 0; } } else { + ASSERT(0); ASSERT(childNotLeaf); // set current new page right-most child From 2bb84eeb2becf4e636d06fff26d9a2443e8903ae Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 08:59:10 +0000 Subject: [PATCH 056/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 1503d68575..50f585ebd3 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -698,6 +698,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } iNew = 0; nNewCells = 0; + tdbBtreeZeroPage(pNews[iNew], &iarg); for (int iOld = 0; iOld < nOlds; iOld++) { SPage *pPage; @@ -735,6 +736,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; nNews = 0; + tdbBtreeZeroPage(pNews[iNew], &iarg); } } else { ASSERT(0); @@ -751,6 +753,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; nNewCells = 0; + tdbBtreeZeroPage(pNews[iNew], &iarg); } } } From 1fba9d5d3d162085f4dabf76ca79a1d009492c93 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 09:03:01 +0000 Subject: [PATCH 057/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 50f585ebd3..81b8be3588 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -735,8 +735,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; - nNews = 0; - tdbBtreeZeroPage(pNews[iNew], &iarg); + nNewCells = 0; + if (iNew < nNews) { + tdbBtreeZeroPage(pNews[iNew], &iarg); + } } } else { ASSERT(0); @@ -753,7 +755,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // move to next new page iNew++; nNewCells = 0; - tdbBtreeZeroPage(pNews[iNew], &iarg); + if (iNew < nNews) { + tdbBtreeZeroPage(pNews[iNew], &iarg); + } } } } From 33e119e9c037c63136bc3729de300e5bc9527e79 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 09:12:21 +0000 Subject: [PATCH 058/121] fix another TDB bug --- source/libs/tdb/src/db/tdbBtree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 81b8be3588..f0055723b6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -729,7 +729,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szNewCell; SPgno pgno; pgno = TDB_PAGE_PGNO(pNews[iNew]); - tdbBtreeEncodeCell(pPage, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); + tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); } From 48306365aed114b632fdf3ce76e7077da130692d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 09:35:25 +0000 Subject: [PATCH 059/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f0055723b6..0f5e6733ed 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -563,10 +563,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nNews = 0; struct { - int cnt; - int size; - SPage *pPage; - int oIdx; + int cnt; + int size; + int iPage; + int oIdx; } infoNews[5] = {0}; { // Get how many new pages are needed and the new distribution @@ -596,7 +596,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } infoNews[nNews].cnt++; infoNews[nNews].size += cellBytes; - infoNews[nNews].pPage = pPage; + infoNews[nNews].iPage = oPage; infoNews[nNews].oIdx = oIdx; } } @@ -609,21 +609,23 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int szLCell, szRCell; for (;;) { - pCell = tdbPageGetCell(infoNews[iNew - 1].pPage, infoNews[iNew - 1].oIdx); + pCell = tdbPageGetCell(pOlds[infoNews[iNew - 1].iPage], infoNews[iNew - 1].oIdx); if (childNotLeaf) { - szLCell = szRCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); + szLCell = szRCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); } else { - szLCell = tdbBtreeCellSize(infoNews[iNew - 1].pPage, pCell); + szLCell = tdbBtreeCellSize(pOlds[infoNews[iNew - 1].iPage], pCell); - SPage *pPage = infoNews[iNew - 1].pPage; + int iPage = infoNews[iNew - 1].iPage; int oIdx = infoNews[iNew - 1].oIdx + 1; + SPage *pPage; for (;;) { + pPage = pOlds[iPage]; if (oIdx < TDB_PAGE_TOTAL_CELLS(pPage)) { break; } - pPage++; + iPage++; oIdx = 0; } @@ -646,8 +648,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { break; } - infoNews[iNew - 1].pPage--; - infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(infoNews[iNew - 1].pPage) - 1; + infoNews[iNew - 1].iPage--; + infoNews[iNew - 1].oIdx = TDB_PAGE_TOTAL_CELLS(pOlds[infoNews[iNew - 1].iPage]) - 1; } infoNews[iNew].cnt++; From 27bcf222cec34b67d40b90d33c0d934f6fffb611 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 10:12:43 +0000 Subject: [PATCH 060/121] fix another bug --- source/libs/tdb/src/page/tdbPage.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index ee1a021a1b..5253a1fd67 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -288,8 +288,6 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { } else { TDB_PAGE_FCELL_SET(pPage, pFreeCell - pPage->pData); } - - goto _alloc_finish; } else { if (pPrevFreeCell) { pPage->pPageMethods->setFreeCellInfo(pPrevFreeCell, szPrevFreeCell, nxFreeCell); @@ -297,6 +295,8 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { TDB_PAGE_FCELL_SET(pPage, nxFreeCell); } } + + goto _alloc_finish; } else { pPrevFreeCell = pFreeCell; szPrevFreeCell = szFreeCell; From b633a96bed37bececfe62551f005a79cedd90644 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 10:17:23 +0000 Subject: [PATCH 061/121] more TDB --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 7bdcc38173..63b53b1732 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -8,7 +8,7 @@ TEST(tdb_test, simple_test) { STDb *pDb; // Open Env - ret = tdbEnvOpen("tdb", 1024, 20, &pEnv); + ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From 2aa6d7e38482a76211f265c6edbc31d52ba7e387 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 23 Mar 2022 10:30:52 +0000 Subject: [PATCH 062/121] start the query --- source/libs/tdb/src/db/tdbDb.c | 5 +++++ source/libs/tdb/src/inc/tdbDb.h | 1 + source/libs/tdb/test/tdbTest.cpp | 35 ++++++++++++++++++++++++-------- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 00f38a19bb..27cbaeaa01 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -85,5 +85,10 @@ int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int v return -1; } + return 0; +} + +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen) { + // TODO return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 06ea74a83e..fdf4c4aca5 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -26,6 +26,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF int tdbDbClose(STDb *pDb); int tdbDbDrop(STDb *pDb); int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 63b53b1732..c5a2787aaf 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,6 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; + int nData = 1000; // Open Env ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); @@ -15,15 +16,33 @@ TEST(tdb_test, simple_test) { ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, NULL, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); - { // Insert some data - char key[64]; - char val[64]; + { + char key[64]; + char val[64]; + void *pVal; + int vLen; - for (int i = 1; i <= 1000; i++) { - sprintf(key, "key%d", i); - sprintf(val, "value%d", i); - ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val)); - GTEST_ASSERT_EQ(ret, 0); + { // Insert some data + + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + sprintf(val, "value%d", i); + ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val)); + GTEST_ASSERT_EQ(ret, 0); + } + } + + { // Query the data + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + sprintf(val, "value%d", i); + + ret = tdbDbGet(pDb, key, strlen(key), pVal, &vLen); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, strlen(val)); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + } } } From 4383b3b53a45d597b8c2cbcd8a11df46cf923d55 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 02:09:56 +0000 Subject: [PATCH 063/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 28 ++++++++++++++++++++++++++++ source/libs/tdb/src/db/tdbDb.c | 5 ++--- source/libs/tdb/src/inc/tdbBtree.h | 1 + source/libs/tdb/src/inc/tdbDb.h | 2 +- source/libs/tdb/test/tdbTest.cpp | 2 +- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0f5e6733ed..f5715866ed 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -203,6 +203,34 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p return 0; } +int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) { + SBtCursor btc; + SCell *pCell; + int cret; + SCellDecoder cd; + + tdbBtreeCursor(&btc, pBt); + + tdbBtCursorMoveTo(&btc, pKey, kLen, &cret); + + if (cret) { + return cret; + } + + pCell = tdbPageGetCell(btc.pPage, btc.idx); + tdbBtreeDecodeCell(btc.pPage, pCell, &cd); + + *vLen = cd.vLen; + // TODO: here may have memory leak + *ppVal = realloc(*ppVal, *vLen); + if (*ppVal == NULL) { + return -1; + } + + memcpy(*ppVal, cd.pVal, cd.vLen); + return 0; +} + static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { int ret; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 27cbaeaa01..2ba9daaa4a 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -88,7 +88,6 @@ int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int v return 0; } -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen) { - // TODO - return 0; +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { + return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index c1fe77c22e..a0b054a6a8 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -37,6 +37,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SB int tdbBtreeClose(SBTree *pBt); int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt); int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen); +int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index fdf4c4aca5..6a9d653eac 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -26,7 +26,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF int tdbDbClose(STDb *pDb); int tdbDbDrop(STDb *pDb); int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void *pVal, int *vLen); +int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index c5a2787aaf..3d06d568e4 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -37,7 +37,7 @@ TEST(tdb_test, simple_test) { sprintf(key, "key%d", i); sprintf(val, "value%d", i); - ret = tdbDbGet(pDb, key, strlen(key), pVal, &vLen); + ret = tdbDbGet(pDb, key, strlen(key), &pVal, &vLen); GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(vLen, strlen(val)); From 20b1c0d236103f458d356c84d93497dc3858f2bd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 02:27:16 +0000 Subject: [PATCH 064/121] more TDB --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 3d06d568e4..1acd78f188 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,7 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 1000; + int nData = 10000; // Open Env ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); From 820de153a80e5a95e02d91e4dcdf0068103918a9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 05:46:59 +0000 Subject: [PATCH 065/121] fix invalid read --- source/libs/tdb/src/db/tdbBtree.c | 4 ++-- source/libs/tdb/src/page/tdbPage.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index f5715866ed..a733c0d981 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -712,7 +712,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } { // Do the real cell distribution - SPage *pOldsCopy[3]; + SPage *pOldsCopy[3] = {0}; SCell *pCell; int szCell; SBtreeInitPageArg iarg; @@ -722,7 +722,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { iarg.pBt = pBt; iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); for (int i = 0; i < nOlds; i++) { - tdbPageCreate(pOlds[0]->pageSize, pOldsCopy + i, NULL, NULL); + tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], NULL, NULL); tdbBtreeZeroPage(pOldsCopy[i], &iarg); tdbPageCopy(pOlds[i], pOldsCopy[i]); } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 5253a1fd67..15b6e218e6 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -52,7 +52,7 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) } ptr = (u8 *)((*xMalloc)(arg, size)); - if (pPage == NULL) { + if (ptr == NULL) { return -1; } From d59b65fa54d958b2caee0b207687000051e10fc8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 06:00:35 +0000 Subject: [PATCH 066/121] fix memory bug --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 1acd78f188..653efaa19a 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -19,7 +19,7 @@ TEST(tdb_test, simple_test) { { char key[64]; char val[64]; - void *pVal; + void *pVal = NULL; int vLen; { // Insert some data From 08a270777c72cb90c8453a3d734ee085ba1e0e96 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 06:58:52 +0000 Subject: [PATCH 067/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 48 ++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index a733c0d981..0bb50fe937 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -517,8 +517,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int nOlds; SPage *pOlds[3]; - SCell *pDivCell[2] = {0}; - int szDivCell[2]; + SCell *pDivCell[3] = {0}; + int szDivCell[3]; int sIdx; u8 childNotLeaf; SPgno rPgno; @@ -562,18 +562,18 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { // copy the parent key out if child pages are not leaf page childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); if (childNotLeaf) { - for (int i = 0; i < nOlds - 1; i++) { - pCell = tdbPageGetCell(pParent, sIdx + i); + for (int i = 0; i < nOlds; i++) { + if (sIdx + i < TDB_PAGE_TOTAL_CELLS(pParent)) { + pCell = tdbPageGetCell(pParent, sIdx + i); + szDivCell[i] = tdbBtreeCellSize(pParent, pCell); + pDivCell[i] = malloc(szDivCell[i]); + memcpy(pDivCell[i], pCell, szDivCell[i]); + } - szDivCell[i] = tdbBtreeCellSize(pParent, pCell); - pDivCell[i] = malloc(szDivCell[i]); - memcpy(pDivCell, pCell, szDivCell[i]); - - ((SPgno *)pDivCell)[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; - ((SIntHdr *)pOlds[i]->pData)->pgno = 0; - - // here we insert the cell as an overflow cell to avoid - // the slow defragment process + if (i < nOlds - 1) { + ((SPgno *)pDivCell[i])[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; + ((SIntHdr *)pOlds[i]->pData)->pgno = 0; + } tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno; @@ -740,6 +740,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { szCell = tdbBtreeCellSize(pPage, pCell); ASSERT(nNewCells <= infoNews[iNew].cnt); + ASSERT(iNew < nNews - 1); if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); @@ -771,8 +772,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } else { - ASSERT(0); ASSERT(childNotLeaf); + ASSERT(iNew < nNews - 1); // set current new page right-most child ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; @@ -790,6 +791,19 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } + + if (childNotLeaf) { + ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); + ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; + + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[nNews - 1]); + } else { + ((SPgno *)pDivCell[nOlds - 1])[0] = TDB_PAGE_PGNO(pNews[nNews - 1]); + tdbPageInsertCell(pParent, sIdx, pDivCell[nOlds - 1], szDivCell[nOlds - 1], 0); + } + } } for (int i = 0; i < nOlds; i++) { @@ -797,6 +811,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } + for (int i = 0; i < 3; i++) { + if (pDivCell[i]) { + free(pDivCell[i]); + } + } + return 0; } From b6d27d3759c55168626212c425b72407211e01c3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 07:43:19 +0000 Subject: [PATCH 068/121] fix another bug --- source/libs/tdb/src/db/tdbBtree.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0bb50fe937..de2ed60ec7 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -516,7 +516,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { int ret; int nOlds; - SPage *pOlds[3]; + SPage *pOlds[3] = {0}; SCell *pDivCell[3] = {0}; int szDivCell[3]; int sIdx; @@ -573,8 +573,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { if (i < nOlds - 1) { ((SPgno *)pDivCell[i])[0] = ((SIntHdr *)pOlds[i]->pData)->pgno; ((SIntHdr *)pOlds[i]->pData)->pgno = 0; + tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } - tdbPageInsertCell(pOlds[i], TDB_PAGE_TOTAL_CELLS(pOlds[i]), pDivCell[i], szDivCell[i], 1); } rPgno = ((SIntHdr *)pOlds[nOlds - 1]->pData)->pgno; } @@ -740,7 +740,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { szCell = tdbBtreeCellSize(pPage, pCell); ASSERT(nNewCells <= infoNews[iNew].cnt); - ASSERT(iNew < nNews - 1); + ASSERT(iNew < nNews); if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); @@ -791,18 +791,18 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } } + } - if (childNotLeaf) { - ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); - ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; + if (childNotLeaf) { + ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); + ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; - SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; - if (pIntHdr->pgno == 0) { - pIntHdr->pgno = TDB_PAGE_PGNO(pNews[nNews - 1]); - } else { - ((SPgno *)pDivCell[nOlds - 1])[0] = TDB_PAGE_PGNO(pNews[nNews - 1]); - tdbPageInsertCell(pParent, sIdx, pDivCell[nOlds - 1], szDivCell[nOlds - 1], 0); - } + SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; + if (pIntHdr->pgno == 0) { + pIntHdr->pgno = TDB_PAGE_PGNO(pNews[nNews - 1]); + } else { + ((SPgno *)pDivCell[nOlds - 1])[0] = TDB_PAGE_PGNO(pNews[nNews - 1]); + tdbPageInsertCell(pParent, sIdx, pDivCell[nOlds - 1], szDivCell[nOlds - 1], 0); } } From 493670d2f0ba7c226af5b00b54c8ef4b0447ee0e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 08:11:07 +0000 Subject: [PATCH 069/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index de2ed60ec7..eec75d3a13 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -482,9 +482,11 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { u8 flags; SIntHdr *pIntHdr; SBtreeInitPageArg zArg; + u8 leaf; pPager = pRoot->pPager; flags = TDB_BTREE_PAGE_GET_FLAGS(pRoot); + leaf = TDB_BTREE_PAGE_IS_LEAF(flags); // Allocate a new child page zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT); @@ -494,6 +496,10 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { return -1; } + if (!leaf) { + ((SIntHdr *)pChild->pData)->pgno = ((SIntHdr *)(pRoot->pData))->pgno; + } + // Copy the root page content to the child page tdbPageCopy(pRoot, pChild); From c9fe5301564e14313331b0231b2c344c3961bbad Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 08:16:14 +0000 Subject: [PATCH 070/121] add more test --- source/libs/tdb/test/tdbTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 653efaa19a..5285a5ee94 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 10000; + int nData = 50000; // Open Env - ret = tdbEnvOpen("tdb", 1024, 256, &pEnv); + ret = tdbEnvOpen("tdb", 1024, 2560, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From eb9d3f2196581a1c65534766e582e9dbb7d1e121 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 08:40:37 +0000 Subject: [PATCH 071/121] more TDB --- source/libs/tdb/src/inc/tdbDb.h | 3 ++- source/libs/tdb/test/tdbTest.cpp | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 6a9d653eac..7a65966be3 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -20,7 +20,8 @@ extern "C" { #endif -typedef struct STDb STDb; +typedef struct STDb STDb; +typedef struct STDbC STDbC; int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb); int tdbDbClose(STDb *pDb); diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 5285a5ee94..79532d07b7 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 50000; + int nData = 200000; // Open Env - ret = tdbEnvOpen("tdb", 1024, 2560, &pEnv); + ret = tdbEnvOpen("tdb", 1024, 25600, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database @@ -44,6 +44,9 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); } } + + { // Loop to query the data + } } ret = tdbDbDrop(pDb); From 347d1e6f9932ddbdfb2e2d352808cad6552b97df Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 09:03:01 +0000 Subject: [PATCH 072/121] start more fix --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 79532d07b7..cca2082e53 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,7 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 200000; + int nData = 160511; // Open Env ret = tdbEnvOpen("tdb", 1024, 25600, &pEnv); From 61b2f52dca24dd7a7ba66b93194aabfa87704c57 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 09:53:34 +0000 Subject: [PATCH 073/121] now TDB can write 1M records --- source/libs/tdb/src/db/tdbBtree.c | 11 ++++++----- source/libs/tdb/test/tdbTest.cpp | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index eec75d3a13..e244b77f95 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -817,11 +817,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - for (int i = 0; i < 3; i++) { - if (pDivCell[i]) { - free(pDivCell[i]); - } - } + // TODO: here has memory leak + // for (int i = 0; i < 3; i++) { + // if (pDivCell[i]) { + // free(pDivCell[i]); + // } + // } return 0; } diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index cca2082e53..0759de54b7 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 160511; + int nData = 1000000; // Open Env - ret = tdbEnvOpen("tdb", 1024, 25600, &pEnv); + ret = tdbEnvOpen("tdb", 4096, 25600, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From e20e13d2dbb9b3234db44b6539a918c6016dcd02 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 10:01:17 +0000 Subject: [PATCH 074/121] more TDB --- source/libs/tdb/test/tdbTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 0759de54b7..56284f0271 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,7 +6,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 1000000; + int nData = 2000000; // Open Env ret = tdbEnvOpen("tdb", 4096, 25600, &pEnv); From 488d44a55515152d0982b1a343591343830b4a9c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 24 Mar 2022 10:14:46 +0000 Subject: [PATCH 075/121] w/r 10M K-V pairs --- source/libs/tdb/test/tdbTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 56284f0271..df96a39943 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -6,10 +6,10 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; STDb *pDb; - int nData = 2000000; + int nData = 10000000; // Open Env - ret = tdbEnvOpen("tdb", 4096, 25600, &pEnv); + ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database From b8ac2d30d5b2658d2dc68560938b07ea55c71b5f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 01:42:03 +0000 Subject: [PATCH 076/121] refact --- source/libs/tdb/src/db/tdbBtree.c | 16 ++++++++-------- source/libs/tdb/src/db/tdbDb.c | 6 +++--- source/libs/tdb/src/inc/tdbBtree.h | 15 ++++++++++----- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e244b77f95..befc021d28 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -67,7 +67,7 @@ typedef struct { u8 *pTmpSpace; } SCellDecoder; -static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst); +static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst); static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); static int tdbBtreeOpenImpl(SBTree *pBt); static int tdbBtreeZeroPage(SPage *pPage, void *arg); @@ -75,7 +75,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg); static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell); static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); -static int tdbBtreeBalance(SBtCursor *pCur); +static int tdbBtreeBalance(SBTC *pCur); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { @@ -131,7 +131,7 @@ int tdbBtreeClose(SBTree *pBt) { return 0; } -int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt) { +int tdbBtreeCursor(SBTC *pCur, SBTree *pBt) { pCur->pBt = pBt; pCur->iPage = -1; pCur->pPage = NULL; @@ -140,7 +140,7 @@ int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt) { return 0; } -int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { +int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { int ret; int idx; SPager *pPager; @@ -204,7 +204,7 @@ int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *p } int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen) { - SBtCursor btc; + SBTC btc; SCell *pCell; int cret; SCellDecoder cd; @@ -231,7 +231,7 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen return 0; } -static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { +static int tdbBtCursorMoveToChild(SBTC *pCur, SPgno pgno) { int ret; pCur->pgStack[pCur->iPage] = pCur->pPage; @@ -248,7 +248,7 @@ static int tdbBtCursorMoveToChild(SBtCursor *pCur, SPgno pgno) { return 0; } -static int tdbBtCursorMoveTo(SBtCursor *pCur, const void *pKey, int kLen, int *pCRst) { +static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst) { int ret; SBTree *pBt; SPager *pPager; @@ -827,7 +827,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { return 0; } -static int tdbBtreeBalance(SBtCursor *pCur) { +static int tdbBtreeBalance(SBTC *pCur) { int iPage; SPage *pParent; SPage *pPage; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 2ba9daaa4a..9c5fb551f0 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -70,9 +70,9 @@ int tdbDbDrop(STDb *pDb) { } int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { - SBtCursor btc; - SBtCursor *pCur; - int ret; + SBTC btc; + SBTC *pCur; + int ret; pCur = &btc; ret = tdbBtreeCursor(pCur, pDb->pBt); diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index a0b054a6a8..60078f68f4 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -20,10 +20,15 @@ extern "C" { #endif -typedef struct SBTree SBTree; -typedef struct SBtCursor SBtCursor; +typedef struct SBTree SBTree; +typedef struct SBTC SBTC; +typedef struct SBtInfo { + SPgno root; + int nLevel; + int nData; +} SBtInfo; -struct SBtCursor { +struct SBTC { SBTree *pBt; i8 iPage; SPage *pPage; @@ -35,8 +40,8 @@ struct SBtCursor { int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); -int tdbBtreeCursor(SBtCursor *pCur, SBTree *pBt); -int tdbBtCursorInsert(SBtCursor *pCur, const void *pKey, int kLen, const void *pVal, int vLen); +int tdbBtreeCursor(SBTC *pCur, SBTree *pBt); +int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus From 6276f2c33e84e90d5b427a574b444a4413cfacc9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 01:48:35 +0000 Subject: [PATCH 077/121] refact --- source/libs/tdb/src/db/tdbDb.c | 16 ++++++++-------- source/libs/tdb/src/inc/tdbDb.h | 12 ++++++------ source/libs/tdb/src/inc/tdbInt.h | 2 +- source/libs/tdb/test/tdbTest.cpp | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 9c5fb551f0..e889e8772b 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -15,13 +15,13 @@ #include "tdbInt.h" -struct STDb { +struct STDB { STEnv *pEnv; SBTree *pBt; }; -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb) { - STDb *pDb; +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb) { + STDB *pDb; SPager *pPager; int ret; char fFullName[TDB_FILENAME_LEN]; @@ -30,7 +30,7 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF *ppDb = NULL; - pDb = (STDb *)calloc(1, sizeof(*pDb)); + pDb = (STDB *)calloc(1, sizeof(*pDb)); if (pDb == NULL) { return -1; } @@ -59,17 +59,17 @@ int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprF return 0; } -int tdbDbClose(STDb *pDb) { +int tdbDbClose(STDB *pDb) { // TODO return 0; } -int tdbDbDrop(STDb *pDb) { +int tdbDbDrop(STDB *pDb) { // TODO return 0; } -int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { +int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) { SBTC btc; SBTC *pCur; int ret; @@ -88,6 +88,6 @@ int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int v return 0; } -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { +int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 7a65966be3..e48de5e8c7 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -20,14 +20,14 @@ extern "C" { #endif -typedef struct STDb STDb; +typedef struct STDB STDB; typedef struct STDbC STDbC; -int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDb **ppDb); -int tdbDbClose(STDb *pDb); -int tdbDbDrop(STDb *pDb); -int tdbDbInsert(STDb *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); -int tdbDbGet(STDb *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb); +int tdbDbClose(STDB *pDb); +int tdbDbDrop(STDB *pDb); +int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); +int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 3ebba0a4b5..98845bb66f 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -95,7 +95,7 @@ static FORCE_INLINE int tdbCmprPgId(const void *p1, const void *p2) { // tdb_log #define tdbError(var) -typedef TD_DLIST(STDb) STDbList; +typedef TD_DLIST(STDB) STDbList; typedef TD_DLIST(SPgFile) SPgFileList; typedef TD_DLIST_NODE(SPgFile) SPgFileListNode; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index df96a39943..4f6bde3c96 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -5,7 +5,7 @@ TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; - STDb *pDb; + STDB *pDb; int nData = 10000000; // Open Env From 1cd404151aac734c9fcfcf96e0c04927dde4d0fe Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Mar 2022 11:44:40 +0800 Subject: [PATCH 078/121] minor changes --- source/dnode/mgmt/container/inc/dndInt.h | 1 - source/dnode/mgmt/container/src/dndExec.c | 2 +- source/dnode/mgmt/container/src/dndFile.c | 28 +++++++++---------- source/dnode/mgmt/container/src/dndMsg.c | 11 ++++---- .../dnode/mgmt/container/src/dndTransport.c | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/source/dnode/mgmt/container/inc/dndInt.h b/source/dnode/mgmt/container/inc/dndInt.h index d10835b67f..8ea496b2fb 100644 --- a/source/dnode/mgmt/container/inc/dndInt.h +++ b/source/dnode/mgmt/container/inc/dndInt.h @@ -56,7 +56,6 @@ void dndCleanupServer(SDnode *pDnode); int32_t dndInitClient(SDnode *pDnode); void dndCleanupClient(SDnode *pDnode); int32_t dndInitMsgHandle(SDnode *pDnode); -void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c index 6d07ca1898..1355c4543a 100644 --- a/source/dnode/mgmt/container/src/dndExec.c +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -138,7 +138,7 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t msgLen, void *pCont, int32_t contLen) { dTrace("msg:%p, get from parent queue", pRsp); pRsp->pCont = pCont; - dndSendRpcRsp(pWrapper, pRsp); + dndSendRsp(pWrapper, pRsp); free(pRsp); } diff --git a/source/dnode/mgmt/container/src/dndFile.c b/source/dnode/mgmt/container/src/dndFile.c index a5f8fb7169..ab04040b57 100644 --- a/source/dnode/mgmt/container/src/dndFile.c +++ b/source/dnode/mgmt/container/src/dndFile.c @@ -16,14 +16,16 @@ #define _DEFAULT_SOURCE #include "dndInt.h" +#define MAXLEN 1024 + int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) { - int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); - cJSON *root = NULL; - char file[PATH_MAX]; - TdFilePtr pFile = NULL; + int32_t code = TSDB_CODE_NODE_PARSE_FILE_ERROR; + int32_t len = 0; + const int32_t maxLen = MAXLEN; + char content[MAXLEN + 1] = {0}; + cJSON *root = NULL; + char file[PATH_MAX]; + TdFilePtr pFile = NULL; snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); pFile = taosOpenFile(file, TD_FILE_READ); @@ -57,7 +59,6 @@ int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) { dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); _OVER: - if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); @@ -66,7 +67,7 @@ _OVER: } int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) { - char file[PATH_MAX]; + char file[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); @@ -76,9 +77,9 @@ int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) { return -1; } - int32_t len = 0; - int32_t maxLen = 1024; - char *content = calloc(1, maxLen + 1); + int32_t len = 0; + const int32_t maxLen = MAXLEN; + char content[MAXLEN + 1] = {0}; len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); @@ -87,9 +88,8 @@ int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) { taosWriteFile(pFile, content, len); taosFsyncFile(pFile); taosCloseFile(&pFile); - free(content); - char realfile[PATH_MAX]; + char realfile[PATH_MAX] = {0}; snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name); if (taosRenameFile(file, realfile) != 0) { diff --git a/source/dnode/mgmt/container/src/dndMsg.c b/source/dnode/mgmt/container/src/dndMsg.c index 37ff4ebc05..bf3200ffaf 100644 --- a/source/dnode/mgmt/container/src/dndMsg.c +++ b/source/dnode/mgmt/container/src/dndMsg.c @@ -43,19 +43,18 @@ static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); - return 0; } void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { - if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { - dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); - } - int32_t code = -1; SNodeMsg *pMsg = NULL; NodeMsgFp msgFp = NULL; + if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { + dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); + } + if (dndMarkWrapper(pWrapper) != 0) goto _OVER; if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; @@ -67,6 +66,8 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { } else if (pWrapper->procType == PROC_PARENT) { code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen); } else { + dFatal(" should not be entered in child processes"); + ASSERT(1); } _OVER: diff --git a/source/dnode/mgmt/container/src/dndTransport.c b/source/dnode/mgmt/container/src/dndTransport.c index 4acb1f459e..b7d0cf26c0 100644 --- a/source/dnode/mgmt/container/src/dndTransport.c +++ b/source/dnode/mgmt/container/src/dndTransport.c @@ -348,7 +348,7 @@ int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { } } -void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { +static void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) { if (pRsp->code == TSDB_CODE_APP_NOT_READY) { SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE); if (pDnodeWrapper != NULL) { From 212e24dda649b879decfd7bf20c0bb03c7065475 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 04:59:20 +0000 Subject: [PATCH 079/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 1 + source/libs/tdb/test/tdbTest.cpp | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index befc021d28..c3adf7d2b6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -914,6 +914,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, u8 *pPayload, const void *pKey, i return 0; } +// TODO: allow vLen = 0 static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell) { u8 flags; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 4f6bde3c96..05a0f3a6ae 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -17,10 +17,8 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(ret, 0); { - char key[64]; - char val[64]; - void *pVal = NULL; - int vLen; + char key[64]; + char val[64]; { // Insert some data @@ -33,6 +31,9 @@ TEST(tdb_test, simple_test) { } { // Query the data + void *pVal = NULL; + int vLen; + for (int i = 1; i <= nData; i++) { sprintf(key, "key%d", i); sprintf(val, "value%d", i); @@ -46,6 +47,8 @@ TEST(tdb_test, simple_test) { } { // Loop to query the data + // STDbC *pDBC; + // tdbDBNext(pDBC, p) } } From f4359a6fb3c82295ac5b8b752e2804b7dd19ad84 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 06:02:38 +0000 Subject: [PATCH 080/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 7 ++++--- source/libs/tdb/src/inc/tdbUtil.h | 22 ++++++++++++++++++++++ source/libs/tdb/test/tdbTest.cpp | 2 ++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c3adf7d2b6..0ef5d64d30 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -207,6 +207,7 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen SBTC btc; SCell *pCell; int cret; + void *pVal; SCellDecoder cd; tdbBtreeCursor(&btc, pBt); @@ -221,12 +222,12 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen tdbBtreeDecodeCell(btc.pPage, pCell, &cd); *vLen = cd.vLen; - // TODO: here may have memory leak - *ppVal = realloc(*ppVal, *vLen); - if (*ppVal == NULL) { + pVal = TDB_REALLOC(*ppVal, *vLen); + if (pVal == NULL) { return -1; } + *ppVal = pVal; memcpy(*ppVal, cd.pVal, cd.vLen); return 0; } diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 9358aae236..88fc846bf1 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -39,6 +39,28 @@ int tdbGetFileSize(const char *fname, int pgSize, SPgno *pSize); int tdbPRead(int fd, void *pData, int count, i64 offset); +#define TDB_REALLOC(PTR, SIZE) \ + ({ \ + void *nPtr; \ + if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ + nPtr = realloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \ + if (nPtr) { \ + ((int *)nPtr)[0] = (SIZE); \ + nPtr = (char *)nPtr + sizeof(int); \ + } \ + } else { \ + nPtr = (PTR); \ + } \ + nPtr; \ + }) + +#define TDB_FREE(PTR) \ + do { \ + if (PTR) { \ + free((char *)(PTR) - sizeof(int)); \ + } \ + } while (0) + static inline void *tdbOsMalloc(void *arg, size_t size) { void *ptr; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 05a0f3a6ae..46437392cd 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -44,6 +44,8 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(vLen, strlen(val)); GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); } + + TDB_FREE(pVal); } { // Loop to query the data From 9ffdf09986e1ee511d12b97e4e39531ef63fa9df Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 06:15:21 +0000 Subject: [PATCH 081/121] design dbc api --- source/libs/tdb/src/db/tdbDb.c | 15 +++++++++++++++ source/libs/tdb/src/inc/tdbDb.h | 8 +++++++- source/libs/tdb/test/tdbTest.cpp | 25 ++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index e889e8772b..98f22e4d25 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -90,4 +90,19 @@ int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int v int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); +} + +int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc) { + // TODO + return 0; +} + +int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { + // TODO + return 0; +} + +int tdbDbcClose(STDBC *pDbc) { + // TODO + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index e48de5e8c7..4e69de2bfa 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -21,14 +21,20 @@ extern "C" { #endif typedef struct STDB STDB; -typedef struct STDbC STDbC; +typedef struct STDBC STDBC; +// STDB int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb); int tdbDbClose(STDB *pDb); int tdbDbDrop(STDB *pDb); int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); +// STDBC +int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc); +int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbDbcClose(STDBC *pDbc); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 46437392cd..164b2c0b83 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -48,9 +48,28 @@ TEST(tdb_test, simple_test) { TDB_FREE(pVal); } - { // Loop to query the data - // STDbC *pDBC; - // tdbDBNext(pDBC, p) + { // Iterate to query the DB data + STDBC *pDBC; + void *pKey = NULL; + void *pVal = NULL; + int vLen, kLen; + int count = 0; + + ret = tdbDbcOpen(pDb, &pDBC); + GTEST_ASSERT_EQ(ret, 0); + + for (;;) { + ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + count++; + } + + GTEST_ASSERT_EQ(count, nData); + + tdbDbcClose(pDBC); + + TDB_FREE(pKey); + TDB_FREE(pVal); } } From cc176b1037ebed74221b6f6d99f63775d2f80987 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 06:42:42 +0000 Subject: [PATCH 082/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 29 +++++++++++++++++++++-- source/libs/tdb/src/db/tdbDb.c | 37 +++++++++++++++++++++++++----- source/libs/tdb/src/inc/tdbBtree.h | 10 +++++++- source/libs/tdb/src/inc/tdbDb.h | 2 +- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 0ef5d64d30..2b0ba62ed3 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -131,7 +131,7 @@ int tdbBtreeClose(SBTree *pBt) { return 0; } -int tdbBtreeCursor(SBTC *pCur, SBTree *pBt) { +int tdbBtcOpen(SBTC *pCur, SBTree *pBt) { pCur->pBt = pBt; pCur->iPage = -1; pCur->pPage = NULL; @@ -210,7 +210,7 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen void *pVal; SCellDecoder cd; - tdbBtreeCursor(&btc, pBt); + tdbBtcOpen(&btc, pBt); tdbBtCursorMoveTo(&btc, pKey, kLen, &cret); @@ -1072,3 +1072,28 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { } #endif + +int tdbBtcMoveToFirst(SBTC *pBtc) { + // TODO + return 0; +} + +int tdbBtcMoveToLast(SBTC *pBtc) { + // TODO + return 0; +} + +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen) { + // TODO + return 0; +} + +int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { + // TODO + return 0; +} + +int tdbBtcClose(SBTC *pBtc) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 98f22e4d25..4e74dc4cbb 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -20,6 +20,10 @@ struct STDB { SBTree *pBt; }; +struct STDBC { + SBTC btc; +}; + int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, STEnv *pEnv, STDB **ppDb) { STDB *pDb; SPager *pPager; @@ -75,7 +79,7 @@ int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int v int ret; pCur = &btc; - ret = tdbBtreeCursor(pCur, pDb->pBt); + ret = tdbBtcOpen(pCur, pDb->pBt); if (ret < 0) { return -1; } @@ -92,17 +96,38 @@ int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } -int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc) { - // TODO +int tdbDbcOpen(STDB *pDb, STDBC **ppDbc) { + int ret; + STDBC *pDbc = NULL; + + *ppDbc = NULL; + pDbc = malloc(sizeof(*pDbc)); + if (pDbc == NULL) { + return -1; + } + + tdbBtcOpen(&pDbc->btc, pDb->pBt); + + // TODO: move to first now, we can move to any key-value + // and in any direction, design new APIs. + ret = tdbBtcMoveToFirst(&pDbc->btc); + if (ret < 0) { + ASSERT(0); + return -1; + } + + *ppDbc = pDbc; return 0; } int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) { - // TODO - return 0; + return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen); } int tdbDbcClose(STDBC *pDbc) { - // TODO + if (pDbc) { + free(pDbc); + } + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index 60078f68f4..e46e8bb78b 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -38,12 +38,20 @@ struct SBTC { void *pBuf; }; +// SBTree int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); -int tdbBtreeCursor(SBTC *pCur, SBTree *pBt); int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); +// SBTC +int tdbBtcOpen(SBTC *pCur, SBTree *pBt); +int tdbBtcMoveToFirst(SBTC *pBtc); +int tdbBtcMoveToLast(SBTC *pBtc); +int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen); +int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); +int tdbBtcClose(SBTC *pBtc); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 4e69de2bfa..b96076b826 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -31,7 +31,7 @@ int tdbDbInsert(STDB *pDb, const void *pKey, int keyLen, const void *pVal, int v int tdbDbGet(STDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); // STDBC -int tdbDbcOpen(STDB *pDb, STDBC **ppTDbc); +int tdbDbcOpen(STDB *pDb, STDBC **ppDbc); int tdbDbNext(STDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbDbcClose(STDBC *pDbc); From 07f6d03a5246aaced36ee3272c6852f703779871 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 07:19:48 +0000 Subject: [PATCH 083/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 107 +++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 2b0ba62ed3..7d925d69f4 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -131,15 +131,6 @@ int tdbBtreeClose(SBTree *pBt) { return 0; } -int tdbBtcOpen(SBTC *pCur, SBTree *pBt) { - pCur->pBt = pBt; - pCur->iPage = -1; - pCur->pPage = NULL; - pCur->idx = -1; - - return 0; -} - int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen) { int ret; int idx; @@ -1073,13 +1064,107 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { #endif +int tdbBtcOpen(SBTC *pCur, SBTree *pBt) { + pCur->pBt = pBt; + pCur->iPage = -1; + pCur->pPage = NULL; + pCur->idx = -1; + + return 0; +} + int tdbBtcMoveToFirst(SBTC *pBtc) { - // TODO + int ret; + SBTree *pBt; + SPager *pPager; + u8 flags; + SCell *pCell; + SPgno pgno; + + pBt = pBtc->pBt; + pPager = pBt->pPager; + + if (pBtc->iPage < 0) { + // move a clean cursor + ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->iPage = 0; + pBtc->idx = 0; + } else { + // move from a position + ASSERT(0); + } + + // move downward + for (;;) { + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + + if (TDB_BTREE_PAGE_IS_LEAF(flags)) break; + + pCell = tdbPageGetCell(pBtc->pPage, 0); + pgno = *(SPgno *)pCell; + + ret = tdbBtCursorMoveToChild(pBtc, pgno); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->idx = 0; + } + return 0; } int tdbBtcMoveToLast(SBTC *pBtc) { - // TODO + int ret; + SBTree *pBt; + SPager *pPager; + u8 flags; + SPgno pgno; + + pBt = pBtc->pBt; + pPager = pBt->pPager; + + if (pBtc->iPage < 0) { + // move a clean cursor + ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->iPage = 0; + } else { + // move from a position + ASSERT(0); + } + + // move downward + for (;;) { + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + + if (TDB_BTREE_PAGE_IS_LEAF(flags)) { + // TODO: handle empty case + ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0); + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1; + break; + } else { + pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; + + ret = tdbBtCursorMoveToChild(pBtc, pgno); + if (ret < 0) { + ASSERT(0); + return -1; + } + } + } + return 0; } From f75cb898663afc37a7c45b964d4b16e462c8d663 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 07:37:41 +0000 Subject: [PATCH 084/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 63 ++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 7d925d69f4..8c86c576c2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -77,6 +77,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder); static int tdbBtreeBalance(SBTC *pCur); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); +static int tdbBtcMoveToNext(SBTC *pBtc); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; @@ -1174,7 +1175,67 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen) { } int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { - // TODO + SCell *pCell; + SCellDecoder cd; + void *pKey, *pVal; + int ret; + + if (0) { + // TODO: no valid data current cursor is pointing to + return -1; + } + + pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); + + tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd); + + pKey = TDB_REALLOC(*ppKey, cd.kLen); + if (pKey == NULL) { + return -1; + } + + // TODO: vLen may be zero + pVal = TDB_REALLOC(*ppVal, cd.vLen); + if (pVal == NULL) { + TDB_FREE(pKey); + return -1; + } + + *ppKey = pKey; + *ppVal = pVal; + + *kLen = cd.kLen; + *vLen = cd.vLen; + + memcpy(pKey, cd.pKey, cd.kLen); + memcpy(pVal, cd.pVal, cd.vLen); + + ret = tdbBtcMoveToNext(pBtc); + + return 0; +} + +static int tdbBtcMoveToNext(SBTC *pBtc) { + pBtc->idx++; + + if (pBtc->idx < 0) return -1; + + if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { + return 0; + } + + // Move upward + for (;;) { + // TODO: release the page + + /* code */ + } + + // Move downward + for (;;) { + /* code */ + } + return 0; } From af98aa400509287f1f196754f69186bbd0bf582b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 07:48:55 +0000 Subject: [PATCH 085/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 51 ++++++++++++++++------------- source/libs/tdb/src/db/tdbPCache.c | 47 ++++++++++---------------- source/libs/tdb/src/db/tdbPager.c | 4 +++ source/libs/tdb/src/inc/tdbPCache.h | 25 +++++++------- source/libs/tdb/src/inc/tdbPager.h | 19 ++++++----- 5 files changed, 73 insertions(+), 73 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8c86c576c2..282c5a98a6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -78,6 +78,8 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD static int tdbBtreeBalance(SBTC *pCur); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); static int tdbBtcMoveToNext(SBTC *pBtc); +static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno); +static int tdbBtcMoveUpward(SBTC *pBtc); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { SBTree *pBt; @@ -224,23 +226,6 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen return 0; } -static int tdbBtCursorMoveToChild(SBTC *pCur, SPgno pgno) { - int ret; - - pCur->pgStack[pCur->iPage] = pCur->pPage; - pCur->idxStack[pCur->iPage] = pCur->idx; - pCur->iPage++; - pCur->pPage = NULL; - pCur->idx = -1; - - ret = tdbPagerFetchPage(pCur->pBt->pPager, pgno, &pCur->pPage, tdbBtreeInitPage, pCur->pBt); - if (ret < 0) { - ASSERT(0); - } - - return 0; -} - static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst) { int ret; SBTree *pBt; @@ -318,16 +303,16 @@ static int tdbBtCursorMoveTo(SBTC *pCur, const void *pKey, int kLen, int *pCRst) } else { if (c <= 0) { pCur->idx = midx; - tdbBtCursorMoveToChild(pCur, cd.pgno); + tdbBtcMoveDownward(pCur, cd.pgno); } else { pCur->idx = midx + 1; if (midx == nCells - 1) { /* Move to right-most child */ - tdbBtCursorMoveToChild(pCur, ((SIntHdr *)pCur->pPage->pData)->pgno); + tdbBtcMoveDownward(pCur, ((SIntHdr *)pCur->pPage->pData)->pgno); } else { pCell = tdbPageGetCell(pPage, pCur->idx); tdbBtreeDecodeCell(pPage, pCell, &cd); - tdbBtCursorMoveToChild(pCur, cd.pgno); + tdbBtcMoveDownward(pCur, cd.pgno); } } } @@ -1109,7 +1094,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { pCell = tdbPageGetCell(pBtc->pPage, 0); pgno = *(SPgno *)pCell; - ret = tdbBtCursorMoveToChild(pBtc, pgno); + ret = tdbBtcMoveDownward(pBtc, pgno); if (ret < 0) { ASSERT(0); return -1; @@ -1158,7 +1143,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; - ret = tdbBtCursorMoveToChild(pBtc, pgno); + ret = tdbBtcMoveDownward(pBtc, pgno); if (ret < 0) { ASSERT(0); return -1; @@ -1242,4 +1227,26 @@ static int tdbBtcMoveToNext(SBTC *pBtc) { int tdbBtcClose(SBTC *pBtc) { // TODO return 0; +} + +static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno) { + int ret; + + pCur->pgStack[pCur->iPage] = pCur->pPage; + pCur->idxStack[pCur->iPage] = pCur->idx; + pCur->iPage++; + pCur->pPage = NULL; + pCur->idx = -1; + + ret = tdbPagerFetchPage(pCur->pBt->pPager, pgno, &pCur->pPage, tdbBtreeInitPage, pCur->pBt); + if (ret < 0) { + ASSERT(0); + } + + return 0; +} + +static int tdbBtcMoveUpward(SBTC *pBtc) { + // TODO + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 1fc0847b38..3c7d037faa 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -53,10 +53,10 @@ static void tdbPCacheLock(SPCache *pCache); static void tdbPCacheUnlock(SPCache *pCache); static bool tdbPCacheLocked(SPCache *pCache); static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); -static void tdbPCachePinPage(SPage *pPage); -static void tdbPCacheRemovePageFromHash(SPage *pPage); -static void tdbPCacheAddPageToHash(SPage *pPage); -static void tdbPCacheUnpinPage(SPage *pPage); +static void tdbPCachePinPage(SPCache *pCache, SPage *pPage); +static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage); +static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage); +static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage); int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) { SPCache *pCache; @@ -100,7 +100,7 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) { return pPage; } -void tdbPCacheRelease(SPage *pPage) { +void tdbPCacheRelease(SPCache *pCache, SPage *pPage) { i32 nRef; nRef = TDB_UNREF_PAGE(pPage); @@ -108,7 +108,7 @@ void tdbPCacheRelease(SPage *pPage) { if (nRef == 0) { if (1 /*TODO: page still clean*/) { - tdbPCacheUnpinPage(pPage); + tdbPCacheUnpinPage(pCache, pPage); } else { // TODO ASSERT(0); @@ -142,7 +142,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe if (pPage || !alcNewPage) { if (pPage) { - tdbPCachePinPage(pPage); + tdbPCachePinPage(pCache, pPage); } return pPage; } @@ -158,8 +158,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe // 3. Try to Recycle a page if (!pPage && !pCache->lru.pLruPrev->isAnchor) { pPage = pCache->lru.pLruPrev; - tdbPCacheRemovePageFromHash(pPage); - tdbPCachePinPage(pPage); + tdbPCacheRemovePageFromHash(pCache, pPage); + tdbPCachePinPage(pCache, pPage); } // 4. Try a stress allocation (TODO) @@ -171,16 +171,13 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid)); pPage->pLruNext = NULL; pPage->pPager = NULL; - tdbPCacheAddPageToHash(pPage); + tdbPCacheAddPageToHash(pCache, pPage); } return pPage; } -static void tdbPCachePinPage(SPage *pPage) { - SPCache *pCache; - - pCache = pPage->pCache; +static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { if (!PAGE_IS_PINNED(pPage)) { pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; @@ -190,11 +187,8 @@ static void tdbPCachePinPage(SPage *pPage) { } } -static void tdbPCacheUnpinPage(SPage *pPage) { - SPCache *pCache; - i32 nRef; - - pCache = pPage->pCache; +static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { + i32 nRef; tdbPCacheLock(pCache); @@ -215,12 +209,10 @@ static void tdbPCacheUnpinPage(SPage *pPage) { tdbPCacheUnlock(pCache); } -static void tdbPCacheRemovePageFromHash(SPage *pPage) { - SPCache *pCache; - SPage **ppPage; - int h; +static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { + SPage **ppPage; + int h; - pCache = pPage->pCache; h = PCACHE_PAGE_HASH(&(pPage->pgid)); for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) ; @@ -230,11 +222,9 @@ static void tdbPCacheRemovePageFromHash(SPage *pPage) { pCache->nPage--; } -static void tdbPCacheAddPageToHash(SPage *pPage) { - SPCache *pCache; - int h; +static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { + int h; - pCache = pPage->pCache; h = PCACHE_PAGE_HASH(&(pPage->pgid)) % pCache->nHash; pPage->pHashNext = pCache->pgHash[h]; @@ -264,7 +254,6 @@ static int tdbPCacheOpenImpl(SPCache *pCache) { // pPage->pgid = 0; pPage->isAnchor = 0; pPage->isLocalPage = 1; - pPage->pCache = pCache; TDB_INIT_PAGE_REF(pPage); pPage->pHashNext = NULL; pPage->pLruNext = NULL; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index ac0bc15e1d..fe4b9aa123 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -255,6 +255,10 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage return 0; } +void tdbPagerReturnPage(SPager *pPager, SPage *pPage) { + tdbPCacheRelease(pPager->pCache, pPage); +} + static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) { // TODO: Allocate a page from the free list return 0; diff --git a/source/libs/tdb/src/inc/tdbPCache.h b/source/libs/tdb/src/inc/tdbPCache.h index ff4f1acbb6..c7fa155615 100644 --- a/source/libs/tdb/src/inc/tdbPCache.h +++ b/source/libs/tdb/src/inc/tdbPCache.h @@ -21,23 +21,22 @@ extern "C" { #endif #define TDB_PCACHE_PAGE \ - u8 isAnchor; \ - u8 isLocalPage; \ - u8 isDirty; \ - i32 nRef; \ - SPCache *pCache; \ - SPage *pFreeNext; \ - SPage *pHashNext; \ - SPage *pLruNext; \ - SPage *pLruPrev; \ - SPage *pDirtyNext; \ - SPager *pPager; \ - SPgid pgid; + u8 isAnchor; \ + u8 isLocalPage; \ + u8 isDirty; \ + i32 nRef; \ + SPage *pFreeNext; \ + SPage *pHashNext; \ + SPage *pLruNext; \ + SPage *pLruPrev; \ + SPage *pDirtyNext; \ + SPager *pPager; \ + SPgid pgid; int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache); int tdbPCacheClose(SPCache *pCache); SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage); -void tdbPCacheRelease(SPage *pPage); +void tdbPCacheRelease(SPCache *pCache, SPage *pPage); int tdbPCacheGetPageSize(SPCache *pCache); #ifdef __cplusplus diff --git a/source/libs/tdb/src/inc/tdbPager.h b/source/libs/tdb/src/inc/tdbPager.h index e4ed8552fd..f4cc822f27 100644 --- a/source/libs/tdb/src/inc/tdbPager.h +++ b/source/libs/tdb/src/inc/tdbPager.h @@ -20,15 +20,16 @@ extern "C" { #endif -int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); -int tdbPagerClose(SPager *pPager); -int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); -int tdbPagerWrite(SPager *pPager, SPage *pPage); -int tdbPagerBegin(SPager *pPager); -int tdbPagerCommit(SPager *pPager); -int tdbPagerGetPageSize(SPager *pPager); -int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); -int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager); +int tdbPagerClose(SPager *pPager); +int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate); +int tdbPagerWrite(SPager *pPager, SPage *pPage); +int tdbPagerBegin(SPager *pPager); +int tdbPagerCommit(SPager *pPager); +int tdbPagerGetPageSize(SPager *pPager); +int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg); +void tdbPagerReturnPage(SPager *pPager, SPage *pPage); #ifdef __cplusplus } From b6c6a11972d01c80c8d57a77b8f5e7610022885f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Mar 2022 16:10:01 +0800 Subject: [PATCH 086/121] minor changes --- source/dnode/mgmt/container/src/dndMsg.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/container/src/dndMsg.c b/source/dnode/mgmt/container/src/dndMsg.c index bf3200ffaf..b72d085861 100644 --- a/source/dnode/mgmt/container/src/dndMsg.c +++ b/source/dnode/mgmt/container/src/dndMsg.c @@ -60,20 +60,23 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; - dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); if (pWrapper->procType == PROC_SINGLE) { + dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); code = (*msgFp)(pWrapper->pMgmt, pMsg); } else if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is created and will put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, + pRpc->ahandle, pMsg->user); code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen); } else { - dFatal(" should not be entered in child processes"); + dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, + pMsg->user); ASSERT(1); } _OVER: if (code == 0) { if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is freed", pMsg); + dTrace("msg:%p, is freed in parent process", pMsg); taosFreeQitem(pMsg); rpcFreeCont(pRpc->pCont); } From cbc8720d293a05ecb3fec048c45bd62d8f470b72 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 08:35:40 +0000 Subject: [PATCH 087/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 51 ++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 282c5a98a6..d135eedb7c 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1201,24 +1201,58 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { } static int tdbBtcMoveToNext(SBTC *pBtc) { - pBtc->idx++; + int nCells; + SPgno pgno; + SCell *pCell; + u8 flags; + + ASSERT(TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage))); if (pBtc->idx < 0) return -1; + pBtc->idx++; if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { return 0; } + if (pBtc->iPage == 0) { + pBtc->idx = -1; + return 0; + } + // Move upward for (;;) { - // TODO: release the page + tdbBtcMoveUpward(pBtc); + pBtc->idx++; - /* code */ + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + if (pBtc->idx <= nCells) { + break; + } + + if (pBtc->iPage == 0) { + pBtc->idx = -1; + return 0; + } } // Move downward for (;;) { - /* code */ + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); + if (pBtc->idx < nCells) { + pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); + pgno = *(SPgno *)pCell; + } else { + pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; + } + + tdbBtcMoveDownward(pBtc, pgno); + pBtc->idx = 0; + + flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + if (TDB_BTREE_PAGE_IS_LEAF(flags)) { + break; + } } return 0; @@ -1247,6 +1281,13 @@ static int tdbBtcMoveDownward(SBTC *pCur, SPgno pgno) { } static int tdbBtcMoveUpward(SBTC *pBtc) { - // TODO + if (pBtc->iPage == 0) return -1; + + // tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage); + + pBtc->iPage--; + pBtc->pPage = pBtc->pgStack[pBtc->iPage]; + pBtc->idx = pBtc->idxStack[pBtc->iPage]; + return 0; } \ No newline at end of file From a9acfce8860d6174a734262e010f8108b469b406 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 08:41:00 +0000 Subject: [PATCH 088/121] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index d135eedb7c..de77fb2649 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1165,8 +1165,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { void *pKey, *pVal; int ret; - if (0) { - // TODO: no valid data current cursor is pointing to + if (pBtc->idx < 0) { return -1; } From f47953797845ae9d17548ef457cee4cb40b3e57f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 09:01:09 +0000 Subject: [PATCH 089/121] more --- source/libs/tdb/test/tdbTest.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 164b2c0b83..b9ae2dff7d 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -2,6 +2,8 @@ #include "tdbInt.h" +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); + TEST(tdb_test, simple_test) { int ret; STEnv *pEnv; @@ -61,6 +63,11 @@ TEST(tdb_test, simple_test) { for (;;) { ret = tdbDbNext(pDBC, &pKey, &kLen, &pVal, &vLen); if (ret < 0) break; + + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; + count++; } @@ -82,4 +89,19 @@ TEST(tdb_test, simple_test) { // Close Env ret = tdbEnvClose(pEnv); GTEST_ASSERT_EQ(ret, 0); +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + k1 = std::strtol((char *)pKey1 + 3, nullptr, kLen1 - 3); + k2 = std::strtol((char *)pKey2 + 3, nullptr, kLen2 - 3); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } } \ No newline at end of file From 4e5df8e3c3369dce2e7cf33688d0fedc1dec6d64 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 25 Mar 2022 05:15:45 -0400 Subject: [PATCH 090/121] bugfix --- source/libs/executor/src/executorimpl.c | 2 +- source/libs/planner/src/planPhysiCreater.c | 322 +++++++++++++-------- 2 files changed, 202 insertions(+), 122 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 177d09be76..2db9406418 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5569,7 +5569,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo *pOperator, bool* newgroup) { blockDataCleanup(pInfo->pRes); - int32_t tableNameSlotId = 1; + int32_t tableNameSlotId = 0; SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId); char * name = NULL; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index fb9a2fc532..c3ad61dd37 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -41,63 +41,147 @@ static int32_t getSlotKey(SNode* pNode, char* pKey) { return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); } -static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) { +static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId, bool output) { SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC); - CHECK_ALLOC(pSlot, NULL); + if (NULL == pSlot) { + return NULL; + } pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = false; - pSlot->output = true; + pSlot->output = output; return (SNode*)pSlot; } -static SNode* createTarget(SNode* pNode, int16_t dataBlockId, int16_t slotId) { +static int32_t createTarget(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode** pOutput) { STargetNode* pTarget = (STargetNode*)nodesMakeNode(QUERY_NODE_TARGET); if (NULL == pTarget) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } + pTarget->dataBlockId = dataBlockId; pTarget->slotId = slotId; pTarget->pExpr = pNode; - return (SNode*)pTarget; + + *pOutput = (SNode*)pTarget; + return TSDB_CODE_SUCCESS; } -static int32_t addDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - SHashObj* pHash = NULL; - if (NULL == pDataBlockDesc->pSlots) { - pDataBlockDesc->pSlots = nodesMakeList(); - CHECK_ALLOC(pDataBlockDesc->pSlots, TSDB_CODE_OUT_OF_MEMORY); +static int32_t putSlotToHashImpl(int16_t dataBlockId, int16_t slotId, const char* pName, int32_t len, SHashObj* pHash) { + SSlotIndex index = { .dataBlockId = dataBlockId, .slotId = slotId }; + return taosHashPut(pHash, pName, len, &index, sizeof(SSlotIndex)); +} - pHash = taosHashInit(LIST_LENGTH(pList), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - CHECK_ALLOC(pHash, TSDB_CODE_OUT_OF_MEMORY); - if (NULL == taosArrayInsert(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId, &pHash)) { - taosHashCleanup(pHash); - return TSDB_CODE_OUT_OF_MEMORY; - } - } else { - pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); +static int32_t putSlotToHash(int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) { + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + int32_t len = getSlotKey(pNode, name); + return putSlotToHashImpl(dataBlockId, slotId, name, len, pHash); +} + +static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId, SHashObj** pDescHash) { + SHashObj* pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == pHash) { + return TSDB_CODE_OUT_OF_MEMORY; } - - SNode* pNode = NULL; - int16_t slotId = taosHashGetSize(pHash); - FOREACH(pNode, pList) { - CHECK_CODE_EXT(nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId))); - - SSlotIndex index = { .dataBlockId = pDataBlockDesc->dataBlockId, .slotId = slotId }; - char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - int32_t len = getSlotKey(pNode, name); - CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY); - - SNode* pTarget = createTarget(pNode, pDataBlockDesc->dataBlockId, slotId); - CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); - REPLACE_NODE(pTarget); - - pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes; - ++slotId; + if (NULL == taosArrayInsert(pCxt->pLocationHelper, dataBlockId, &pHash)) { + taosHashCleanup(pHash); + return TSDB_CODE_OUT_OF_MEMORY; } + + *pDescHash = pHash; return TSDB_CODE_SUCCESS; } +static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, SHashObj* pHash) { + pDataBlockDesc->pSlots = nodesMakeList(); + if (NULL == pDataBlockDesc->pSlots) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + int16_t slotId = 0; + SNode* pNode = NULL; + FOREACH(pNode, pList) { + code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId, true)); + if (TSDB_CODE_SUCCESS == code) { + code = putSlotToHash(pDataBlockDesc->dataBlockId, slotId, pNode, pHash); + } + if (TSDB_CODE_SUCCESS == code) { + pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes; + ++slotId; + } else { + break; + } + } + return code; +} + +static int32_t createDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode** pDataBlockDesc) { + SDataBlockDescNode* pDesc = nodesMakeNode(QUERY_NODE_DATABLOCK_DESC); + if (NULL == pDesc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pDesc->dataBlockId = pCxt->nextDataBlockId++; + + SHashObj* pHash = NULL; + int32_t code = createDataBlockDescHash(pCxt, LIST_LENGTH(pList), pDesc->dataBlockId, &pHash); + if (TSDB_CODE_SUCCESS == code) { + code = buildDataBlockSlots(pCxt, pList, pDesc, pHash); + } + + if (TSDB_CODE_SUCCESS == code) { + *pDataBlockDesc = pDesc; + } else { + nodesDestroyNode(pDesc); + } + + return code; +} + +static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, bool output) { + int32_t code = TSDB_CODE_SUCCESS; + SHashObj* pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); + int16_t nextSlotId = taosHashGetSize(pHash), slotId = 0; + SNode* pNode = NULL; + FOREACH(pNode, pList) { + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN] = {0}; + int32_t len = getSlotKey(pNode, name); + SSlotIndex* pIndex = taosHashGet(pHash, name, len); + if (NULL == pIndex) { + code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, nextSlotId, output)); + if (TSDB_CODE_SUCCESS == code) { + code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash); + } + pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes; + slotId = nextSlotId; + ++nextSlotId; + } else { + slotId = pIndex->slotId; + } + + if (TSDB_CODE_SUCCESS == code) { + SNode* pTarget = NULL; + code = createTarget(pNode, pDataBlockDesc->dataBlockId, slotId, &pTarget); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(pTarget); + } + } + + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static int32_t addDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, false); +} + +static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, false); +} + typedef struct SSetSlotIdCxt { int32_t errCode; SHashObj* pLeftHash; @@ -114,10 +198,11 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { pIndex = taosHashGet(pCxt->pRightHash, name, len); } // pIndex is definitely not NULL, otherwise it is a bug - CHECK_ALLOC(pIndex, DEAL_RES_ERROR); + if (NULL == pIndex) { + return DEAL_RES_ERROR; + } ((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId; ((SColumnNode*)pNode)->slotId = pIndex->slotId; - CHECK_ALLOC(pNode, DEAL_RES_ERROR); return DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; @@ -144,7 +229,7 @@ static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i return TSDB_CODE_SUCCESS; } -static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNodeList* pList, SNodeList** pOutput) { +static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, const SNodeList* pList, SNodeList** pOutput) { SNodeList* pRes = nodesCloneList(pList); if (NULL == pRes) { return TSDB_CODE_OUT_OF_MEMORY; @@ -164,18 +249,17 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i return TSDB_CODE_SUCCESS; } -static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) { +static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, ENodeType type) { SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); if (NULL == pPhysiNode) { return NULL; } - pPhysiNode->pOutputDataBlockDesc = nodesMakeNode(QUERY_NODE_DATABLOCK_DESC); - if (NULL == pPhysiNode->pOutputDataBlockDesc) { + + int32_t code = createDataBlockDesc(pCxt, pLogicNode->pTargets, &pPhysiNode->pOutputDataBlockDesc); + if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode(pPhysiNode); return NULL; } - pPhysiNode->pOutputDataBlockDesc->dataBlockId = pCxt->nextDataBlockId++; - pPhysiNode->pOutputDataBlockDesc->type = QUERY_NODE_DATABLOCK_DESC; return pPhysiNode; } @@ -186,24 +270,11 @@ static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pL return TSDB_CODE_SUCCESS; } -static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, SDataBlockDescNode* pDataBlockDesc) { - SHashObj* pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); - char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - SNode* pNode; - FOREACH(pNode, pTargets) { - int32_t len = getSlotKey(pNode, name); - SSlotIndex* pIndex = taosHashGet(pHash, name, len); - // pIndex is definitely not NULL, otherwise it is a bug - CHECK_ALLOC(pIndex, TSDB_CODE_FAILED); - ((SSlotDescNode*)nodesListGetNode(pDataBlockDesc->pSlots, pIndex->slotId))->output = true; - } - - return TSDB_CODE_SUCCESS; -} - static SNodeptr createPrimaryKeyCol(SPhysiPlanContext* pCxt, uint64_t tableId) { SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); - CHECK_ALLOC(pCol, NULL); + if (NULL == pCol) { + return NULL; + } pCol->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; pCol->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; pCol->tableId = tableId; @@ -244,8 +315,12 @@ static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhys if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanPhysiNode) || QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN == nodeType(pScanPhysiNode)) { pScanPhysiNode->pScanCols = nodesMakeList(); - CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))); + if (NULL == pScanPhysiNode->pScanCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))) { + return TSDB_CODE_OUT_OF_MEMORY; + } SNode* pNode; FOREACH(pNode, pScanCols) { @@ -255,29 +330,29 @@ static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhys strcpy(pCol->colName, ((SColumnNode*)pNode)->colName); continue; } - CHECK_CODE_EXT(nodesListStrictAppend(pScanPhysiNode->pScanCols, nodesCloneNode(pNode))); + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pScanPhysiNode->pScanCols, nodesCloneNode(pNode))) { + return TSDB_CODE_OUT_OF_MEMORY; + } } } else { pScanPhysiNode->pScanCols = nodesCloneList(pScanCols); - CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pScanPhysiNode->pScanCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - // return sortScanCols(pScanPhysiNode->pScanCols); - return TSDB_CODE_SUCCESS; + return sortScanCols(pScanPhysiNode->pScanCols); } static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode, SPhysiNode** pPhyNode) { int32_t code = createScanCols(pCxt, pScanPhysiNode, pScanLogicNode->pScanCols); if (TSDB_CODE_SUCCESS == code) { // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t - code = addDataBlockDesc(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); + code = addDataBlockSlots(pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode); } - if (TSDB_CODE_SUCCESS == code) { - code = setSlotOutput(pCxt, pScanLogicNode->node.pTargets, pScanPhysiNode->node.pOutputDataBlockDesc); - } if (TSDB_CODE_SUCCESS == code) { pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; @@ -302,7 +377,7 @@ static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAdd } static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); + STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); if (NULL == pTagScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -310,7 +385,7 @@ static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* p } static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); if (NULL == pTableScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -326,7 +401,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp } static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - SSystemTableScanPhysiNode* pScan = (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN); + SSystemTableScanPhysiNode* pScan = (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -348,7 +423,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* } static int32_t createStreamScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -412,7 +487,7 @@ static int32_t createJoinOutputCols(SPhysiPlanContext* pCxt, SDataBlockDescNode* } static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode, SPhysiNode** pPhyNode) { - SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN); + SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pJoinLogicNode, QUERY_NODE_PHYSICAL_PLAN_JOIN); if (NULL == pJoin) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -425,14 +500,11 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren code = createJoinOutputCols(pCxt, pLeftDesc, pRightDesc, &pJoin->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc); + code = addDataBlockSlots(pCxt, pJoin->pTargets, pJoin->node.pOutputDataBlockDesc); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); } - if (TSDB_CODE_SUCCESS == code) { - code = setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, pJoin->node.pOutputDataBlockDesc); - } if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pJoin; @@ -452,7 +524,9 @@ typedef struct SRewritePrecalcExprsCxt { static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) { SNode* pExpr = nodesCloneNode(*pNode); - CHECK_ALLOC(pExpr, DEAL_RES_ERROR); + if (NULL == pExpr) { + return DEAL_RES_ERROR; + } if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) { nodesDestroyNode(pExpr); return DEAL_RES_ERROR; @@ -500,11 +574,15 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN if (NULL == *pPrecalcExprs) { *pPrecalcExprs = nodesMakeList(); - CHECK_ALLOC(*pPrecalcExprs, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == *pPrecalcExprs) { + return TSDB_CODE_OUT_OF_MEMORY; + } } if (NULL == *pRewrittenList) { *pRewrittenList = nodesMakeList(); - CHECK_ALLOC(*pRewrittenList, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == *pRewrittenList) { + return TSDB_CODE_OUT_OF_MEMORY; + } } SNode* pNode = NULL; FOREACH(pNode, pList) { @@ -514,8 +592,12 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN } else { pNew = nodesCloneNode(pNode); } - CHECK_ALLOC(pNew, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE(nodesListAppend(*pRewrittenList, pNew), TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pNew) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS != nodesListAppend(*pRewrittenList, pNew)) { + return TSDB_CODE_OUT_OF_MEMORY; + } } SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs }; nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt); @@ -527,7 +609,7 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN } static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode, SPhysiNode** pPhyNode) { - SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_AGG); + SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pAggLogicNode, QUERY_NODE_PHYSICAL_PLAN_AGG); if (NULL == pAgg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -545,30 +627,27 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pAgg->pExprs); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pAgg->pExprs, pChildTupe); + code = pushdownDataBlockSlots(pCxt, pAgg->pExprs, pChildTupe); } } if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pGroupKeys, &pAgg->pGroupKeys); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pAgg->pGroupKeys, pAgg->node.pOutputDataBlockDesc); + code = addDataBlockSlots(pCxt, pAgg->pGroupKeys, pAgg->node.pOutputDataBlockDesc); } } if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncs) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pAggFuncs, &pAgg->pAggFuncs); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pAgg->pAggFuncs, pAgg->node.pOutputDataBlockDesc); + code = addDataBlockSlots(pCxt, pAgg->pAggFuncs, pAgg->node.pOutputDataBlockDesc); } } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg); } - if (TSDB_CODE_SUCCESS == code) { - code = setSlotOutput(pCxt, pAggLogicNode->node.pTargets, pAgg->node.pOutputDataBlockDesc); - } if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pAgg; @@ -576,18 +655,22 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, nodesDestroyNode(pAgg); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pGroupKeys); + nodesDestroyList(pAggFuncs); + return code; } static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) { - SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT); + SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pProjectLogicNode, QUERY_NODE_PHYSICAL_PLAN_PROJECT); if (NULL == pProject) { return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, pProjectLogicNode->pProjections, &pProject->pProjections); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pProject->pProjections, pProject->node.pOutputDataBlockDesc); + code = addDataBlockSlots(pCxt, pProject->pProjections, pProject->node.pOutputDataBlockDesc); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject); @@ -603,34 +686,30 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild } static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { - SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); + SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; } pExchange->srcGroupId = pExchangeLogicNode->srcGroupId; - int32_t code = addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pExchange->node.pOutputDataBlockDesc); + *pPhyNode = (SPhysiNode*)pExchange; - if (TSDB_CODE_SUCCESS == code) { - *pPhyNode = (SPhysiNode*)pExchange; - } else { - nodesDestroyNode(pExchange); - } - - return code; + return TSDB_CODE_SUCCESS; } static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { - SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + SStreamScanPhysiNode* pScan = (SStreamScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } - int32_t code = addDataBlockDesc(pCxt, pExchangeLogicNode->node.pTargets, pScan->node.pOutputDataBlockDesc); + int32_t code = TSDB_CODE_SUCCESS; + + pScan->pScanCols = nodesCloneList(pExchangeLogicNode->node.pTargets); + if (NULL == pScan->pScanCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } if (TSDB_CODE_SUCCESS == code) { - pScan->pScanCols = nodesCloneList(pExchangeLogicNode->node.pTargets); - if (NULL == pScan->pScanCols) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + code = addDataBlockSlots(pCxt, pScan->pScanCols, pScan->node.pOutputDataBlockDesc); } if (TSDB_CODE_SUCCESS == code) { @@ -660,21 +739,17 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pWindow->pExprs); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pWindow->pExprs, pChildTupe); + code = addDataBlockSlots(pCxt, pWindow->pExprs, pChildTupe); } } if (TSDB_CODE_SUCCESS == code && NULL != pFuncs) { code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pFuncs, &pWindow->pFuncs); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockDesc(pCxt, pWindow->pFuncs, pWindow->node.pOutputDataBlockDesc); + code = addDataBlockSlots(pCxt, pWindow->pFuncs, pWindow->node.pOutputDataBlockDesc); } } - if (TSDB_CODE_SUCCESS == code) { - code = setSlotOutput(pCxt, pWindowLogicNode->node.pTargets, pWindow->node.pOutputDataBlockDesc); - } - if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pWindow; } else { @@ -685,7 +760,7 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* } static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { - SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_INTERVAL); + SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pWindowLogicNode, QUERY_NODE_PHYSICAL_PLAN_INTERVAL); if (NULL == pInterval) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -706,7 +781,7 @@ static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil } static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { - SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW); + SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pWindowLogicNode, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW); if (NULL == pSession) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -875,17 +950,22 @@ static int32_t pushSubplan(SPhysiPlanContext* pCxt, SNodeptr pSubplan, int32_t l SNodeListNode* pGroup; if (level >= LIST_LENGTH(pSubplans)) { pGroup = nodesMakeNode(QUERY_NODE_NODE_LIST); - CHECK_ALLOC(pGroup, TSDB_CODE_OUT_OF_MEMORY); - CHECK_CODE(nodesListStrictAppend(pSubplans, pGroup), TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pGroup) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pSubplans, pGroup)) { + return TSDB_CODE_OUT_OF_MEMORY; + } } else { pGroup = nodesListGetNode(pSubplans, level); } if (NULL == pGroup->pNodeList) { pGroup->pNodeList = nodesMakeList(); - CHECK_ALLOC(pGroup->pNodeList, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pGroup->pNodeList) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - CHECK_CODE(nodesListStrictAppend(pGroup->pNodeList, pSubplan), TSDB_CODE_OUT_OF_MEMORY); - return TSDB_CODE_SUCCESS; + return nodesListStrictAppend(pGroup->pNodeList, pSubplan); } static int32_t buildPhysiPlan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pParent, SQueryPlan* pQueryPlan) { From 474939ee4976ec4f8ec538d124363fd0e8cec61b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Mar 2022 17:28:38 +0800 Subject: [PATCH 091/121] shm --- source/dnode/mgmt/container/src/dndExec.c | 1 - source/util/src/tprocess.c | 78 +++++++++++------------ 2 files changed, 38 insertions(+), 41 deletions(-) diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c index 1355c4543a..4dcf7e80ec 100644 --- a/source/dnode/mgmt/container/src/dndExec.c +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -178,7 +178,6 @@ static int32_t dndRunInMultiProcess(SDnode *pDnode) { .parentFreeHeadFp = (ProcFreeFp)free, .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .testFlag = 0, .pParent = pWrapper, .name = pWrapper->name}; SProcObj *pProc = taosProcInit(&cfg); diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index f19a17fdb6..24f3f8a6e4 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -56,7 +56,6 @@ typedef struct SProcObj { int32_t pid; bool isChild; bool stopFlag; - bool testFlag; } SProcObj; static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) { @@ -77,7 +76,7 @@ static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) { goto _OVER; } - shmid = shmget(IPC_PRIVATE, sizeof(TdThreadMutex), 0600); + shmid = shmget(IPC_PRIVATE, sizeof(TdThreadMutex), IPC_CREAT | 0600); if (shmid <= 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to init mutex while shmget since %s", terrstr()); @@ -101,8 +100,13 @@ static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) { _OVER: if (code != 0) { - taosThreadMutexDestroy(pMutex); - shmctl(shmid, IPC_RMID, NULL); + if (pMutex != NULL) { + taosThreadMutexDestroy(pMutex); + shmdt(pMutex); + } + if (shmid >= 0) { + shmctl(shmid, IPC_RMID, NULL); + } } else { *ppMutex = pMutex; *pShmid = shmid; @@ -112,12 +116,12 @@ _OVER: return code; } -static void taosProcDestroyMutex(TdThreadMutex *pMutex, int32_t *pShmid) { +static void taosProcDestroyMutex(TdThreadMutex *pMutex, int32_t shmid) { if (pMutex != NULL) { taosThreadMutexDestroy(pMutex); } - if (*pShmid > 0) { - shmctl(*pShmid, IPC_RMID, NULL); + if (shmid >= 0) { + shmctl(shmid, IPC_RMID, NULL); } } @@ -141,9 +145,10 @@ static int32_t taosProcInitBuffer(void **ppBuffer, int32_t size) { return shmid; } -static void taosProcDestroyBuffer(void *pBuffer, int32_t *pShmid) { - if (*pShmid > 0) { - shmctl(*pShmid, IPC_RMID, NULL); +static void taosProcDestroyBuffer(void *pBuffer, int32_t shmid) { + if (shmid > 0) { + shmdt(pBuffer); + shmctl(shmid, IPC_RMID, NULL); } } @@ -155,29 +160,28 @@ static SProcQueue *taosProcQueueInit(int32_t size) { SProcQueue *pQueue = NULL; int32_t shmId = taosProcInitBuffer((void **)&pQueue, bufSize + headSize); - if (shmId <= 0) { + if (shmId < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pQueue->bufferShmid = shmId; if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { - free(pQueue); + taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); return NULL; } if (tsem_init(&pQueue->sem, 1, 0) != 0) { - taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); - free(pQueue); + taosProcDestroyMutex(pQueue->mutex, pQueue->mutexShmid); + taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { - taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); tsem_destroy(&pQueue->sem); - free(pQueue); + taosProcDestroyMutex(pQueue->mutex, pQueue->mutexShmid); + taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); return NULL; } @@ -190,12 +194,12 @@ static SProcQueue *taosProcQueueInit(int32_t size) { return pQueue; } -static void taosProcQueueCleanup(SProcQueue *pQueue) { +static void taosProcCleanupQueue(SProcQueue *pQueue) { if (pQueue != NULL) { uDebug("proc:%s, queue:%p clean up", pQueue->name, pQueue); - taosProcDestroyMutex(pQueue->mutex, &pQueue->mutexShmid); tsem_destroy(&pQueue->sem); - free(pQueue); + taosProcDestroyMutex(pQueue->mutex, pQueue->mutexShmid); + taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); } } @@ -344,12 +348,10 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { } pProc->name = pCfg->name; - pProc->testFlag = pCfg->testFlag; - pProc->pChildQueue = taosProcQueueInit(pCfg->childQueueSize); pProc->pParentQueue = taosProcQueueInit(pCfg->parentQueueSize); if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { - taosProcQueueCleanup(pProc->pChildQueue); + taosProcCleanupQueue(pProc->pChildQueue); free(pProc); return NULL; } @@ -369,17 +371,15 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { pProc->pParentQueue->freeBodyFp = pCfg->parentFreeBodyFp; pProc->pParentQueue->consumeFp = pCfg->parentConsumeFp; - uDebug("proc:%s, initialized, child queue:%p parent queue:%p", pProc->name, pProc->pChildQueue, pProc->pParentQueue); + uDebug("proc:%s, is initialized, child queue:%p parent queue:%p", pProc->name, pProc->pChildQueue, pProc->pParentQueue); - if (!pProc->testFlag) { - pProc->pid = fork(); - if (pProc->pid == 0) { - pProc->isChild = 1; - uInfo("this is child process, pid:%d", pProc->pid); - } else { - pProc->isChild = 0; - uInfo("this is parent process, pid:%d", pProc->pid); - } + pProc->pid = fork(); + if (pProc->pid == 0) { + pProc->isChild = 1; + uInfo("this is child process, pid:%d", pProc->pid); + } else { + pProc->isChild = 0; + uInfo("this is parent process, child pid:%d", pProc->pid); } return pProc; @@ -398,7 +398,7 @@ static void taosProcThreadLoop(SProcQueue *pQueue) { if (code < 0) { uDebug("proc:%s, get no message from queue:%p and exiting", pQueue->name, pQueue); break; - } else if (code < 0) { + } else if (code == 0) { uTrace("proc:%s, get no message from queue:%p since %s", pQueue->name, pQueue, terrstr()); taosMsleep(1); continue; @@ -413,16 +413,14 @@ int32_t taosProcRun(SProcObj *pProc) { taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pProc->isChild || pProc->testFlag) { + if (pProc->isChild) { if (taosThreadCreate(&pProc->childThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pChildQueue) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to create thread since %s", terrstr()); return -1; } uDebug("proc:%s, child start to consume queue:%p", pProc->name, pProc->pChildQueue); - } - - if (!pProc->isChild || pProc->testFlag) { + } else { if (taosThreadCreate(&pProc->parentThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pParentQueue) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to create thread since %s", terrstr()); @@ -445,8 +443,8 @@ void taosProcCleanup(SProcObj *pProc) { if (pProc != NULL) { uDebug("proc:%s, clean up", pProc->name); taosProcStop(pProc); - taosProcQueueCleanup(pProc->pChildQueue); - taosProcQueueCleanup(pProc->pParentQueue); + taosProcCleanupQueue(pProc->pChildQueue); + taosProcCleanupQueue(pProc->pParentQueue); free(pProc); } } From 4ea8c88810ed1f578f7817e483fd2a7b0751513a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 09:30:07 +0000 Subject: [PATCH 092/121] more TDB --- source/libs/tdb/test/tdbTest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index b9ae2dff7d..4c7af58b14 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -15,7 +15,7 @@ TEST(tdb_test, simple_test) { GTEST_ASSERT_EQ(ret, 0); // Create a database - ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, NULL, pEnv, &pDb); + ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, tKeyCmpr, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -94,8 +94,8 @@ TEST(tdb_test, simple_test) { static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { int k1, k2; - k1 = std::strtol((char *)pKey1 + 3, nullptr, kLen1 - 3); - k2 = std::strtol((char *)pKey2 + 3, nullptr, kLen2 - 3); + k1 = std::strtol((char *)pKey1 + 3, nullptr, 10); + k2 = std::strtol((char *)pKey2 + 3, nullptr, 10); if (k1 < k2) { return -1; From d7e13ef739236875e63b368dfabd7afe826d4228 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 25 Mar 2022 18:22:59 +0800 Subject: [PATCH 093/121] shm --- include/util/tprocess.h | 1 + source/dnode/mgmt/container/src/dndExec.c | 25 ++++++++++++++--------- source/dnode/mgmt/container/src/dndObj.c | 2 +- source/util/src/tprocess.c | 10 +++++++-- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/include/util/tprocess.h b/include/util/tprocess.h index 4ce536fd96..a0be38a3ad 100644 --- a/include/util/tprocess.h +++ b/include/util/tprocess.h @@ -51,6 +51,7 @@ void taosProcCleanup(SProcObj *pProc); int32_t taosProcRun(SProcObj *pProc); void taosProcStop(SProcObj *pProc); bool taosProcIsChild(SProcObj *pProc); +int32_t taosProcChildId(SProcObj *pProc); int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen); diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c index 4dcf7e80ec..a6c9852546 100644 --- a/source/dnode/mgmt/container/src/dndExec.c +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -20,7 +20,7 @@ static void dndResetLog(SMgmtWrapper *pMgmt) { char logname[24] = {0}; snprintf(logname, sizeof(logname), "%slog", pMgmt->name); - dInfo("node:%s, reset log to %s", pMgmt->name, logname); + dInfo("node:%s, reset log to %s in child process", pMgmt->name, logname); taosCloseLog(); taosInitLog(logname, 1); } @@ -51,6 +51,7 @@ int32_t dndOpenNode(SMgmtWrapper *pWrapper) { void dndCloseNode(SMgmtWrapper *pWrapper) { dDebug("node:%s, start to close", pWrapper->name); + pWrapper->required = false; taosWLockLatch(&pWrapper->latch); if (pWrapper->deployed) { (*pWrapper->fp.closeFp)(pWrapper); @@ -199,7 +200,7 @@ static int32_t dndRunInMultiProcess(SDnode *pDnode) { dInfo("node:%s, will be initialized in child process", pWrapper->name); dndOpenNode(pWrapper); } else { - dInfo("node:%s, will not start in parent process", pWrapper->name); + dInfo("node:%s, will not start in parent process, child pid:%d", pWrapper->name, taosProcChildId(pProc)); pWrapper->procType = PROC_PARENT; } @@ -209,16 +210,20 @@ static int32_t dndRunInMultiProcess(SDnode *pDnode) { } } -#if 0 - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper->procType == PROC_PARENT && dmStart(pWrapper->pMgmt) != 0) { - dndReleaseWrapper(pWrapper); - dError("failed to start dnode worker since %s", terrstr()); - return -1; + dndSetStatus(pDnode, DND_STAT_RUNNING); + + for (ENodeType n = 0; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + if (!pWrapper->required) continue; + if (pWrapper->fp.startFp == NULL) continue; + if (pWrapper->procType == PROC_PARENT && n != DNODE) continue; + if (pWrapper->procType == PROC_CHILD && n == DNODE) continue; + if ((*pWrapper->fp.startFp)(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } } - dndReleaseWrapper(pWrapper); -#endif return 0; } diff --git a/source/dnode/mgmt/container/src/dndObj.c b/source/dnode/mgmt/container/src/dndObj.c index d618f4e503..2123f5946b 100644 --- a/source/dnode/mgmt/container/src/dndObj.c +++ b/source/dnode/mgmt/container/src/dndObj.c @@ -175,7 +175,7 @@ int32_t dndMarkWrapper(SMgmtWrapper *pWrapper) { int32_t code = 0; taosRLockLatch(&pWrapper->latch); - if (pWrapper->deployed) { + if (pWrapper->deployed || (pWrapper->procType == PROC_PARENT && pWrapper->required)) { int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount); } else { diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index 24f3f8a6e4..f32f578952 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -208,6 +208,11 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHea const int32_t bodyLen = CEIL8(rawBodyLen); const int32_t fullLen = headLen + bodyLen + 8; + if (headLen <= 0 || bodyLen <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + return -1; + } + taosThreadMutexLock(pQueue->mutex); if (fullLen > pQueue->avail) { taosThreadMutexUnlock(pQueue->mutex); @@ -259,7 +264,7 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHea taosThreadMutexUnlock(pQueue->mutex); tsem_post(&pQueue->sem); - uTrace("proc:%s, push msg:%p:%d cont:%p:%d to queue:%p", pQueue->name, pHead, rawHeadLen, pBody, rawBodyLen, pQueue); + uTrace("proc:%s, push msg:%p:%d cont:%p:%d to queue:%p", pQueue->name, pHead, headLen, pBody, bodyLen, pQueue); return 0; } @@ -376,7 +381,6 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { pProc->pid = fork(); if (pProc->pid == 0) { pProc->isChild = 1; - uInfo("this is child process, pid:%d", pProc->pid); } else { pProc->isChild = 0; uInfo("this is parent process, child pid:%d", pProc->pid); @@ -439,6 +443,8 @@ void taosProcStop(SProcObj *pProc) { bool taosProcIsChild(SProcObj *pProc) { return pProc->isChild; } +int32_t taosProcChildId(SProcObj *pProc) { return pProc->pid; } + void taosProcCleanup(SProcObj *pProc) { if (pProc != NULL) { uDebug("proc:%s, clean up", pProc->name); From 72f1719a93254028be4f9d2248557acf7e4c380b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 10:28:23 +0000 Subject: [PATCH 094/121] add btree debug helper function --- source/libs/tdb/src/db/tdbBtree.c | 37 ++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index de77fb2649..45706ae3dc 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1289,4 +1289,39 @@ static int tdbBtcMoveUpward(SBTC *pBtc) { pBtc->idx = pBtc->idxStack[pBtc->iPage]; return 0; -} \ No newline at end of file +} + +#ifndef NODEBUG +typedef struct { + SPgno pgno; + u8 root; + u8 leaf; + SPgno rChild; + int nCells; + int nOvfl; +} SBtPageInfo; + +SBtPageInfo btPageInfos[20]; + +void tdbBtPageInfo(SPage *pPage, int idx) { + u8 flags; + SBtPageInfo *pBtPageInfo; + + pBtPageInfo = btPageInfos + idx; + + pBtPageInfo->pgno = TDB_PAGE_PGNO(pPage); + + flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); + + pBtPageInfo->root = TDB_BTREE_PAGE_IS_ROOT(flags); + pBtPageInfo->leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + + pBtPageInfo->rChild = 0; + if (!pBtPageInfo->leaf) { + pBtPageInfo->rChild = *(SPgno *)(pPage->pData + 1); + } + + pBtPageInfo->nCells = TDB_PAGE_TOTAL_CELLS(pPage) - pPage->nOverflow; + pBtPageInfo->nOvfl = pPage->nOverflow; +} +#endif \ No newline at end of file From fc64a2c72d00464356d2d4f8b7612a98610568fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 25 Mar 2022 10:54:54 +0000 Subject: [PATCH 095/121] more --- source/libs/tdb/test/tdbTest.cpp | 34 +++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 4c7af58b14..c256f07cf6 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -3,19 +3,23 @@ #include "tdbInt.h" static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); TEST(tdb_test, simple_test) { - int ret; - STEnv *pEnv; - STDB *pDb; - int nData = 10000000; + int ret; + STEnv *pEnv; + STDB *pDb; + FKeyComparator compFunc; + int nData = 10000000; + // int nData = 8508; // Open Env ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv); GTEST_ASSERT_EQ(ret, 0); // Create a database - ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, tKeyCmpr, pEnv, &pDb); + compFunc = tKeyCmpr; + ret = tdbDbOpen("db.db", TDB_VARIANT_LEN, TDB_VARIANT_LEN, compFunc, pEnv, &pDb); GTEST_ASSERT_EQ(ret, 0); { @@ -104,4 +108,24 @@ static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) } else { return 0; } +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; } \ No newline at end of file From 0b12117fb263c35179fb85942183e3cad431e313 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 26 Mar 2022 02:53:38 +0000 Subject: [PATCH 096/121] fix a test bug --- source/libs/tdb/test/tdbTest.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index c256f07cf6..52bf26f0ef 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -2,6 +2,8 @@ #include "tdbInt.h" +#include + static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); @@ -11,7 +13,6 @@ TEST(tdb_test, simple_test) { STDB *pDb; FKeyComparator compFunc; int nData = 10000000; - // int nData = 8508; // Open Env ret = tdbEnvOpen("tdb", 4096, 256000, &pEnv); @@ -98,8 +99,10 @@ TEST(tdb_test, simple_test) { static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { int k1, k2; - k1 = std::strtol((char *)pKey1 + 3, nullptr, 10); - k2 = std::strtol((char *)pKey2 + 3, nullptr, 10); + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); if (k1 < k2) { return -1; From f6bcae191314a419b785de1ad2bcbeaef0c2232b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 26 Mar 2022 01:39:10 -0400 Subject: [PATCH 097/121] sort scan cols --- include/libs/nodes/plannodes.h | 1 + include/libs/nodes/querynodes.h | 1 + source/libs/executor/src/executorimpl.c | 13 +++++++---- source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/parser/src/parAstCreater.c | 6 +++++ source/libs/parser/src/parTranslater.c | 2 ++ source/libs/planner/src/planLogicCreater.c | 10 +++++---- source/libs/planner/src/planPhysiCreater.c | 26 +++++++++++++++------- source/libs/planner/test/plannerTest.cpp | 10 ++++++++- source/libs/qworker/inc/qworkerInt.h | 1 + source/libs/qworker/src/qworker.c | 2 ++ 11 files changed, 56 insertions(+), 17 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 794e0ca85a..421e9042b7 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -65,6 +65,7 @@ typedef struct SAggLogicNode { typedef struct SProjectLogicNode { SLogicNode node; SNodeList* pProjections; + char stmtName[TSDB_TABLE_NAME_LEN]; } SProjectLogicNode; typedef struct SVnodeModifLogicNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 616e24d67d..5609bb4269 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -231,6 +231,7 @@ typedef struct SSelectStmt { SNodeList* pOrderByList; // SOrderByExprNode SNode* pLimit; SNode* pSlimit; + char stmtName[TSDB_TABLE_NAME_LEN]; } SSelectStmt; typedef enum ESetOperatorType { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fdf216c7eb..a601b9b0b7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5557,7 +5557,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo *pOperator, bool* newgroup) { blockDataCleanup(pInfo->pRes); - int32_t tableNameSlotId = 0; + int32_t tableNameSlotId = 1; SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId); char * name = NULL; @@ -8644,9 +8644,14 @@ SArray* extractScanColumnId(SNodeList* pNodeList) { } for(int32_t i = 0; i < numOfCols; ++i) { - STargetNode* pNode = (STargetNode*) nodesListGetNode(pNodeList, i); - SColumnNode* pColNode = (SColumnNode*) pNode->pExpr; - taosArrayPush(pList, &pColNode->colId); + for (int32_t j = 0; j < numOfCols; ++j) { + STargetNode* pNode = (STargetNode*) nodesListGetNode(pNodeList, j); + if (pNode->slotId == i) { + SColumnNode* pColNode = (SColumnNode*) pNode->pExpr; + taosArrayPush(pList, &pColNode->colId); + break; + } + } } return pList; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 60692323f5..3bc5e5d823 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -250,6 +250,7 @@ static SNode* logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pProjections); + COPY_CHAR_ARRAY_FIELD(stmtName); return (SNode*)pDst; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 4f27743b2b..5b2f36878b 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -645,6 +645,11 @@ SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const STok tempTable->pSubquery = pSubquery; if (NULL != pTableAlias && TK_NK_NIL != pTableAlias->type) { strncpy(tempTable->table.tableAlias, pTableAlias->z, pTableAlias->n); + } else { + sprintf(tempTable->table.tableAlias, "%p", tempTable); + } + if (QUERY_NODE_SELECT_STMT == nodeType(pSubquery)) { + strcpy(((SSelectStmt*)pSubquery)->stmtName, tempTable->table.tableAlias); } return (SNode*)tempTable; } @@ -792,6 +797,7 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr select->isDistinct = isDistinct; select->pProjectionList = pProjectionList; select->pFromTable = pTable; + sprintf(select->stmtName, "%p", select); return (SNode*)select; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8c842db161..19c6cb3f27 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1850,6 +1850,7 @@ static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) if (NULL == pSelect) { return TSDB_CODE_OUT_OF_MEMORY; } + sprintf(pSelect->stmtName, "%p", pSelect); SRealTableNode* pTable = nodesMakeNode(QUERY_NODE_REAL_TABLE); if (NULL == pTable) { @@ -1858,6 +1859,7 @@ static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) } strcpy(pTable->table.dbName, TSDB_INFORMATION_SCHEMA_DB); strcpy(pTable->table.tableName, getSysTableName(showType)); + strcpy(pTable->table.tableAlias, pTable->table.tableName); pSelect->pFromTable = (SNode*)pTable; *pStmt = pSelect; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 84fa52a070..6111597ebc 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -290,13 +290,14 @@ static int32_t createLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSel return code; } -static SColumnNode* createColumnByExpr(SExprNode* pExpr) { +static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) { SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { return NULL; } pCol->node.resType = pExpr->resType; strcpy(pCol->colName, pExpr->aliasName); + strcpy(pCol->tableAlias, pStmtName); return pCol; } @@ -484,7 +485,7 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return TSDB_CODE_FAILED; } -static int32_t createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pCols) { +static int32_t createColumnByProjections(SLogicPlanContext* pCxt, const char* pStmtName, SNodeList* pExprs, SNodeList** pCols) { SNodeList* pList = nodesMakeList(); if (NULL == pList) { return TSDB_CODE_OUT_OF_MEMORY; @@ -492,7 +493,7 @@ static int32_t createColumnByProjections(SLogicPlanContext* pCxt, SNodeList* pEx SNode* pNode; FOREACH(pNode, pExprs) { - if (TSDB_CODE_SUCCESS != nodesListAppend(pList, createColumnByExpr((SExprNode*)pNode))) { + if (TSDB_CODE_SUCCESS != nodesListAppend(pList, createColumnByExpr(pStmtName, (SExprNode*)pNode))) { nodesDestroyList(pList); return TSDB_CODE_OUT_OF_MEMORY; } @@ -514,9 +515,10 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel if (NULL == pProject->pProjections) { code = TSDB_CODE_OUT_OF_MEMORY; } + strcpy(pProject->stmtName, pSelect->stmtName); if (TSDB_CODE_SUCCESS == code) { - code = createColumnByProjections(pCxt,pSelect->pProjectionList, &pProject->node.pTargets); + code = createColumnByProjections(pCxt, pSelect->stmtName, pSelect->pProjectionList, &pProject->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index c3ad61dd37..26f0a0cdaa 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -30,14 +30,20 @@ typedef struct SPhysiPlanContext { SArray* pExecNodeList; } SPhysiPlanContext; -static int32_t getSlotKey(SNode* pNode, char* pKey) { +static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; + if (NULL != pStmtName) { + return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName); + } if ('\0' == pCol->tableAlias[0]) { return sprintf(pKey, "%s", pCol->colName); } return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName); } + if (NULL != pStmtName) { + return sprintf(pKey, "%s.%s", pStmtName, ((SExprNode*)pNode)->aliasName); + } return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); } @@ -74,7 +80,7 @@ static int32_t putSlotToHashImpl(int16_t dataBlockId, int16_t slotId, const char static int32_t putSlotToHash(int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) { char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - int32_t len = getSlotKey(pNode, name); + int32_t len = getSlotKey(pNode, NULL, name); return putSlotToHashImpl(dataBlockId, slotId, name, len, pHash); } @@ -138,14 +144,14 @@ static int32_t createDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SD return code; } -static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, bool output) { +static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, const char* pStmtName, bool output) { int32_t code = TSDB_CODE_SUCCESS; SHashObj* pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); int16_t nextSlotId = taosHashGetSize(pHash), slotId = 0; SNode* pNode = NULL; FOREACH(pNode, pList) { char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN] = {0}; - int32_t len = getSlotKey(pNode, name); + int32_t len = getSlotKey(pNode, pStmtName, name); SSlotIndex* pIndex = taosHashGet(pHash, name, len); if (NULL == pIndex) { code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, nextSlotId, output)); @@ -175,11 +181,15 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, } static int32_t addDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, false); + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, false); +} + +static int32_t addDataBlockSlotsForProject(SPhysiPlanContext* pCxt, const char* pStmtName, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, pStmtName, true); } static int32_t pushdownDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, false); + return addDataBlockSlotsImpl(pCxt, pList, pDataBlockDesc, NULL, true); } typedef struct SSetSlotIdCxt { @@ -192,7 +202,7 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) { SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - int32_t len = getSlotKey(pNode, name); + int32_t len = getSlotKey(pNode, NULL, name); SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len); if (NULL == pIndex) { pIndex = taosHashGet(pCxt->pRightHash, name, len); @@ -670,7 +680,7 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild int32_t code = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, pProjectLogicNode->pProjections, &pProject->pProjections); if (TSDB_CODE_SUCCESS == code) { - code = addDataBlockSlots(pCxt, pProject->pProjections, pProject->node.pOutputDataBlockDesc); + code = addDataBlockSlotsForProject(pCxt, pProjectLogicNode->stmtName, pProject->pProjections, pProject->node.pOutputDataBlockDesc); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject); diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index 319e539fa8..c318652b76 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -170,7 +170,7 @@ TEST_F(PlannerTest, groupBy) { bind("SELECT count(*) FROM t1"); ASSERT_TRUE(run()); - bind("SELECT c1, count(*) FROM t1 GROUP BY c1"); + bind("SELECT c1, max(c3), min(c2), count(*) FROM t1 GROUP BY c1"); ASSERT_TRUE(run()); bind("SELECT c1 + c3, c1 + count(*) FROM t1 where c2 = 'abc' GROUP BY c1, c3"); @@ -205,6 +205,14 @@ TEST_F(PlannerTest, showTables) { setDatabase("root", "test"); bind("show tables"); + ASSERT_TRUE(run()); +} + +TEST_F(PlannerTest, showStables) { + setDatabase("root", "test"); + + bind("show stables"); + ASSERT_TRUE(run()); } TEST_F(PlannerTest, createTopic) { diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index e768aa2de0..9d0dfb4174 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -178,6 +178,7 @@ typedef struct SQWorkerMgmt { #define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) #define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) #define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOGL(param, ...) qDebugL("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) #define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) #define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ae2dccfe39..93ba62170c 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -910,6 +910,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { atomic_store_8(&ctx->taskType, taskType); + QW_TASK_DLOGL("task string : %s", qwMsg->msg); + code = qStringToSubplan(qwMsg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { QW_TASK_ELOG("task string to subplan failed, code:%x - %s", code, tstrerror(code)); From 10f073e79cc58b9ddc112bff8d611f30efcf26b1 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 26 Mar 2022 02:58:44 -0400 Subject: [PATCH 098/121] order by plan implement --- include/libs/nodes/nodes.h | 1 + include/libs/nodes/plannodes.h | 13 +++- source/libs/nodes/src/nodesCloneFuncs.c | 20 ++++++ source/libs/nodes/src/nodesCodeFuncs.c | 71 +++++++++++++++++++++- source/libs/nodes/src/nodesTraverseFuncs.c | 4 +- source/libs/nodes/src/nodesUtilFuncs.c | 6 +- source/libs/planner/inc/planInt.h | 26 -------- source/libs/planner/src/planLogicCreater.c | 56 ++++++++++++++--- source/libs/planner/src/planPhysiCreater.c | 42 +++++++++++++ source/libs/planner/src/planSpliter.c | 4 +- source/libs/planner/test/plannerTest.cpp | 13 ++++ 11 files changed, 215 insertions(+), 41 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 4c83a30bb9..9b8739a4f3 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -120,6 +120,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN_VNODE_MODIF, QUERY_NODE_LOGIC_PLAN_EXCHANGE, QUERY_NODE_LOGIC_PLAN_WINDOW, + QUERY_NODE_LOGIC_PLAN_SORT, QUERY_NODE_LOGIC_SUBPLAN, QUERY_NODE_LOGIC_PLAN, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 421e9042b7..7b0a563907 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -99,6 +99,11 @@ typedef struct SWindowLogicNode { int64_t sessionGap; } SWindowLogicNode; +typedef struct SSortLogicNode { + SLogicNode node; + SNodeList* pSortKeys; +} SSortLogicNode; + typedef enum ESubplanType { SUBPLAN_TYPE_MERGE = 1, SUBPLAN_TYPE_PARTIAL, @@ -198,7 +203,7 @@ typedef struct SJoinPhysiNode { typedef struct SAggPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function - SNodeList* pGroupKeys; // SColumnRefNode list + SNodeList* pGroupKeys; SNodeList* pAggFuncs; } SAggPhysiNode; @@ -236,6 +241,12 @@ typedef struct SSessionWinodwPhysiNode { int64_t gap; } SSessionWinodwPhysiNode; +typedef struct SSortPhysiNode { + SPhysiNode node; + SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function + SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode +} SSortPhysiNode; + typedef struct SDataSinkNode { ENodeType type; SDataBlockDescNode* pInputDataBlockDesc; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 3bc5e5d823..161427b4b5 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -19,6 +19,11 @@ #include "taos.h" #include "taoserror.h" +#define COPY_ALL_SCALAR_FIELDS \ + do { \ + memcpy((pDst), (pSrc), sizeof(*pSrc)); \ + } while (0) + #define COPY_SCALAR_FIELD(fldname) \ do { \ (pDst)->fldname = (pSrc)->fldname; \ @@ -195,6 +200,12 @@ static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode return (SNode*)pDst; } +static SNode* orderByExprNodeCopy(const SOrderByExprNode* pSrc, SOrderByExprNode* pDst) { + COPY_ALL_SCALAR_FIELDS; + CLONE_NODE_FIELD(pExpr); + return (SNode*)pDst; +} + static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) { COPY_SCALAR_FIELD(mode); CLONE_NODE_FIELD(pValues); @@ -280,6 +291,12 @@ static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pD return (SNode*)pDst; } +static SNode* logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) { + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + CLONE_NODE_LIST_FIELD(pSortKeys); + return (SNode*)pDst; +} + static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) { CLONE_NODE_FIELD(pNode); COPY_SCALAR_FIELD(subplanType); @@ -339,6 +356,7 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { case QUERY_NODE_GROUPING_SET: return groupingSetNodeCopy((const SGroupingSetNode*)pNode, (SGroupingSetNode*)pDst); case QUERY_NODE_ORDER_BY_EXPR: + return orderByExprNodeCopy((const SOrderByExprNode*)pNode, (SOrderByExprNode*)pDst); case QUERY_NODE_LIMIT: break; case QUERY_NODE_FILL: @@ -361,6 +379,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicExchangeCopy((const SExchangeLogicNode*)pNode, (SExchangeLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_WINDOW: return logicWindowCopy((const SWindowLogicNode*)pNode, (SWindowLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_SORT: + return logicSortCopy((const SSortLogicNode*)pNode, (SSortLogicNode*)pDst); case QUERY_NODE_LOGIC_SUBPLAN: return logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst); default: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 37be210610..4225b5c9a8 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -920,6 +920,37 @@ static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkSortPhysiPlanExprs = "Exprs"; +static const char* jkSortPhysiPlanSortKeys = "SortKeys"; + +static int32_t physiSortNodeToJson(const void* pObj, SJson* pJson) { + const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkSortPhysiPlanExprs, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkSortPhysiPlanSortKeys, pNode->pSortKeys); + } + + return code; +} + +static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) { + SSortPhysiNode* pNode = (SSortPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSortPhysiPlanExprs, &pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkSortPhysiPlanSortKeys, &pNode->pSortKeys); + } + + return code; +} + static const char* jkWindowPhysiPlanExprs = "Exprs"; static const char* jkWindowPhysiPlanFuncs = "Funcs"; @@ -1807,6 +1838,38 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) { return code; } +static const char* jkOrderByExprExpr = "Expr"; +static const char* jkOrderByExprOrder = "Order"; +static const char* jkOrderByExprNullOrder = "NullOrder"; + +static int32_t orderByExprNodeToJson(const void* pObj, SJson* pJson) { + const SOrderByExprNode* pNode = (const SOrderByExprNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkOrderByExprExpr, nodeToJson, pNode->pExpr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkOrderByExprOrder, pNode->order); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkOrderByExprNullOrder, pNode->nullOrder); + } + + return code; +} + +static int32_t jsonToOrderByExprNode(const SJson* pJson, void* pObj) { + SOrderByExprNode* pNode = (SOrderByExprNode*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkOrderByExprExpr, &pNode->pExpr); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder); + } + + return code; +} + static const char* jkIntervalWindowInterval = "Interval"; static const char* jkIntervalWindowOffset = "Offset"; static const char* jkIntervalWindowSliding = "Sliding"; @@ -2155,6 +2218,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_GROUPING_SET: return groupingSetNodeToJson(pObj, pJson); case QUERY_NODE_ORDER_BY_EXPR: + return orderByExprNodeToJson(pObj, pJson); case QUERY_NODE_LIMIT: case QUERY_NODE_STATE_WINDOW: case QUERY_NODE_SESSION_WINDOW: @@ -2218,7 +2282,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return physiExchangeNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SORT: - break; + return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return physiIntervalNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: @@ -2258,7 +2322,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { // break; // case QUERY_NODE_GROUPING_SET: // return jsonToGroupingSetNode(pJson, pObj); - // case QUERY_NODE_ORDER_BY_EXPR: + case QUERY_NODE_ORDER_BY_EXPR: + return jsonToOrderByExprNode(pJson, pObj); // case QUERY_NODE_LIMIT: // case QUERY_NODE_STATE_WINDOW: // case QUERY_NODE_SESSION_WINDOW: @@ -2307,6 +2372,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiAggNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return jsonToPhysiExchangeNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_SORT: + return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return jsonToPhysiIntervalNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index ff71c3bd58..3af5a6d8cc 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -294,10 +294,10 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa case SQL_CLAUSE_GROUP_BY: nodesWalkNode(pSelect->pHaving, walker, pContext); case SQL_CLAUSE_HAVING: - nodesWalkList(pSelect->pProjectionList, walker, pContext); - case SQL_CLAUSE_SELECT: nodesWalkList(pSelect->pOrderByList, walker, pContext); case SQL_CLAUSE_ORDER_BY: + nodesWalkList(pSelect->pProjectionList, walker, pContext); + case SQL_CLAUSE_SELECT: default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index d529ee2b1a..b22aa4fea4 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -159,6 +159,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SExchangeLogicNode)); case QUERY_NODE_LOGIC_PLAN_WINDOW: return makeNode(type, sizeof(SWindowLogicNode)); + case QUERY_NODE_LOGIC_PLAN_SORT: + return makeNode(type, sizeof(SSortLogicNode)); case QUERY_NODE_LOGIC_SUBPLAN: return makeNode(type, sizeof(SLogicSubplan)); case QUERY_NODE_LOGIC_PLAN: @@ -182,7 +184,7 @@ SNodeptr nodesMakeNode(ENodeType type) { case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return makeNode(type, sizeof(SExchangePhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SORT: - return makeNode(type, sizeof(SNode)); + return makeNode(type, sizeof(SSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: @@ -555,7 +557,7 @@ static EDealRes collectColumns(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; int32_t colId = pCol->colId; - if (0 == strcmp(pCxt->pTableAlias, pCol->tableAlias)) { + if (NULL == pCxt->pTableAlias || 0 == strcmp(pCxt->pTableAlias, pCol->tableAlias)) { return doCollect(pCxt, colId, pNode); } } diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 42449d63d6..144254b042 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -22,32 +22,6 @@ extern "C" { #include "planner.h" -#define CHECK_ALLOC(p, res) \ - do { \ - if (NULL == (p)) { \ - pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \ - return (res); \ - } \ - } while (0) - -#define CHECK_CODE(exec, res) \ - do { \ - int32_t code = (exec); \ - if (TSDB_CODE_SUCCESS != code) { \ - pCxt->errCode = code; \ - return (res); \ - } \ - } while (0) - -#define CHECK_CODE_EXT(exec) \ - do { \ - int32_t code = (exec); \ - if (TSDB_CODE_SUCCESS != code) { \ - pCxt->errCode = code; \ - return code; \ - } \ - } while (0) - #define planFatal(param, ...) qFatal("PLAN: " param, __VA_ARGS__) #define planError(param, ...) qError("PLAN: " param, __VA_ARGS__) #define planWarn(param, ...) qWarn("PLAN: " param, __VA_ARGS__) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 6111597ebc..7e3b67f809 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -45,7 +45,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { } if (nodesEqualNode(pExpr, *pNode)) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - CHECK_ALLOC(pCol, DEAL_RES_ERROR); + if (NULL == pCol) { + return DEAL_RES_ERROR; + } SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); @@ -311,20 +313,22 @@ static EDealRes doCreateColumn(SNode* pNode, void* pContext) { switch (nodeType(pNode)) { case QUERY_NODE_COLUMN: { SNode* pCol = nodesCloneNode(pNode); - CHECK_ALLOC(pCol, DEAL_RES_ERROR); - CHECK_CODE(nodesListAppend(pCxt->pList, pCol), DEAL_RES_ERROR); - return DEAL_RES_IGNORE_CHILD; + if (NULL == pCol) { + return DEAL_RES_ERROR; + } + return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: { SExprNode* pExpr = (SExprNode*)pNode; SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - CHECK_ALLOC(pCol, DEAL_RES_ERROR); + if (NULL == pCol) { + return DEAL_RES_ERROR; + } pCol->node.resType = pExpr->resType; strcpy(pCol->colName, pExpr->aliasName); - CHECK_CODE(nodesListAppend(pCxt->pList, (SNode*)pCol), DEAL_RES_ERROR); - return DEAL_RES_IGNORE_CHILD; + return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } default: break; @@ -485,6 +489,41 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return TSDB_CODE_FAILED; } +static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + if (NULL == pSelect->pOrderByList) { + return TSDB_CODE_SUCCESS; + } + + SSortLogicNode* pSort = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SORT); + if (NULL == pSort) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNodeList* pCols = NULL; + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, &pCols); + if (TSDB_CODE_SUCCESS == code && NULL != pCols) { + pSort->node.pTargets = nodesCloneList(pCols); + if (NULL == pSort->node.pTargets) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + pSort->pSortKeys = nodesCloneList(pSelect->pOrderByList); + if (NULL == pSort->pSortKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pSort; + } else { + nodesDestroyNode(pSort); + } + + return code; +} + static int32_t createColumnByProjections(SLogicPlanContext* pCxt, const char* pStmtName, SNodeList* pExprs, SNodeList** pCols) { SNodeList* pList = nodesMakeList(); if (NULL == pList) { @@ -539,6 +578,9 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele if (TSDB_CODE_SUCCESS == code) { code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot); } + if (TSDB_CODE_SUCCESS == code) { + code = createChildLogicNode(pCxt, pSelect, createSortLogicNode, &pRoot); + } if (TSDB_CODE_SUCCESS == code) { code = createChildLogicNode(pCxt, pSelect, createProjectLogicNode, &pRoot); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 26f0a0cdaa..dbaf64bdbf 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -31,6 +31,10 @@ typedef struct SPhysiPlanContext { } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { + if (QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode)) { + return getSlotKey(((SOrderByExprNode*)pNode)->pExpr, pStmtName, pKey); + } + if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; if (NULL != pStmtName) { @@ -41,6 +45,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { } return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName); } + if (NULL != pStmtName) { return sprintf(pKey, "%s.%s", pStmtName, ((SExprNode*)pNode)->aliasName); } @@ -815,6 +820,41 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr return TSDB_CODE_FAILED; } +static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SSortLogicNode* pSortLogicNode, SPhysiNode** pPhyNode) { + SSortPhysiNode* pSort = (SSortPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pSortLogicNode, QUERY_NODE_PHYSICAL_PLAN_SORT); + if (NULL == pSort) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SNodeList* pPrecalcExprs = NULL; + SNodeList* pSortKeys = NULL; + int32_t code = rewritePrecalcExprs(pCxt, pSortLogicNode->pSortKeys, &pPrecalcExprs, &pSortKeys); + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + // push down expression to pOutputDataBlockDesc of child node + if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pSort->pExprs); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pSort->pExprs, pChildTupe); + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pSortKeys, &pSort->pSortKeys); + if (TSDB_CODE_SUCCESS == code) { + code = addDataBlockSlots(pCxt, pSort->pSortKeys, pSort->node.pOutputDataBlockDesc); + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pSort; + } else { + nodesDestroyNode(pSort); + } + + return code; +} + static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SNodeList* pChildren, SPhysiNode** pPhyNode) { switch (nodeType(pLogicNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: @@ -829,6 +869,8 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode return createExchangePhysiNode(pCxt, (SExchangeLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: return createWindowPhysiNode(pCxt, pChildren, (SWindowLogicNode*)pLogicNode, pPhyNode); + case QUERY_NODE_LOGIC_PLAN_SORT: + return createSortPhysiNode(pCxt, pChildren, (SSortLogicNode*)pLogicNode, pPhyNode); default: break; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index d203d41cb0..a38fea04a5 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -65,7 +65,9 @@ static int32_t stsMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan) { SLogicNode* pSplitNode = stsMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { SStsInfo* pInfo = calloc(1, sizeof(SStsInfo)); - CHECK_ALLOC(pInfo, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pInfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } pInfo->pScan = (SScanLogicNode*)pSplitNode; pInfo->pSubplan = pSubplan; pCxt->pInfo = pInfo; diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index c318652b76..ce9aca7f46 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -201,6 +201,19 @@ TEST_F(PlannerTest, sessionWindow) { ASSERT_TRUE(run()); } +TEST_F(PlannerTest, orderBy) { + setDatabase("root", "test"); + + bind("SELECT * FROM t1 order by c1"); + ASSERT_TRUE(run()); + + bind("SELECT c1 FROM t1 order by c2"); + ASSERT_TRUE(run()); + + bind("SELECT * FROM t1 order by c1 + 10, c2"); + ASSERT_TRUE(run()); +} + TEST_F(PlannerTest, showTables) { setDatabase("root", "test"); From 1d52dd9d430586df706e78484bdbd741d71877ac Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 26 Mar 2022 15:02:29 +0800 Subject: [PATCH 099/121] [td-13039] fix bug. --- source/libs/executor/inc/executorimpl.h | 5 ++- source/libs/executor/src/executorimpl.c | 59 ++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c582873315..b6b5cdb727 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -415,6 +415,7 @@ typedef struct STableScanInfo { int32_t* rowCellInfoOffset; SExprInfo* pExpr; SSDataBlock block; + SArray* pColMatchInfo; int32_t numOfOutput; int64_t elapsedTime; int32_t prevGroupId; // previous table group id @@ -646,8 +647,8 @@ typedef struct SDistinctOperatorInfo { } SDistinctOperatorInfo; SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, - int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t repeatTime, + int32_t reverseTime, SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a601b9b0b7..872ce173ad 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -66,6 +66,11 @@ typedef enum SResultTsInterpType { RESULT_ROW_END_INTERP = 2, } SResultTsInterpType; +typedef struct SColMatchInfo { + int32_t colId; + int32_t targetSlotId; +} SColMatchInfo; + #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = taosRand(); @@ -2944,12 +2949,21 @@ int32_t loadDataBlock(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, *status = BLK_DATA_ALL_NEEDED; - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); - if (pBlock->pDataBlock == NULL) { + SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); + if (pCols == NULL) { return terrno; - } else { - return TSDB_CODE_SUCCESS; } + + int32_t numOfCols = pBlock->info.numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pCols, i); + SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); + ASSERT(pColMatchInfo->colId == p->info.colId); + + taosArraySet(pBlock->pDataBlock, pColMatchInfo->targetSlotId, p); + } + + return TSDB_CODE_SUCCESS; } int32_t loadDataBlockOnDemand(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { @@ -5374,7 +5388,8 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { return pResBlock; } -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, + SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); @@ -5387,12 +5402,19 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, return NULL; } + pInfo->block.pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + for(int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData idata = {0}; + taosArrayPush(pInfo->block.pDataBlock, &idata); + } + pInfo->pTsdbReadHandle = pTsdbReadHandle; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->order = order; pInfo->current = 0; pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColMatchInfo; pOperator->name = "TableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; pOperator->blockingOptr = false; @@ -8497,6 +8519,7 @@ static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); static SArray* extractScanColumnId(SNodeList* pNodeList); static SArray* extractColumnInfo(SNodeList* pNodeList); +static SArray* extractColMatchInfo(SNodeList* pNodeList); SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { @@ -8505,7 +8528,9 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa size_t numOfCols = LIST_LENGTH(pScanPhyNode->pScanCols); tsdbReaderT pDataReader = doCreateDataReader((STableScanPhysiNode*)pPhyNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); - return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); + SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols); + + return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pColList, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pPhyNode)) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createOutputBuf_rv1(pExchange->node.pOutputDataBlockDesc); @@ -8683,6 +8708,28 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { return pList; } +SArray* extractColMatchInfo(SNodeList* pNodeList) { + size_t numOfCols = LIST_LENGTH(pNodeList); + SArray* pList = taosArrayInit(numOfCols, sizeof(SColMatchInfo)); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for(int32_t i = 0; i < numOfCols; ++i) { + STargetNode* pNode = (STargetNode*) nodesListGetNode(pNodeList, i); + SColumnNode* pColNode = (SColumnNode*) pNode->pExpr; + + SColMatchInfo c = {0}; + c.colId = pColNode->colId; + c.targetSlotId = pNode->slotId; + + taosArrayPush(pList, &c); + } + + return pList; +} + int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { int32_t code = 0; if (tableType == TSDB_SUPER_TABLE) { From 71097ea56acaf1bcf1d73a490e668acae24c0513 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 26 Mar 2022 16:48:14 +0800 Subject: [PATCH 100/121] put schema into stream --- example/src/tstream.c | 2 +- include/common/tcommon.h | 4 +-- include/common/tmsg.h | 25 +++++++++++++++-- include/util/taoserror.h | 2 ++ source/dnode/mnode/impl/inc/mndDef.h | 12 ++++---- source/dnode/mnode/impl/src/mndDef.c | 10 ++++++- source/dnode/mnode/impl/src/mndStream.c | 14 ++++------ source/libs/wal/src/walMeta.c | 37 ++++++++++++------------- source/libs/wal/src/walRead.c | 8 ++++-- 9 files changed, 72 insertions(+), 42 deletions(-) diff --git a/example/src/tstream.c b/example/src/tstream.c index 8ffa932bd2..51578bd27b 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -25,7 +25,7 @@ int32_t init_env() { return -1; } - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); if (taos_errno(pRes) != 0) { printf("error in create db, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/common/tcommon.h b/include/common/tcommon.h index eb9f450872..67611d9563 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -127,7 +127,7 @@ static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics); if (pRsp->numOfTopics == 0) return tlen; - tlen += tEncodeSSchemaWrapper(buf, pRsp->schema); + tlen += taosEncodeSSchemaWrapper(buf, pRsp->schema); if (pRsp->pBlockData) { sz = taosArrayGetSize(pRsp->pBlockData); } @@ -149,7 +149,7 @@ static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { if (pRsp->numOfTopics == 0) return buf; pRsp->schema = (SSchemaWrapper*)taosMemoryCalloc(1, sizeof(SSchemaWrapper)); if (pRsp->schema == NULL) return NULL; - buf = tDecodeSSchemaWrapper(buf, pRsp->schema); + buf = taosDecodeSSchemaWrapper(buf, pRsp->schema); buf = taosDecodeFixedI32(buf, &sz); pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock)); for (int32_t i = 0; i < sz; i++) { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 136a576c7c..4c7be6b541 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1931,7 +1931,7 @@ static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) { return 0; } -static FORCE_INLINE int32_t tEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { +static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { int32_t tlen = 0; tlen += taosEncodeFixedU32(buf, pSW->nCols); for (int32_t i = 0; i < pSW->nCols; i++) { @@ -1940,7 +1940,7 @@ static FORCE_INLINE int32_t tEncodeSSchemaWrapper(void** buf, const SSchemaWrapp return tlen; } -static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) { +static FORCE_INLINE void* taosDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) { buf = taosDecodeFixedU32(buf, &pSW->nCols); pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { @@ -1953,6 +1953,27 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) return buf; } +static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SCoder* pEncoder, const SSchemaWrapper* pSW) { + if (tEncodeU32(pEncoder, pSW->nCols) < 0) return -1; + for (int32_t i = 0; i < pSW->nCols; i++) { + if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1; + } + return pEncoder->pos; +} + +static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapper* pSW) { + if (tDecodeU32(pDecoder, &pSW->nCols) < 0) return -1; + void* ptr = taosMemoryRealloc(pSW->pSchema, pSW->nCols * sizeof(SSchema)); + if (ptr == NULL) { + return -1; + } + pSW->pSchema = (SSchema*)ptr; + for (int32_t i = 0; i < pSW->nCols; i++) { + if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1; + } + return 0; +} + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char stb[TSDB_TABLE_FNAME_LEN]; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 87781b6313..994ad7afc6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -16,6 +16,8 @@ #ifndef _TD_UTIL_TAOS_ERROR_H_ #define _TD_UTIL_TAOS_ERROR_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index eb7ac5b353..2c4f2e82ac 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -733,12 +733,12 @@ typedef struct { int8_t sourceType; int8_t sinkType; // int32_t sqlLen; - int32_t sinkVgId; // 0 for automatic - char* sql; - char* logicalPlan; - char* physicalPlan; - SArray* tasks; // SArray> - SArray* ColAlias; // SArray + int32_t sinkVgId; // 0 for automatic + char* sql; + char* logicalPlan; + char* physicalPlan; + SArray* tasks; // SArray> + SSchemaWrapper outputSchema; } SStreamObj; int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 6fa926d548..24f2a5df22 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -17,7 +17,7 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { int32_t sz = 0; - int32_t outputNameSz = 0; + /*int32_t outputNameSz = 0;*/ if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1; if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; @@ -45,6 +45,9 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { } } + if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1; + +#if 0 if (pObj->ColAlias != NULL) { outputNameSz = taosArrayGetSize(pObj->ColAlias); } @@ -53,6 +56,7 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) { char *name = taosArrayGetP(pObj->ColAlias, i); if (tEncodeCStr(pEncoder, name) < 0) return -1; } +#endif return pEncoder->pos; } @@ -85,6 +89,9 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { taosArrayPush(pObj->tasks, pArray); } } + + if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; +#if 0 int32_t outputNameSz; if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1; if (outputNameSz != 0) { @@ -98,5 +105,6 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) { if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1; taosArrayPush(pObj->ColAlias, &name); } +#endif return 0; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index dd62bc0364..47624dfcc1 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -14,7 +14,6 @@ */ #include "mndStream.h" -#include "parser.h" #include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" @@ -26,6 +25,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "parser.h" #include "tname.h" #define MND_STREAM_VER_NUMBER 1 @@ -248,23 +248,21 @@ static int32_t mndStreamGetPlanString(const char *ast, char **pStr) { int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) { SNode *pAst = NULL; -#if 1 // TODO: remove debug info later - printf("ast = %s\n", ast); +#if 1 // TODO: remove debug info later + printf("ast = %s\n", ast); #endif if (nodesStringToNode(ast, &pAst) < 0) { return -1; } #if 1 - SSchemaWrapper sw = {0}; - qExtractResultSchema(pAst, (int32_t*)&sw.nCols, &sw.pSchema); + qExtractResultSchema(pAst, (int32_t *)&pStream->outputSchema.nCols, &pStream->outputSchema.pSchema); printf("|"); - for (int i = 0; i < sw.nCols; i++) { - printf(" %15s |", (char *)sw.pSchema[i].name); + for (int i = 0; i < pStream->outputSchema.nCols; i++) { + printf(" %15s |", (char *)pStream->outputSchema.pSchema[i].name); } printf("\n=======================================================\n"); - pStream->ColAlias = NULL; #endif if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(ast, &pStream->physicalPlan)) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index cbe7cb81db..83c48628a3 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -37,8 +37,7 @@ void* tmemmem(char* haystack, int hlen, char* needle, int nlen) { } limit = haystack + hlen - nlen + 1; - while ((haystack = (char*)memchr( - haystack, needle[0], limit - haystack)) != NULL) { + while ((haystack = (char*)memchr(haystack, needle[0], limit - haystack)) != NULL) { if (memcmp(haystack, needle, nlen) == 0) { return haystack; } @@ -57,8 +56,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { } #endif - SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz-1); - char fnameStr[WAL_FILE_LEN]; + SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1); + char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); int64_t file_size = 0; @@ -88,20 +87,20 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - + char* haystack = buf; char* found = NULL; - char *candidate; - while((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { + char* candidate; + while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { // read and validate - SWalHead *logContent = (SWalHead*)candidate; + SWalHead* logContent = (SWalHead*)candidate; if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { found = candidate; } haystack = candidate + 1; } if (found == buf) { - SWalHead *logContent = (SWalHead*)found; + SWalHead* logContent = (SWalHead*)found; if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { // file has to be deleted taosMemoryFree(buf); @@ -111,7 +110,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { } } taosCloseFile(&pFile); - SWalHead *lastEntry = (SWalHead*)found; + SWalHead* lastEntry = (SWalHead*)found; return lastEntry->head.version; } @@ -158,10 +157,10 @@ int walCheckAndRepairMeta(SWal* pWal) { int newSz = taosArrayGetSize(pLogInfoArray); if (oldSz > newSz) { - taosArrayPopFrontBatch(pWal->fileInfoSet, oldSz - newSz); + taosArrayPopFrontBatch(pWal->fileInfoSet, oldSz - newSz); } else if (oldSz < newSz) { for (int i = oldSz; i < newSz; i++) { - SWalFileInfo *pFileInfo = taosArrayGet(pLogInfoArray, i); + SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, i); taosArrayPush(pWal->fileInfoSet, pFileInfo); } } @@ -171,8 +170,8 @@ int walCheckAndRepairMeta(SWal* pWal) { if (newSz > 0) { pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; - SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, newSz-1); - char fnameStr[WAL_FILE_LEN]; + SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, newSz - 1); + char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); int64_t file_size = 0; taosStatFile(fnameStr, &file_size, NULL); @@ -191,8 +190,8 @@ int walCheckAndRepairMeta(SWal* pWal) { } } - //TODO: set fileSize and lastVer if necessary - + // TODO: set fileSize and lastVer if necessary + return 0; } @@ -239,13 +238,13 @@ char* walMetaSerialize(SWal* pWal) { cJSON* pFiles = cJSON_CreateArray(); cJSON* pField; if (pRoot == NULL || pMeta == NULL || pFiles == NULL) { - if(pRoot) { + if (pRoot) { cJSON_Delete(pRoot); } - if(pMeta) { + if (pMeta) { cJSON_Delete(pMeta); } - if(pFiles) { + if (pFiles) { cJSON_Delete(pFiles); } terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index e15c162048..9d7c9ed9df 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "walInt.h" #include "taoserror.h" +#include "walInt.h" SWalReadHandle *walOpenReadHandle(SWal *pWal) { SWalReadHandle *pRead = taosMemoryMalloc(sizeof(SWalReadHandle)); @@ -92,6 +92,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pIdxTFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -169,7 +170,8 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } if (pRead->pHead->head.version != ver) { - wError("unexpected wal log version: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, ver); + wError("unexpected wal log version: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, + ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; @@ -177,7 +179,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { code = walValidBodyCksum(pRead->pHead); if (code != 0) { - wError("unexpected wal log version: checksum not passed"); + wError("unexpected wal log version: % " PRId64 "checksum not passed", ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; From dbec5f1fa4044c4198452b6dcf32b0d0e843ba27 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 26 Mar 2022 09:29:49 +0000 Subject: [PATCH 101/121] fix logical bug with memory leak --- source/libs/tdb/src/db/tdbBtree.c | 12 +-- source/libs/tdb/src/page/tdbPage.c | 6 +- source/libs/tdb/test/tdbTest.cpp | 145 +++++++++++++++++++++-------- 3 files changed, 117 insertions(+), 46 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 45706ae3dc..5980c2b531 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -746,6 +746,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { pgno = TDB_PAGE_PGNO(pNews[iNew]); tdbBtreeEncodeCell(pParent, cd.pKey, cd.kLen, (void *)&pgno, sizeof(SPgno), pNewCell, &szNewCell); tdbPageInsertCell(pParent, sIdx++, pNewCell, szNewCell, 0); + free(pNewCell); } // move to next new page @@ -795,12 +796,11 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } - // TODO: here has memory leak - // for (int i = 0; i < 3; i++) { - // if (pDivCell[i]) { - // free(pDivCell[i]); - // } - // } + for (int i = 0; i < 3; i++) { + if (pDivCell[i]) { + free(pDivCell[i]); + } + } return 0; } diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 15b6e218e6..516330e4e6 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -143,7 +143,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl pPage->aiOvfl[i] = pPage->aiOvfl[i - 1]; } - pPage->apOvfl[iOvfl] = pCell; + // TODO: here has memory leak + pNewCell = (SCell *)malloc(szCell); + memcpy(pNewCell, pCell, szCell); + + pPage->apOvfl[iOvfl] = pNewCell; pPage->aiOvfl[iOvfl] = idx; pPage->nOverflow++; iOvfl++; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index 52bf26f0ef..63341e5430 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -4,8 +4,112 @@ #include -static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2); -static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)malloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void closePool(SPoolMem *pPool) { + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + free(pMem); + } while (1); + + assert(pPool->size == 0); + + free(pPool); +} + +static void *poolMalloc(void *arg, int size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)malloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + free(pMem); +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} TEST(tdb_test, simple_test) { int ret; @@ -94,41 +198,4 @@ TEST(tdb_test, simple_test) { // Close Env ret = tdbEnvClose(pEnv); GTEST_ASSERT_EQ(ret, 0); -} - -static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { - int k1, k2; - - std::string s1((char *)pKey1 + 3, kLen1 - 3); - std::string s2((char *)pKey2 + 3, kLen2 - 3); - k1 = stoi(s1); - k2 = stoi(s2); - - if (k1 < k2) { - return -1; - } else if (k1 > k2) { - return 1; - } else { - return 0; - } -} - -static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { - int mlen; - int cret; - - ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); - - mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; - cret = memcmp(pKey1, pKey2, mlen); - if (cret == 0) { - if (keyLen1 < keyLen2) { - cret = -1; - } else if (keyLen1 > keyLen2) { - cret = 1; - } else { - cret = 0; - } - } - return cret; } \ No newline at end of file From 85f87ae246900401712c0c83e3d15c266b55b305 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 26 Mar 2022 05:45:48 -0400 Subject: [PATCH 102/121] bugfix --- .gitignore | 1 + include/libs/nodes/plannodes.h | 2 + include/libs/nodes/querynodes.h | 3 +- source/libs/nodes/src/nodesCloneFuncs.c | 16 ++++---- source/libs/nodes/src/nodesCodeFuncs.c | 7 ++++ source/libs/nodes/src/nodesTraverseFuncs.c | 6 +++ source/libs/parser/inc/parUtil.h | 2 + source/libs/parser/src/parAstCreater.c | 7 ++++ source/libs/parser/src/parTranslater.c | 4 ++ source/libs/planner/src/planLogicCreater.c | 8 +++- source/libs/planner/src/planPhysiCreater.c | 46 +++++++++++++++++++--- 11 files changed, 88 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 1bfbf00cd5..b62bd62d9c 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.settings/ source/libs/parser/inc/sql.* tests/script/tmqResult.txt +tests/tmqResult.txt # Emacs # -*- mode: gitignore; -*- diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 7b0a563907..c87dd0ec8b 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -97,6 +97,7 @@ typedef struct SWindowLogicNode { int8_t slidingUnit; SFillNode* pFill; int64_t sessionGap; + SNode* pTspk; } SWindowLogicNode; typedef struct SSortLogicNode { @@ -228,6 +229,7 @@ typedef struct SWinodwPhysiNode { typedef struct SIntervalPhysiNode { SWinodwPhysiNode window; + SNode* pTspk; // timestamp primary key int64_t interval; int64_t offset; int64_t sliding; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5609bb4269..1ba7ae10d5 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -191,12 +191,13 @@ typedef struct SStateWindowNode { typedef struct SSessionWindowNode { ENodeType type; // QUERY_NODE_SESSION_WINDOW - SNode* pCol; + SNode* pCol; // timestamp primary key SNode* pGap; // gap between two session window(in microseconds) } SSessionWindowNode; typedef struct SIntervalWindowNode { ENodeType type; // QUERY_NODE_INTERVAL_WINDOW + SNode* pCol; // timestamp primary key SNode* pInterval; // SValueNode SNode* pOffset; // SValueNode SNode* pSliding; // SValueNode diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 161427b4b5..17c66f6ed7 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -278,16 +278,18 @@ static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNo } static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pDst) { + COPY_ALL_SCALAR_FIELDS; COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); - COPY_SCALAR_FIELD(winType); + // COPY_SCALAR_FIELD(winType); CLONE_NODE_LIST_FIELD(pFuncs); - COPY_SCALAR_FIELD(interval); - COPY_SCALAR_FIELD(offset); - COPY_SCALAR_FIELD(sliding); - COPY_SCALAR_FIELD(intervalUnit); - COPY_SCALAR_FIELD(slidingUnit); + // COPY_SCALAR_FIELD(interval); + // COPY_SCALAR_FIELD(offset); + // COPY_SCALAR_FIELD(sliding); + // COPY_SCALAR_FIELD(intervalUnit); + // COPY_SCALAR_FIELD(slidingUnit); CLONE_NODE_FIELD(pFill); - COPY_SCALAR_FIELD(sessionGap); + // COPY_SCALAR_FIELD(sessionGap); + CLONE_NODE_FIELD(pTspk); return (SNode*)pDst; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 4225b5c9a8..849764c995 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -988,6 +988,7 @@ static const char* jkIntervalPhysiPlanSliding = "Sliding"; static const char* jkIntervalPhysiPlanIntervalUnit = "intervalUnit"; static const char* jkIntervalPhysiPlanSlidingUnit = "slidingUnit"; static const char* jkIntervalPhysiPlanFill = "Fill"; +static const char* jkIntervalPhysiPlanTsPk = "TsPk"; static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; @@ -1011,6 +1012,9 @@ static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkIntervalPhysiPlanFill, nodeToJson, pNode->pFill); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalPhysiPlanTsPk, nodeToJson, pNode->pTspk); + } return code; } @@ -1037,6 +1041,9 @@ static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkIntervalPhysiPlanFill, (SNode**)&pNode->pFill); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalPhysiPlanTsPk, (SNode**)&pNode->pTspk); + } return code; } diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 3af5a6d8cc..7eaa049946 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -99,6 +99,9 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker if (DEAL_RES_ERROR != res) { res = walkNode(pInterval->pFill, order, walker, pContext); } + if (DEAL_RES_ERROR != res) { + res = walkNode(pInterval->pCol, order, walker, pContext); + } break; } case QUERY_NODE_NODE_LIST: @@ -225,6 +228,9 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit if (DEAL_RES_ERROR != res) { res = rewriteNode(&(pInterval->pFill), order, rewriter, pContext); } + if (DEAL_RES_ERROR != res) { + res = rewriteNode(&(pInterval->pCol), order, rewriter, pContext); + } break; } case QUERY_NODE_NODE_LIST: diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 171b406e18..742ab303d3 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -30,6 +30,8 @@ extern "C" { #define parserDebug(param, ...) qDebug("PARSER: " param, __VA_ARGS__) #define parserTrace(param, ...) qTrace("PARSER: " param, __VA_ARGS__) +#define PK_TS_COL_INTERNAL_NAME "_rowts" + typedef struct SMsgBuf { int32_t len; char *buf; diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 5b2f36878b..d9ee1309ad 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -702,6 +702,13 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol) { SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill) { SIntervalWindowNode* interval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); CHECK_OUT_OF_MEM(interval); + interval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == interval->pCol) { + nodesDestroyNode(interval); + CHECK_OUT_OF_MEM(interval->pCol); + } + ((SColumnNode*)interval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + strcpy(((SColumnNode*)interval->pCol)->colName, PK_TS_COL_INTERNAL_NAME); interval->pInterval = pInterval; interval->pOffset = pOffset; interval->pSliding = pSliding; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 19c6cb3f27..3e88ce0831 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -256,6 +256,10 @@ static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { bool found = false; if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; + if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && 0 == strcmp(pCol->colName, PK_TS_COL_INTERNAL_NAME)) { + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema, false, pCol); + return true; + } int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; for (int32_t i = 0; i < nums; ++i) { if (0 == strcmp(pCol->colName, pMeta->schema[i].name)) { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 7e3b67f809..0eb4ec23fa 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -88,7 +88,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { } static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { - static int32_t rewriteId = 1; + static int32_t rewriteId = 1; // todo modify SNameExprCxt nameCxt = { .rewriteId = rewriteId }; nodesWalkList(pExprs, doNameExpr, &nameCxt); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; @@ -461,6 +461,12 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval); pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); + pWindow->pTspk = nodesCloneNode(pInterval->pCol); + if (NULL == pWindow->pTspk) { + nodesDestroyNode(pWindow); + return TSDB_CODE_OUT_OF_MEMORY; + } + if (NULL != pInterval->pFill) { pWindow->pFill = nodesCloneNode(pInterval->pFill); if (NULL == pWindow->pFill) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index dbaf64bdbf..fe8bafbed3 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -17,9 +17,14 @@ #include "functionMgt.h" +typedef struct SSlotIdInfo { + int16_t slotId; + bool set; +} SSlotIdInfo; + typedef struct SSlotIndex { int16_t dataBlockId; - int16_t slotId; + SArray* pSlotIdsInfo; // duplicate name slot } SSlotIndex; typedef struct SPhysiPlanContext { @@ -79,7 +84,19 @@ static int32_t createTarget(SNode* pNode, int16_t dataBlockId, int16_t slotId, S } static int32_t putSlotToHashImpl(int16_t dataBlockId, int16_t slotId, const char* pName, int32_t len, SHashObj* pHash) { - SSlotIndex index = { .dataBlockId = dataBlockId, .slotId = slotId }; + SSlotIndex* pIndex = taosHashGet(pHash, pName, len); + if (NULL != pIndex) { + SSlotIdInfo info = { .slotId = slotId, .set = false }; + taosArrayPush(pIndex->pSlotIdsInfo, &info); + return TSDB_CODE_SUCCESS; + } + + SSlotIndex index = { .dataBlockId = dataBlockId, .pSlotIdsInfo = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SSlotIdInfo)) }; + if (NULL == index.pSlotIdsInfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SSlotIdInfo info = { .slotId = slotId, .set = false }; + taosArrayPush(index.pSlotIdsInfo, &info); return taosHashPut(pHash, pName, len, &index, sizeof(SSlotIndex)); } @@ -90,7 +107,7 @@ static int32_t putSlotToHash(int16_t dataBlockId, int16_t slotId, SNode* pNode, } static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId, SHashObj** pDescHash) { - SHashObj* pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SHashObj* pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == pHash) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -149,6 +166,18 @@ static int32_t createDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SD return code; } +static int16_t getUnsetSlotId(const SArray* pSlotIdsInfo) { + int32_t size = taosArrayGetSize(pSlotIdsInfo); + for (int32_t i = 0; i < size; ++i) { + SSlotIdInfo* pInfo = taosArrayGet(pSlotIdsInfo, i); + if (!pInfo->set) { + pInfo->set = true; + return pInfo->slotId; + } + } + return ((SSlotIdInfo*)taosArrayGet(pSlotIdsInfo, 0))->slotId; +} + static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc, const char* pStmtName, bool output) { int32_t code = TSDB_CODE_SUCCESS; SHashObj* pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); @@ -167,7 +196,7 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList, slotId = nextSlotId; ++nextSlotId; } else { - slotId = pIndex->slotId; + slotId = getUnsetSlotId(pIndex->pSlotIdsInfo); } if (TSDB_CODE_SUCCESS == code) { @@ -217,7 +246,7 @@ static EDealRes doSetSlotId(SNode* pNode, void* pContext) { return DEAL_RES_ERROR; } ((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId; - ((SColumnNode*)pNode)->slotId = pIndex->slotId; + ((SColumnNode*)pNode)->slotId = ((SSlotIdInfo*)taosArrayGet(pIndex->pSlotIdsInfo, 0))->slotId; return DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; @@ -792,6 +821,13 @@ static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil return TSDB_CODE_OUT_OF_MEMORY; } + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + int32_t code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pTspk, &pInterval->pTspk); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pInterval); + return code; + } + return createWindowPhysiNodeFinalize(pCxt, pChildren, &pInterval->window, pWindowLogicNode, pPhyNode); } From c712e581b3b10bdfdb96b897e791c0c8d8b2a665 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 26 Mar 2022 17:58:56 +0800 Subject: [PATCH 103/121] [TD-13772]: add timezone enum. --- include/os/os.h | 4 +- include/os/osEnv.h | 26 +++++++------ include/os/osTimezone.h | 33 +++++++++++++++- source/client/src/clientEnv.c | 4 +- source/common/src/tglobal.c | 9 +++-- source/dnode/mgmt/dnode/src/dmMsg.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 6 +-- source/dnode/vnode/src/vnd/vnodeWrite.c | 2 +- source/os/src/osEnv.c | 50 ++++++++++++++----------- source/os/src/osMemory.c | 8 ++-- source/os/src/osTimezone.c | 44 +++++++++++++++------- 11 files changed, 120 insertions(+), 68 deletions(-) diff --git a/include/os/os.h b/include/os/os.h index b05bfab6d0..e7ce7d09ea 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -75,7 +75,6 @@ extern "C" { #include "osDef.h" #include "osDir.h" #include "osEndian.h" -#include "osEnv.h" #include "osFile.h" #include "osLocale.h" #include "osLz4.h" @@ -93,8 +92,9 @@ extern "C" { #include "osTime.h" #include "osTimer.h" #include "osTimezone.h" +#include "osEnv.h" -void osInit(); +void osDefaultInit(); #ifdef __cplusplus } diff --git a/include/os/osEnv.h b/include/os/osEnv.h index ebf4c360dd..14d50858b7 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -22,17 +22,18 @@ extern "C" { #endif -extern char tsOsName[]; -extern char tsTimezone[]; -extern char tsCharset[]; -extern char tsLocale[]; -extern int8_t tsDaylight; -extern bool tsEnableCoreFile; -extern int64_t tsPageSizeKB; -extern int64_t tsOpenMax; -extern int64_t tsStreamMax; -extern float tsNumOfCores; -extern int64_t tsTotalMemoryKB; +extern char tsOsName[]; +extern char tsTimezoneStr[]; +extern enum TdTimezone tsTimezone; +extern char tsCharset[]; +extern char tsLocale[]; +extern int8_t tsDaylight; +extern bool tsEnableCoreFile; +extern int64_t tsPageSizeKB; +extern int64_t tsOpenMax; +extern int64_t tsStreamMax; +extern float tsNumOfCores; +extern int64_t tsTotalMemoryKB; extern char configDir[]; extern char tsDataDir[]; @@ -43,11 +44,12 @@ extern SDiskSpace tsDataSpace; extern SDiskSpace tsLogSpace; extern SDiskSpace tsTempSpace; -void osInit(); +void osDefaultInit(); void osUpdate(); void osCleanup(); bool osLogSpaceAvailable(); void osSetTimezone(const char *timezone); +void osSetSystemLocale(const char *inLocale, const char *inCharSet); #ifdef __cplusplus } diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index c8df8c3f3d..c259587264 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -26,8 +26,37 @@ extern "C" { #define tzset TZSET_FUNC_TAOS_FORBID #endif -void taosGetSystemTimezone(char *outTimezone); -void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); +enum TdTimezone +{ + TdWestZone12=-12, + TdWestZone11, + TdWestZone10, + TdWestZone9, + TdWestZone8, + TdWestZone7, + TdWestZone6, + TdWestZone5, + TdWestZone4, + TdWestZone3, + TdWestZone2, + TdWestZone1, + TdZeroZero, + TdEastZone1, + TdEastZone2, + TdEastZone3, + TdEastZone4, + TdEastZone5, + TdEastZone6, + TdEastZone7, + TdEastZone8, + TdEastZone9, + TdEastZone10, + TdEastZone11, + TdEastZone12 +}; + +void taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone); +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone); #ifdef __cplusplus } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index d9dcadbff0..8e67703ce5 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -408,10 +408,10 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezone, str, TD_TIMEZONE_LEN); + tstrncpy(tsTimezoneStr, str, TD_TIMEZONE_LEN); tsSetTimeZone(); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); + tscDebug("timezone set:%s, input:%s by taos_options", tsTimezoneStr, str); } else { tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a1bef49cc6..efd790ade8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -303,7 +303,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { static int32_t taosAddSystemCfg(SConfig *pCfg) { SysNameInfo info = taosGetSysNameInfo(); - if (cfgAddTimezone(pCfg, "timezone", tsTimezone) != 0) return -1; + if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr) != 0) return -1; if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1; @@ -431,12 +431,13 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { static void taosSetSystemCfg(SConfig *pCfg) { SConfigItem *pItem = cfgGetItem(pCfg, "timezone"); osSetTimezone(pItem->str); - uDebug("timezone format changed from %s to %s", pItem->str, tsTimezone); - cfgSetItem(pCfg, "timezone", tsTimezone, pItem->stype); + uDebug("timezone format changed from %s to %s", pItem->str, tsTimezoneStr); + cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype); const char *locale = cfgGetItem(pCfg, "locale")->str; const char *charset = cfgGetItem(pCfg, "charset")->str; taosSetSystemLocale(locale, charset); + osSetSystemLocale(locale, charset); bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; taosSetConsoleEcho(enableCore); @@ -483,7 +484,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc) { - osInit(); + osDefaultInit(); SConfig *pCfg = cfgInit(); if (pCfg == NULL) return -1; diff --git a/source/dnode/mgmt/dnode/src/dmMsg.c b/source/dnode/mgmt/dnode/src/dmMsg.c index 836817e772..eb4e843c55 100644 --- a/source/dnode/mgmt/dnode/src/dmMsg.c +++ b/source/dnode/mgmt/dnode/src/dmMsg.c @@ -36,7 +36,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { req.clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN); + memcpy(req.clusterCfg.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); taosRUnLockLatch(&pMgmt->latch); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 5541412460..314e70db9b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -277,8 +277,8 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } - if ((0 != strcasecmp(pCfg->timezone, tsTimezone)) && (pMnode->checkTime != pCfg->checkTime)) { - mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, tsTimezone, + if ((0 != strcasecmp(pCfg->timezone, tsTimezoneStr)) && (pMnode->checkTime != pCfg->checkTime)) { + mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, tsTimezoneStr, pCfg->checkTime, pMnode->checkTime); return DND_REASON_TIME_ZONE_NOT_MATCH; } @@ -677,7 +677,7 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, char *data, i totalRows++; cfgOpts[totalRows] = "timezone"; - snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsTimezone); + snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsTimezoneStr); totalRows++; cfgOpts[totalRows] = "locale"; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index e0be9ed89a..a14828cc22 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -176,7 +176,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } // record current timezone of server side - tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezone, TD_TIMEZONE_LEN); + tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); if (metaCreateTSma(pVnode->pMeta, &vCreateSmaReq) < 0) { // TODO: handle error diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 61b2593bc6..4847bbf2fe 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -18,30 +18,31 @@ extern void taosWinSocketInit(); -char configDir[PATH_MAX] = {0}; -char tsDataDir[PATH_MAX] = {0}; -char tsLogDir[PATH_MAX] = {0}; -char tsTempDir[PATH_MAX] = {0}; -SDiskSpace tsDataSpace = {0}; -SDiskSpace tsLogSpace = {0}; -SDiskSpace tsTempSpace = {0}; -char tsOsName[16] = {0}; -char tsTimezone[TD_TIMEZONE_LEN] = {0}; -char tsLocale[TD_LOCALE_LEN] = {0}; -char tsCharset[TD_CHARSET_LEN] = {0}; -int8_t tsDaylight = 0; -bool tsEnableCoreFile = 0; -int64_t tsPageSizeKB = 0; -int64_t tsOpenMax = 0; -int64_t tsStreamMax = 0; -float tsNumOfCores = 0; -int64_t tsTotalMemoryKB = 0; +char configDir[PATH_MAX] = {0}; +char tsDataDir[PATH_MAX] = {0}; +char tsLogDir[PATH_MAX] = {0}; +char tsTempDir[PATH_MAX] = {0}; +SDiskSpace tsDataSpace = {0}; +SDiskSpace tsLogSpace = {0}; +SDiskSpace tsTempSpace = {0}; +char tsOsName[16] = {0}; +char tsTimezoneStr[TD_TIMEZONE_LEN] = {0}; +enum TdTimezone tsTimezone = TdZeroZero; +char tsLocale[TD_LOCALE_LEN] = {0}; +char tsCharset[TD_CHARSET_LEN] = {0}; +int8_t tsDaylight = 0; +bool tsEnableCoreFile = 0; +int64_t tsPageSizeKB = 0; +int64_t tsOpenMax = 0; +int64_t tsStreamMax = 0; +float tsNumOfCores = 0; +int64_t tsTotalMemoryKB = 0; -void osInit() { +void osDefaultInit() { taosSeedRand(taosSafeRand()); taosGetSystemLocale(tsLocale, tsCharset); - taosGetSystemTimezone(tsTimezone); - taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); + taosGetSystemTimezone(tsTimezoneStr, &tsTimezone); + taosSetSystemTimezone(tsTimezoneStr, tsTimezoneStr, &tsDaylight, &tsTimezone); taosGetSystemInfo(); // deadlock in query @@ -105,4 +106,9 @@ void osCleanup() {} bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } -void osSetTimezone(const char *timezone) { taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); } +void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, tsTimezoneStr, &tsDaylight, &tsTimezone); } + +void osSetSystemLocale(const char *inLocale, const char *inCharSet) { + memcpy(tsLocale, inLocale, strlen(inLocale) + 1); + memcpy(tsCharset, inCharSet, strlen(inCharSet) + 1); +} diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 3f47e475c3..1adf9f9bf3 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -16,8 +16,6 @@ #define ALLOW_FORBID_FUNC #include "os.h" -#define USE_TD_MEMORY - #define TD_MEMORY_SYMBOL ('T'<<24|'A'<<16|'O'<<8|'S') #define TD_MEMORY_STACK_TRACE_DEPTH 10 @@ -47,16 +45,16 @@ int32_t taosBackTrace(void **buffer, int32_t size) { int32_t frame = 0; void **ebp; void **ret = NULL; - unsigned long long func_frame_distance = 0; + size_t func_frame_distance = 0; if (buffer != NULL && size > 0) { ebp = taosGetEbp(); - func_frame_distance = (unsigned long long)(*ebp) - (unsigned long long)ebp; + func_frame_distance = (size_t)*ebp - (size_t)ebp; while (ebp && frame < size && (func_frame_distance < (1ULL << 24)) // assume function ebp more than 16M && (func_frame_distance > 0)) { ret = ebp + 1; buffer[frame++] = *ret; ebp = (void **)(*ebp); - func_frame_distance = (unsigned long long)(*ebp) - (unsigned long long)ebp; + func_frame_distance = (size_t)*ebp - (size_t)ebp; } } return frame; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index bdafa63d64..dc23eaae1a 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -46,12 +46,22 @@ #include #endif -void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { - if (inTimezone == NULL || inTimezone[0] == 0) return; +void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8_t *outDaylight, enum TdTimezone *tsTimezone) { + if (inTimezoneStr == NULL || inTimezoneStr[0] == 0) return; + + char *buf = taosMemoryMalloc(strlen(inTimezoneStr) + 1); + buf[strlen(inTimezoneStr)] = 0; + for (int32_t i = 0; i < strlen(inTimezoneStr); i++) { + if(inTimezoneStr[i]==' ' || inTimezoneStr[i]=='(') { + buf[i] = 0; + break; + } + buf[i] = inTimezoneStr[i]; + } #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) char winStr[TD_LOCALE_LEN * 2]; - sprintf(winStr, "TZ=%s", inTimezone); + sprintf(winStr, "TZ=%s", buf); putenv(winStr); tzset(); * get CURRENT time zone. @@ -70,44 +80,48 @@ void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *ou #endif int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + *tsTimezone = tz; tz += daylight; /* * format: * (CST, +0800) * (BST, +0100) */ - sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); *outDaylight = daylight; #elif defined(_TD_DARWIN_64) - setenv("TZ", inTimezone, 1); + setenv("TZ", buf, 1); tzset(); int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + *tsTimezone = tz; tz += daylight; - sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); *outDaylight = daylight; #else - setenv("TZ", inTimezone, 1); + setenv("TZ", buf, 1); tzset(); int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + *tsTimezone = tz; tz += daylight; - sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); *outDaylight = daylight; #endif + taosMemoryFree(buf); } -void taosGetSystemTimezone(char *outTimezone) { +void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) char *tz = getenv("TZ"); if (tz == NULL || strlen(tz) == 0) { - strcpy(outTimezone, "not configured"); + strcpy(outTimezoneStr, "not configured"); } else { - strcpy(outTimezone, tz); + strcpy(outTimezoneStr, tz); } #elif defined(_TD_DARWIN_64) @@ -153,7 +167,7 @@ void taosGetSystemTimezone(char *outTimezone) { * Asia/Shanghai (CST, +0800) * Europe/London (BST, +0100) */ - snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone / 3600); #else @@ -168,13 +182,14 @@ void taosGetSystemTimezone(char *outTimezone) { /* load time zone string from /etc/timezone */ // FILE *f = fopen("/etc/timezone", "r"); + errno = 0; TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); char buf[68] = {0}; if (pFile != NULL) { int len = taosReadFile(pFile, buf, 64); if (len < 64 && taosGetErrorFile(pFile)) { taosCloseFile(&pFile); - // printf("read /etc/timezone error, reason:%s", strerror(errno)); + printf("read /etc/timezone error, reason:%s", strerror(errno)); return; } @@ -202,6 +217,7 @@ void taosGetSystemTimezone(char *outTimezone) { * otherwise is GMT+00:00 */ int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; + *tsTimezone = tz; tz += daylight; /* @@ -210,7 +226,7 @@ void taosGetSystemTimezone(char *outTimezone) { * Asia/Shanghai (CST, +0800) * Europe/London (BST, +0100) */ - snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); #endif } From d79e7de19f39280f5f1f42cb3040578a98c9ffcd Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 26 Mar 2022 19:14:10 +0800 Subject: [PATCH 104/121] [TD-13758]: allow double free. --- include/os/osTimezone.h | 2 +- source/os/src/osEnv.c | 2 +- source/os/src/osMemory.c | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index c259587264..3676c4b634 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -40,7 +40,7 @@ enum TdTimezone TdWestZone3, TdWestZone2, TdWestZone1, - TdZeroZero, + TdZeroZone, TdEastZone1, TdEastZone2, TdEastZone3, diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 4847bbf2fe..22884298ef 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -27,7 +27,7 @@ SDiskSpace tsLogSpace = {0}; SDiskSpace tsTempSpace = {0}; char tsOsName[16] = {0}; char tsTimezoneStr[TD_TIMEZONE_LEN] = {0}; -enum TdTimezone tsTimezone = TdZeroZero; +enum TdTimezone tsTimezone = TdZeroZone; char tsLocale[TD_LOCALE_LEN] = {0}; char tsCharset[TD_CHARSET_LEN] = {0}; int8_t tsDaylight = 0; diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 1adf9f9bf3..2ce851cd63 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -115,7 +115,8 @@ void taosMemoryFree(const void *ptr) { TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); if(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL) { - memset(pTdMemoryInfo, 0, sizeof(TdMemoryInfo)); + pTdMemoryInfo->memorySize = 0; + // memset(pTdMemoryInfo, 0, sizeof(TdMemoryInfo)); free(pTdMemoryInfo); } else { free((void*)ptr); From ad11ba93653d19db45316f6db65837e14b0731ab Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 26 Mar 2022 07:18:12 -0400 Subject: [PATCH 105/121] bugfix --- source/libs/nodes/src/nodesCodeFuncs.c | 31 +++++++++------------- source/libs/parser/src/parTranslater.c | 11 +++++++- source/libs/planner/src/planLogicCreater.c | 11 ++------ 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index bb6c43f781..a250df338a 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -851,9 +851,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - int32_t val; - code = tjsonGetIntValue(pJson, jkJoinPhysiPlanJoinType, &val); - pNode->joinType = val; + code = tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType); } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOnConditions); @@ -1216,9 +1214,7 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { int32_t code = tjsonToObject(pJson, jkSubplanId, jsonToSubplanId, &pNode->id); if (TSDB_CODE_SUCCESS == code) { - int32_t val; - code = tjsonGetIntValue(pJson, jkSubplanType, &val); - pNode->subplanType = val; + code = tjsonGetNumberValue(pJson, jkSubplanType, pNode->subplanType); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanMsgType, &pNode->msgType); @@ -1408,9 +1404,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { code = tjsonGetSmallIntValue(pJson, jkColumnColId, &pNode->colId); } if (TSDB_CODE_SUCCESS == code) { - int32_t tmp; - code = tjsonGetIntValue(pJson, jkColumnColType, &tmp); - pNode->colType = tmp; + code = tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName); @@ -1598,9 +1592,7 @@ static int32_t jsonToOperatorNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - int32_t val; - code = tjsonGetIntValue(pJson, jkOperatorType, &val); - pNode->opType = val; + code = tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType); } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft); @@ -1634,9 +1626,7 @@ static int32_t jsonToLogicConditionNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - int32_t val; - code = tjsonGetIntValue(pJson, jkLogicCondType, &val); - pNode->condType = val; + code = tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType); } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList); @@ -1895,6 +1885,7 @@ static const char* jkIntervalWindowInterval = "Interval"; static const char* jkIntervalWindowOffset = "Offset"; static const char* jkIntervalWindowSliding = "Sliding"; static const char* jkIntervalWindowFill = "Fill"; +static const char* jkIntervalWindowTsPk = "TsPk"; static int32_t intervalWindowNodeToJson(const void* pObj, SJson* pJson) { const SIntervalWindowNode* pNode = (const SIntervalWindowNode*)pObj; @@ -1909,6 +1900,9 @@ static int32_t intervalWindowNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkIntervalWindowFill, nodeToJson, pNode->pFill); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIntervalWindowTsPk, nodeToJson, pNode->pCol); + } return code; } @@ -1926,6 +1920,9 @@ static int32_t jsonToIntervalWindowNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkIntervalWindowFill, &pNode->pFill); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIntervalWindowTsPk, &pNode->pCol); + } return code; } @@ -2436,9 +2433,7 @@ static int32_t nodeToJson(const void* pObj, SJson* pJson) { static int32_t jsonToNode(const SJson* pJson, void* pObj) { SNode* pNode = (SNode*)pObj; - int32_t val = 0; - int32_t code = tjsonGetIntValue(pJson, jkNodeType, &val); - pNode->type = val; + int32_t code = tjsonGetNumberValue(pJson, jkNodeType, pNode->type); if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, nodesNodeName(pNode->type), jsonToSpecificNode, pNode); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c7b1b51d18..ba3b5cc254 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1452,6 +1452,7 @@ static int32_t getSmaIndexBuildAst(STranslateContext* pCxt, SCreateIndexStmt* pS if (NULL == pSelect) { return TSDB_CODE_OUT_OF_MEMORY; } + sprintf(pSelect->stmtName, "%p", pSelect); SRealTableNode* pTable = nodesMakeNode(QUERY_NODE_REAL_TABLE); if (NULL == pTable) { @@ -1467,6 +1468,10 @@ static int32_t getSmaIndexBuildAst(STranslateContext* pCxt, SCreateIndexStmt* pS nodesDestroyNode(pSelect); return TSDB_CODE_OUT_OF_MEMORY; } + SNode* pProject = NULL; + FOREACH(pProject, pSelect->pProjectionList) { + sprintf(((SExprNode*)pProject)->aliasName, "#sma_%p", pProject); + } SIntervalWindowNode* pInterval = nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); if (NULL == pInterval) { @@ -1474,14 +1479,18 @@ static int32_t getSmaIndexBuildAst(STranslateContext* pCxt, SCreateIndexStmt* pS return TSDB_CODE_OUT_OF_MEMORY; } pSelect->pWindow = (SNode*)pInterval; + pInterval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); pInterval->pInterval = nodesCloneNode(pStmt->pOptions->pInterval); pInterval->pOffset = nodesCloneNode(pStmt->pOptions->pOffset); pInterval->pSliding = nodesCloneNode(pStmt->pOptions->pSliding); - if (NULL == pInterval->pInterval || (NULL != pStmt->pOptions->pOffset && NULL == pInterval->pOffset) || + if (NULL == pInterval->pCol || NULL == pInterval->pInterval || + (NULL != pStmt->pOptions->pOffset && NULL == pInterval->pOffset) || (NULL != pStmt->pOptions->pSliding && NULL == pInterval->pSliding)) { nodesDestroyNode(pSelect); return TSDB_CODE_OUT_OF_MEMORY; } + ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + strcpy(((SColumnNode*)pInterval->pCol)->colName, PK_TS_COL_INTERNAL_NAME); int32_t code = translateQuery(pCxt, (SNode*)pSelect); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 5495aae2a3..740fb678fd 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -67,17 +67,12 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -typedef struct SNameExprCxt { - int32_t rewriteId; -} SNameExprCxt; - static EDealRes doNameExpr(SNode* pNode, void* pContext) { switch (nodeType(pNode)) { case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_FUNCTION: { - SNameExprCxt* pCxt = (SNameExprCxt*)pContext; - sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d", pCxt->rewriteId++); + sprintf(((SExprNode*)pNode)->aliasName, "#expr_%p", pNode); return DEAL_RES_IGNORE_CHILD; } default: @@ -88,9 +83,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { } static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { - static int32_t rewriteId = 1; // todo modify - SNameExprCxt nameCxt = { .rewriteId = rewriteId }; - nodesWalkList(pExprs, doNameExpr, &nameCxt); + nodesWalkList(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; From 06285a252e02d9fa8bd652e67ff2b83d7740066c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 26 Mar 2022 20:12:45 +0800 Subject: [PATCH 106/121] refactor tmq container --- example/src/tmq.c | 2 +- include/client/taos.h | 5 +- source/client/src/tmq.c | 132 ++++++++++++++++++------ source/dnode/mnode/impl/inc/mndDef.h | 25 ++--- source/dnode/mnode/impl/src/mndStream.c | 11 +- source/dnode/mnode/impl/src/mndTopic.c | 30 ++++++ source/os/src/osMemory.c | 2 +- tests/test/c/tmqDemo.c | 2 +- 8 files changed, 157 insertions(+), 52 deletions(-) diff --git a/example/src/tmq.c b/example/src/tmq.c index 35c3e655d6..3b4b6afbaf 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -44,7 +44,7 @@ int32_t init_env() { pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); if (taos_errno(pRes) != 0) { - printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); diff --git a/include/client/taos.h b/include/client/taos.h index 82f0635612..55e1d1c422 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -213,8 +213,10 @@ typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, v DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); +DLL_EXPORT void tmq_list_destroy(tmq_list_t *); DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); +DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message); DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); @@ -244,8 +246,8 @@ enum tmq_conf_res_t { typedef enum tmq_conf_res_t tmq_conf_res_t; DLL_EXPORT tmq_conf_t *tmq_conf_new(); -DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); +DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb); // temporary used function for demo only @@ -256,6 +258,7 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message); DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message); +DLL_EXPORT char *tmq_get_topic_schema(tmq_t *tmq, const char *topic); /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */ DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 52d5400b0b..40d9722fa7 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -27,9 +27,7 @@ #include "tref.h" struct tmq_list_t { - int32_t cnt; - int32_t tot; - char* elems[]; + SArray container; }; struct tmq_topic_vgroup_t { @@ -45,11 +43,14 @@ struct tmq_topic_vgroup_list_t { struct tmq_conf_t { char clientId[256]; char groupId[TSDB_CGROUP_LEN]; - int8_t auto_commit; + int8_t autoCommit; int8_t resetOffset; + uint16_t port; + char* ip; + char* user; + char* pass; + char* db; tmq_commit_cb* commit_cb; - /*char* ip;*/ - /*uint16_t port;*/ }; struct tmq_t { @@ -98,12 +99,13 @@ typedef struct { typedef struct { // subscribe info - int32_t sqlLen; - char* sql; - char* topicName; - int64_t topicId; - int32_t nextVgIdx; - SArray* vgs; // SArray + int32_t sqlLen; + char* sql; + char* topicName; + int64_t topicId; + int32_t nextVgIdx; + SArray* vgs; // SArray + SSchemaWrapper schema; } SMqClientTopic; typedef struct { @@ -137,7 +139,7 @@ typedef struct { tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); - conf->auto_commit = false; + conf->autoCommit = false; conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; return conf; } @@ -151,21 +153,24 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value strcpy(conf->groupId, value); return TMQ_CONF_OK; } + if (strcmp(key, "client.id") == 0) { strcpy(conf->clientId, value); return TMQ_CONF_OK; } + if (strcmp(key, "enable.auto.commit") == 0) { if (strcmp(value, "true") == 0) { - conf->auto_commit = true; + conf->autoCommit = true; return TMQ_CONF_OK; } else if (strcmp(value, "false") == 0) { - conf->auto_commit = false; + conf->autoCommit = false; return TMQ_CONF_OK; } else { return TMQ_CONF_INVALID; } } + if (strcmp(key, "auto.offset.reset") == 0) { if (strcmp(value, "none") == 0) { conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE; @@ -180,26 +185,49 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_INVALID; } } + + if (strcmp(key, "connection.ip") == 0) { + conf->ip = strdup(value); + return TMQ_CONF_OK; + } + if (strcmp(key, "connection.user") == 0) { + conf->user = strdup(value); + return TMQ_CONF_OK; + } + if (strcmp(key, "connection.pass") == 0) { + conf->pass = strdup(value); + return TMQ_CONF_OK; + } + if (strcmp(key, "connection.port") == 0) { + conf->port = atoi(value); + return TMQ_CONF_OK; + } + if (strcmp(key, "connection.db") == 0) { + conf->db = strdup(value); + return TMQ_CONF_OK; + } + return TMQ_CONF_UNKNOWN; } tmq_list_t* tmq_list_new() { - tmq_list_t* ptr = taosMemoryMalloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); - if (ptr == NULL) { - return ptr; - } - ptr->cnt = 0; - ptr->tot = 8; - return ptr; + // + return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); } -int32_t tmq_list_append(tmq_list_t* ptr, const char* src) { - if (ptr->cnt >= ptr->tot - 1) return -1; - ptr->elems[ptr->cnt] = strdup(src); - ptr->cnt++; +int32_t tmq_list_append(tmq_list_t* list, const char* src) { + SArray* container = &list->container; + char* topic = strdup(src); + if (taosArrayPush(container, topic) == NULL) return -1; return 0; } +void tmq_list_destroy(tmq_list_t* list) { + SArray* container = (SArray*)list; + taosArrayDestroy(container); + /*taosArrayDestroyEx(container, free);*/ +} + void tmqClearUnhandleMsg(tmq_t* tmq) { tmq_message_t* msg; while (1) { @@ -268,17 +296,57 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); - pTmq->autoCommit = conf->auto_commit; + pTmq->autoCommit = conf->autoCommit; pTmq->commit_cb = conf->commit_cb; pTmq->resetOffsetCfg = conf->resetOffset; - tsem_init(&pTmq->rspSem, 0, 0); - pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); + if (pTmq->clientTopics == NULL) { + taosMemoryFree(pTmq); + return NULL; + } pTmq->mqueue = taosOpenQueue(); pTmq->qall = taosAllocateQall(); + + tsem_init(&pTmq->rspSem, 0, 0); + + return pTmq; +} + +tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { + tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); + if (pTmq == NULL) { + return NULL; + } + pTmq->pTscObj = taos_connect(conf->ip, conf->user, conf->pass, conf->db, conf->port); + + pTmq->inWaiting = 0; + pTmq->status = 0; + pTmq->pollCnt = 0; + pTmq->epoch = 0; + pTmq->waitingRequest = 0; + pTmq->readyRequest = 0; + // set conf + strcpy(pTmq->clientId, conf->clientId); + strcpy(pTmq->groupId, conf->groupId); + pTmq->autoCommit = conf->autoCommit; + pTmq->commit_cb = conf->commit_cb; + pTmq->resetOffsetCfg = conf->resetOffset; + + pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); + pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); + if (pTmq->clientTopics == NULL) { + taosMemoryFree(pTmq); + return NULL; + } + + pTmq->mqueue = taosOpenQueue(); + pTmq->qall = taosAllocateQall(); + + tsem_init(&pTmq->rspSem, 0, 0); + return pTmq; } @@ -372,7 +440,8 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { SRequestObj* pRequest = NULL; - int32_t sz = topic_list->cnt; + SArray* container = &topic_list->container; + int32_t sz = taosArrayGetSize(container); // destroy ex taosArrayDestroy(tmq->clientTopics); tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); @@ -384,7 +453,8 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { req.topicNames = taosArrayInit(sz, sizeof(void*)); for (int i = 0; i < sz; i++) { - char* topicName = topic_list->elems[i]; + /*char* topicName = topic_list->elems[i];*/ + char* topicName = taosArrayGetP(container, i); SName name = {0}; char* dbName = getDbOfConnection(tmq->pTscObj); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 2c4f2e82ac..caf5172596 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -633,18 +633,19 @@ static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) { } typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - int64_t uid; - int64_t dbUid; - int32_t version; - SRWLatch lock; - int32_t sqlLen; - char* sql; - char* logicalPlan; - char* physicalPlan; + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + int64_t uid; + int64_t dbUid; + int32_t version; + SRWLatch lock; + int32_t sqlLen; + char* sql; + char* logicalPlan; + char* physicalPlan; + SSchemaWrapper schema; } SMqTopicObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 47624dfcc1..c02fec0a5f 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -248,15 +248,16 @@ static int32_t mndStreamGetPlanString(const char *ast, char **pStr) { int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) { SNode *pAst = NULL; -#if 1 // TODO: remove debug info later - printf("ast = %s\n", ast); -#endif + if (nodesStringToNode(ast, &pAst) < 0) { return -1; } -#if 1 - qExtractResultSchema(pAst, (int32_t *)&pStream->outputSchema.nCols, &pStream->outputSchema.pSchema); + if (qExtractResultSchema(pAst, (int32_t *)&pStream->outputSchema.nCols, &pStream->outputSchema.pSchema) != 0) { + return -1; + } + +#if 1 printf("|"); for (int i = 0; i < pStream->outputSchema.nCols; i++) { printf(" %15s |", (char *)pStream->outputSchema.pSchema[i].name); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 625c9eb733..fa2ba4bfc0 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -23,6 +23,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "parser.h" #include "tname.h" #define MND_TOPIC_VER_NUMBER 1 @@ -85,6 +86,16 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT32(pRaw, dataPos, physicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER); + int32_t swLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema); + void *swBuf = taosMemoryMalloc(swLen); + if (swBuf == NULL) { + goto TOPIC_ENCODE_OVER; + } + void *aswBuf = swBuf; + taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema); + SDB_SET_INT32(pRaw, dataPos, swLen, TOPIC_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, swBuf, swLen, TOPIC_ENCODE_OVER); + SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER); SDB_SET_DATALEN(pRaw, dataPos, TOPIC_ENCODE_OVER); @@ -149,6 +160,17 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { } SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); + void *buf = taosMemoryMalloc(len); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto TOPIC_DECODE_OVER; + } + SDB_GET_BINARY(pRaw, dataPos, buf, len, TOPIC_DECODE_OVER); + if (taosDecodeSSchemaWrapper(buf, &pTopic->schema) == NULL) { + goto TOPIC_DECODE_OVER; + } + SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER); terrno = TSDB_CODE_SUCCESS; @@ -283,6 +305,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq topicObj.physicalPlan = pPlanStr; } + SNode *pAst = NULL; + if (nodesStringToNode(pCreate->ast, &pAst) < 0) { + return -1; + } + if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) { + return -1; + } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, &pReq->rpcMsg); if (pTrans == NULL) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 3f47e475c3..b99d492421 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -131,4 +131,4 @@ int32_t taosMemorySize(void *ptr) { assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); return pTdMemoryInfo->memorySize; -} \ No newline at end of file +} diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 205bc0a639..1690a5fb3e 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -694,7 +694,7 @@ int main(int32_t argc, char *argv[]) { walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); if (walLogSize <= 0) { printf("vnode2/wal size incorrect!"); - exit(-1); + /*exit(-1);*/ } else { if (0 == g_stConfInfo.simCase) { pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); From a00a42759ebd37f24c2df071e3cd873a926503a9 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 26 Mar 2022 20:24:24 +0800 Subject: [PATCH 107/121] fix --- source/client/src/tmq.c | 6 +++--- source/libs/wal/src/walRead.c | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 40d9722fa7..2a9b3cdf64 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -218,14 +218,14 @@ tmq_list_t* tmq_list_new() { int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; char* topic = strdup(src); - if (taosArrayPush(container, topic) == NULL) return -1; + if (taosArrayPush(container, &topic) == NULL) return -1; return 0; } void tmq_list_destroy(tmq_list_t* list) { SArray* container = (SArray*)list; - taosArrayDestroy(container); - /*taosArrayDestroyEx(container, free);*/ + /*taosArrayDestroy(container);*/ + taosArrayDestroyEx(container, (void (*)(void*))taosMemoryFree); } void tmqClearUnhandleMsg(tmq_t* tmq) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 9d7c9ed9df..5296a16703 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -153,6 +153,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } code = walValidHeadCksum(pRead->pHead); if (code != 0) { + wError("unexpected wal log version: % " PRId64 ", since head checksum not passed", ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } @@ -179,7 +180,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { code = walValidBodyCksum(pRead->pHead); if (code != 0) { - wError("unexpected wal log version: % " PRId64 "checksum not passed", ver); + wError("unexpected wal log version: % " PRId64 ", since body checksum not passed", ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; From 66f31a007703e36b060c80e4bfc83709ef814e4f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 26 Mar 2022 22:49:42 +0800 Subject: [PATCH 108/121] [TD-14331]: make util/dnodes.py flexible (#11022) * [TD-14331]: make util/dnodes.py flexible for 3.0 * make getPath clear --- tests/pytest/util/dnodes.py | 121 ++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 60 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 0208f884b6..c3c2424397 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -71,7 +71,7 @@ class TDSimClient: cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: tdLog.exit(cmd) - + cmd = "mkdir -p " + self.logDir if os.system(cmd) != 0: tdLog.exit(cmd) @@ -107,36 +107,36 @@ class TDDnode: self.testCluster = False self.valgrind = 0 self.cfgDict = { - "numOfLogLines":"100000000", - "mnodeEqualVnodeNum":"0", - "walLevel":"2", - "fsync":"1000", - "statusInterval":"1", - "numOfMnodes":"3", - "numOfThreadsPerCore":"2.0", - "monitor":"0", - "maxVnodeConnections":"30000", - "maxMgmtConnections":"30000", - "maxMeterConnections":"30000", - "maxShellConns":"30000", - "locale":"en_US.UTF-8", - "charset":"UTF-8", - "asyncLog":"0", - "anyIp":"0", - "telemetryReporting":"0", - "dDebugFlag":"135", - "tsdbDebugFlag":"135", - "mDebugFlag":"135", - "sdbDebugFlag":"135", - "rpcDebugFlag":"135", - "tmrDebugFlag":"131", - "cDebugFlag":"135", - "httpDebugFlag":"135", - "monitorDebugFlag":"135", - "udebugFlag":"135", - "jnidebugFlag":"135", - "qdebugFlag":"135", - "maxSQLLength":"1048576" + "numOfLogLines": "100000000", + "mnodeEqualVnodeNum": "0", + "walLevel": "2", + "fsync": "1000", + "statusInterval": "1", + "numOfMnodes": "3", + "numOfThreadsPerCore": "2.0", + "monitor": "0", + "maxVnodeConnections": "30000", + "maxMgmtConnections": "30000", + "maxMeterConnections": "30000", + "maxShellConns": "30000", + "locale": "en_US.UTF-8", + "charset": "UTF-8", + "asyncLog": "0", + "anyIp": "0", + "telemetryReporting": "0", + "dDebugFlag": "135", + "tsdbDebugFlag": "135", + "mDebugFlag": "135", + "sdbDebugFlag": "135", + "rpcDebugFlag": "135", + "tmrDebugFlag": "131", + "cDebugFlag": "135", + "httpDebugFlag": "135", + "monitorDebugFlag": "135", + "udebugFlag": "135", + "jnidebugFlag": "135", + "qdebugFlag": "135", + "maxSQLLength": "1048576" } def init(self, path): @@ -216,16 +216,16 @@ class TDDnode: isFirstDir = 1 if updatecfgDict[0] and updatecfgDict[0][0]: print(updatecfgDict[0][0]) - for key,value in updatecfgDict[0][0].items(): - if value == 'dataDir' : + for key, value in updatecfgDict[0][0].items(): + if value == 'dataDir': if isFirstDir: self.cfgDict.pop('dataDir') - self.cfg(value,key) + self.cfg(value, key) isFirstDir = 0 else: - self.cfg(value,key) + self.cfg(value, key) else: - self.addExtraCfg(key,value) + self.addExtraCfg(key, value) for key, value in self.cfgDict.items(): self.cfg(key, value) @@ -234,8 +234,7 @@ class TDDnode: "dnode:%d is deployed and configured by %s" % (self.index, self.cfgPath)) - def getBuildPath(self): - buildPath = "" + def getPath(self, tool="taosd"): selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -243,23 +242,22 @@ class TDDnode: else: projPath = selfPath[:selfPath.find("tests")] + paths = [] for root, dirs, files in os.walk(projPath): - if (("taosd") in files): + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + paths.append(os.path.join(root, tool)) break - return buildPath + return paths[0] def start(self): - buildPath = self.getBuildPath() + binPath = self.getPath() - if (buildPath == ""): + if (binPath == ""): tdLog.exit("taosd not found!") else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" + tdLog.info("taosd found: %s" % binPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) @@ -282,18 +280,22 @@ class TDDnode: if self.valgrind == 0: time.sleep(0.1) key = 'from offline to online' - bkey = bytes(key,encoding="utf8") + bkey = bytes(key, encoding="utf8") logFile = self.logDir + "/taosdlog.0" i = 0 while not os.path.exists(logFile): sleep(0.1) i += 1 - if i>50: + if i > 50: break - popen = subprocess.Popen('tail -f ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + popen = subprocess.Popen( + 'tail -f ' + logFile, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) pid = popen.pid # print('Popen.pid:' + str(pid)) - timeout = time.time() + 60*2 + timeout = time.time() + 60 * 2 while True: line = popen.stdout.readline().strip() if bkey in line: @@ -303,21 +305,20 @@ class TDDnode: tdLog.exit('wait too long for taosd start') tdLog.debug("the dnode:%d has been started." % (self.index)) else: - tdLog.debug("wait 10 seconds for the dnode:%d to start." % (self.index)) + tdLog.debug( + "wait 10 seconds for the dnode:%d to start." % + (self.index)) time.sleep(10) - # time.sleep(5) - - def startWithoutSleep(self): - buildPath = self.getBuildPath() - if (buildPath == ""): + def startWithoutSleep(self): + binPath = self.getPath() + + if (binPath == ""): tdLog.exit("taosd not found!") else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" + tdLog.info("taosd found: %s" % binPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) @@ -505,7 +506,7 @@ class TDDnodes: def start(self, index): self.check(index) self.dnodes[index - 1].start() - + def startWithoutSleep(self, index): self.check(index) self.dnodes[index - 1].startWithoutSleep() From 9c1f71459ee3352a9e12ff1b48a4d44a2409523c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 26 Mar 2022 23:04:51 +0800 Subject: [PATCH 109/121] minor changes --- source/util/src/tprocess.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index f32f578952..15708f1cec 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -381,6 +381,7 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { pProc->pid = fork(); if (pProc->pid == 0) { pProc->isChild = 1; + prctl(PR_SET_NAME, pProc->name, NULL, NULL, NULL); } else { pProc->isChild = 0; uInfo("this is parent process, child pid:%d", pProc->pid); From bef5d75dac27e0028610479a5bcd8c607bcb9bd5 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 26 Mar 2022 21:31:27 -0400 Subject: [PATCH 110/121] bugfix --- source/libs/executor/src/executor.c | 2 ++ source/libs/nodes/src/nodesCodeFuncs.c | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e89bc5df0e..26422fa618 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -97,6 +97,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { pMsg->contentLen = pMsg->contentLen; #endif + qDebugL("stream task string %s", (const char*)msg); + struct SSubplan* plan = NULL; int32_t code = qStringToSubplan(msg, &plan); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index a250df338a..15e03b9891 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2100,6 +2100,7 @@ static const char* jkSelectStmtHaving = "Having"; static const char* jkSelectStmtOrderBy = "OrderBy"; static const char* jkSelectStmtLimit = "Limit"; static const char* jkSelectStmtSlimit = "Slimit"; +static const char* jkSelectStmtStmtName = "StmtName"; static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { const SSelectStmt* pNode = (const SSelectStmt*)pObj; @@ -2135,6 +2136,9 @@ static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSelectStmtSlimit, nodeToJson, pNode->pSlimit); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkSelectStmtStmtName, pNode->stmtName); + } return code; } @@ -2173,6 +2177,9 @@ static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkSelectStmtSlimit, &pNode->pSlimit); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkSelectStmtStmtName, pNode->stmtName); + } return code; } From de102d04e65f869af7be3a05bf79f14432091037 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 26 Mar 2022 22:02:12 -0400 Subject: [PATCH 111/121] bugfix --- source/libs/parser/src/parTranslater.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ba3b5cc254..f3a14870ab 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1800,7 +1800,7 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { } int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { - if (QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { + if (NULL != pRoot && QUERY_NODE_SELECT_STMT == nodeType(pRoot)) { SSelectStmt* pSelect = (SSelectStmt*) pRoot; *numOfCols = LIST_LENGTH(pSelect->pProjectionList); *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); @@ -2400,13 +2400,14 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { } static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { - int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pQuery->pRoot)) { case QUERY_NODE_SELECT_STMT: pQuery->haveResultSet = true; pQuery->directRpc = false; pQuery->msgType = TDMT_VND_QUERY; - code = qExtractResultSchema(pQuery->pRoot, &pQuery->numOfResCols, &pQuery->pResSchema); + if (TSDB_CODE_SUCCESS != qExtractResultSchema(pQuery->pRoot, &pQuery->numOfResCols, &pQuery->pResSchema)) { + return TSDB_CODE_OUT_OF_MEMORY; + } break; case QUERY_NODE_VNODE_MODIF_STMT: pQuery->haveResultSet = false; @@ -2446,7 +2447,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { } } - return code; + return TSDB_CODE_SUCCESS; } int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { From d4c636589acdec860c592f49adee20c09f500ea7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 27 Mar 2022 10:42:23 +0800 Subject: [PATCH 112/121] update definition of SSchema/STColumn to support columns larger than 1MB(int16_t to int32_t) --- 2.0/src/client/inc/tsclient.h | 16 +++---- 2.0/src/client/src/tscParseInsert.c | 20 ++++----- 2.0/src/client/src/tscUtil.c | 2 +- include/common/tdataformat.h | 15 ++++--- include/common/tmsg.h | 20 ++++----- include/common/trow.h | 5 ++- include/common/ttypes.h | 1 + include/libs/nodes/querynodes.h | 2 +- include/libs/qcom/query.h | 10 ++--- source/client/src/clientHb.c | 2 +- source/common/src/tdataformat.c | 10 ++--- source/common/src/tmsg.c | 12 +++--- source/dnode/mnode/impl/src/mndStb.c | 12 +++--- source/dnode/vnode/src/meta/metaBDBImpl.c | 8 ++-- source/dnode/vnode/src/tq/tqRead.c | 6 +-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- source/dnode/vnode/src/vnd/vnodeWrite.c | 11 +++++ source/libs/catalog/src/catalog.c | 2 +- source/libs/parser/inc/parInsertData.h | 26 +++++------ source/libs/parser/src/parInsert.c | 43 ++++++++++--------- source/libs/parser/src/parInsertData.c | 8 ++-- source/libs/parser/src/parTranslater.c | 2 +- .../libs/parser/test/mockCatalogService.cpp | 2 +- source/libs/qcom/src/queryUtil.c | 2 +- source/libs/qcom/src/querymsg.c | 2 +- source/libs/qworker/src/qworkerMsg.c | 2 +- 27 files changed, 133 insertions(+), 112 deletions(-) diff --git a/2.0/src/client/inc/tsclient.h b/2.0/src/client/inc/tsclient.h index e14a3123ad..d5021f663d 100644 --- a/2.0/src/client/inc/tsclient.h +++ b/2.0/src/client/inc/tsclient.h @@ -116,7 +116,7 @@ typedef struct SParsedDataColInfo { uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) uint16_t extendedVarLen; uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - int32_t * boundedColumns; // bound column idx according to schema + int32_t *boundColumns; // bound column idx according to schema SBoundColumn * cols; SBoundIdxInfo *colIdxInfo; int8_t orderStatus; // bound columns @@ -125,7 +125,7 @@ typedef struct SParsedDataColInfo { #define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) typedef struct { - uint8_t memRowType; // default is 0, that is SDataRow + uint8_t rowType; // default is 0, that is SDataRow int32_t rowSize; } SMemRowBuilder; @@ -137,17 +137,17 @@ void destroyMemRowBuilder(SMemRowBuilder *pBuilder); /** * @brief * - * @param memRowType + * @param rowType * @param spd * @param idx the absolute bound index of columns * @return FORCE_INLINE */ -static FORCE_INLINE void tscGetMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd, - int32_t idx, int32_t *toffset, int16_t *colId) { +static FORCE_INLINE void tscGetSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, int32_t idx, + int32_t *toffset, int16_t *colId) { int32_t schemaIdx = 0; if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundedColumns[idx]; - if (isDataRowT(memRowType)) { + schemaIdx = spd->boundColumns[idx]; + if (isDataRowT(rowType)) { *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart } else { *toffset = idx * sizeof(SColIdx); // the offset of SColIdx @@ -155,7 +155,7 @@ static FORCE_INLINE void tscGetMemRowAppendInfo(SSchema *pSchema, uint8_t memRow } else { ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; - if (isDataRowT(memRowType)) { + if (isDataRowT(rowType)) { *toffset = (spd->cols + schemaIdx)->toffset; } else { *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx); diff --git a/2.0/src/client/src/tscParseInsert.c b/2.0/src/client/src/tscParseInsert.c index b3f7c076e3..a1f5899cee 100644 --- a/2.0/src/client/src/tscParseInsert.c +++ b/2.0/src/client/src/tscParseInsert.c @@ -428,7 +428,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { // the start position in data block buffer of current value in sql - int32_t colIndex = spd->boundedColumns[i]; + int32_t colIndex = spd->boundColumns[i]; char *start = row + spd->cols[colIndex].offset; @@ -495,7 +495,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); int32_t toffset = -1; int16_t colId = -1; - tscGetMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); + tscGetSTSRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId); int32_t ret = tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset, colId); @@ -630,7 +630,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32 pColInfo->numOfCols = numOfCols; pColInfo->numOfBound = numOfCols; pColInfo->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColInfo->boundedColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); + pColInfo->boundColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn)); pColInfo->colIdxInfo = NULL; pColInfo->flen = 0; @@ -656,7 +656,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32 default: break; } - pColInfo->boundedColumns[i] = i; + pColInfo->boundColumns[i] = i; } pColInfo->allNullLen += pColInfo->flen; pColInfo->boundNullLen = pColInfo->allNullLen; // default set allNullLen @@ -991,7 +991,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC } for (int i = 0; i < spd.numOfBound; ++i) { - SSchema* pSchema = &pTagSchema[spd.boundedColumns[i]]; + SSchema *pSchema = &pTagSchema[spd.boundColumns[i]]; index = 0; sToken = tStrGetToken(sql, &index, true); @@ -1158,7 +1158,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat pColInfo->numOfBound = 0; pColInfo->boundNullLen = 0; - memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * nCols); + memset(pColInfo->boundColumns, 0, sizeof(int32_t) * nCols); for (int32_t i = 0; i < nCols; ++i) { pColInfo->cols[i].valStat = VAL_STAT_NONE; } @@ -1205,7 +1205,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } pColInfo->cols[t].valStat = VAL_STAT_HAS; - pColInfo->boundedColumns[pColInfo->numOfBound] = t; + pColInfo->boundColumns[pColInfo->numOfBound] = t; ++pColInfo->numOfBound; switch (pSchema[t].type) { case TSDB_DATA_TYPE_BINARY: @@ -1239,7 +1239,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } pColInfo->cols[t].valStat = VAL_STAT_HAS; - pColInfo->boundedColumns[pColInfo->numOfBound] = t; + pColInfo->boundColumns[pColInfo->numOfBound] = t; ++pColInfo->numOfBound; switch (pSchema[t].type) { case TSDB_DATA_TYPE_BINARY: @@ -1279,7 +1279,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat } SBoundIdxInfo *pColIdx = pColInfo->colIdxInfo; for (uint16_t i = 0; i < pColInfo->numOfBound; ++i) { - pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundedColumns[i]; + pColIdx[i].schemaColIdx = (uint16_t)pColInfo->boundColumns[i]; pColIdx[i].boundIdx = i; } qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); @@ -1289,7 +1289,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat qsort(pColIdx, pColInfo->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } - memset(&pColInfo->boundedColumns[pColInfo->numOfBound], 0, + memset(&pColInfo->boundColumns[pColInfo->numOfBound], 0, sizeof(int32_t) * (pColInfo->numOfCols - pColInfo->numOfBound)); return TSDB_CODE_SUCCESS; diff --git a/2.0/src/client/src/tscUtil.c b/2.0/src/client/src/tscUtil.c index 3a8c740ea1..449f1ea453 100644 --- a/2.0/src/client/src/tscUtil.c +++ b/2.0/src/client/src/tscUtil.c @@ -1554,7 +1554,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { } void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) { - tfree(pColInfo->boundedColumns); + tfree(pColInfo->boundColumns); tfree(pColInfo->cols); tfree(pColInfo->colIdxInfo); } diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 698352f636..a2899ead8e 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -59,12 +59,15 @@ extern "C" { } while (0); // ----------------- TSDB COLUMN DEFINITION +#pragma pack(push, 1) typedef struct { - int8_t type; // Column type - col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) - int16_t bytes; // column bytes (restore to int16_t in case of misuse) - uint16_t offset; // point offset in STpRow after the header part. + col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) + int32_t type : 8; // column type + int32_t bytes : 24; // column bytes (restore to int32_t in case of misuse) + int32_t sma : 8; // block SMA: 0, no SMA, 1, sum/min/max, 2, ... + int32_t offset : 24; // point offset in STpRow after the header part. } STColumn; +#pragma pack(pop) #define colType(col) ((col)->type) #define colColId(col) ((col)->colId) @@ -136,7 +139,7 @@ typedef struct { int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition @@ -590,7 +593,7 @@ void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); -static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) { +static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 136a576c7c..9c10137654 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -259,10 +259,10 @@ typedef struct { } SSubmitRsp; typedef struct SSchema { - int8_t type; - int32_t colId; - int32_t bytes; - char name[TSDB_COL_NAME_LEN]; + int8_t type; + col_id_t colId; + int32_t bytes; + char name[TSDB_COL_NAME_LEN]; } SSchema; typedef struct { @@ -438,8 +438,8 @@ typedef struct { */ typedef struct { union { - int16_t colId; - int16_t slotId; + col_id_t colId; + int16_t slotId; }; int16_t type; @@ -1902,7 +1902,7 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pSchema->type); tlen += taosEncodeFixedI32(buf, pSchema->bytes); - tlen += taosEncodeFixedI32(buf, pSchema->colId); + tlen += taosEncodeFixedI16(buf, pSchema->colId); tlen += taosEncodeString(buf, pSchema->name); return tlen; } @@ -1910,7 +1910,7 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) { buf = taosDecodeFixedI8(buf, &pSchema->type); buf = taosDecodeFixedI32(buf, &pSchema->bytes); - buf = taosDecodeFixedI32(buf, &pSchema->colId); + buf = taosDecodeFixedI16(buf, &pSchema->colId); buf = taosDecodeStringTo(buf, pSchema->name); return buf; } @@ -1918,7 +1918,7 @@ static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) { static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) { if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1; if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1; - if (tEncodeI32(pEncoder, pSchema->colId) < 0) return -1; + if (tEncodeI16(pEncoder, pSchema->colId) < 0) return -1; if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1; return 0; } @@ -1926,7 +1926,7 @@ static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSch static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) { if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1; if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1; - if (tDecodeI32(pDecoder, &pSchema->colId) < 0) return -1; + if (tDecodeI16(pDecoder, &pSchema->colId) < 0) return -1; if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1; return 0; } diff --git a/include/common/trow.h b/include/common/trow.h index fc99cbc5b2..df28bc9962 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -671,8 +671,9 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa * @param colIdx sorted column index, start from 0 * @return FORCE_INLINE */ -static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, int16_t colId, int8_t colType, TDRowValT valType, - const void *val, bool isCopyVarData, int32_t offset, int16_t colIdx) { +static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, + TDRowValT valType, const void *val, bool isCopyVarData, int32_t offset, + col_id_t colIdx) { STSRow *pRow = pBuilder->pBuf; if (!val) { #ifdef TD_SUPPORT_BITMAP diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 59af14c226..87dc752703 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -29,6 +29,7 @@ typedef uint32_t TDRowLenT; typedef uint8_t TDRowValT; typedef int16_t col_id_t; typedef int8_t col_type_t; +typedef int32_t col_bytes_t; #pragma pack(push, 1) typedef struct { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 616e24d67d..a78a080e73 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -56,7 +56,7 @@ typedef enum EColumnType { typedef struct SColumnNode { SExprNode node; // QUERY_NODE_COLUMN uint64_t tableId; - int16_t colId; + col_id_t colId; EColumnType colType; // column or tag char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 839feb7800..1676f0a267 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -43,10 +43,10 @@ typedef enum { } ETaskType; typedef struct STableComInfo { - uint8_t numOfTags; // the number of tags in schema - uint8_t precision; // the number of precision - int16_t numOfColumns; // the number of columns - int32_t rowSize; // row size of the schema + uint8_t numOfTags; // the number of tags in schema + uint8_t precision; // the number of precision + col_id_t numOfColumns; // the number of columns + int32_t rowSize; // row size of the schema } STableComInfo; typedef struct SIndexMeta { @@ -171,7 +171,7 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); char *jobTaskStatusStr(int32_t status); -SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); +SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 832a415efa..abb6e7fbd1 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -91,7 +91,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } else { tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - tscError("invalid colId[%d] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); + tscError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); tFreeSTableMetaBatchRsp(&batchMetaRsp); return TSDB_CODE_TSC_INVALID_VALUE; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 5b27fd01f4..1b7157c49c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -106,12 +106,12 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { if (tdInitTSchemaBuilder(&schemaBuilder, version) < 0) return NULL; for (int i = 0; i < numOfCols; i++) { - int8_t type = 0; - int16_t colId = 0; - int16_t bytes = 0; + col_type_t type = 0; + col_id_t colId = 0; + col_bytes_t bytes = 0; buf = taosDecodeFixedI8(buf, &type); buf = taosDecodeFixedI16(buf, &colId); - buf = taosDecodeFixedI16(buf, &bytes); + buf = taosDecodeFixedI32(buf, &bytes); if (tdAddColToSchema(&schemaBuilder, type, colId, bytes) < 0) { tdDestroyTSchemaBuilder(&schemaBuilder); return NULL; @@ -148,7 +148,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { pBuilder->version = version; } -int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes) { +int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes) { if (!isValidDataType(type)) return -1; if (pBuilder->nCols >= pBuilder->tCols) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d7df3e115f..d73897eb20 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -299,14 +299,14 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols); for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name); } tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols); for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].colId); + tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); } @@ -333,7 +333,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols); for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); } @@ -374,7 +374,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); - buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].colId)); + buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId)); buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes)); buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); } @@ -382,7 +382,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); - buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].colId); + buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); } @@ -422,7 +422,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); - buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].colId); + buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index acf45a31d9..c23d9fc5c9 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -98,7 +98,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } @@ -106,7 +106,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } @@ -114,7 +114,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { for (int32_t i = 0; i < pStb->numOfSmas; ++i) { SSchema *pSchema = &pStb->pSmas[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, STB_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) + SDB_SET_INT16(pRaw, dataPos, pSchema->colId, STB_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, STB_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER) } @@ -185,7 +185,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } @@ -193,7 +193,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfTags; ++i) { SSchema *pSchema = &pStb->pTags[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } @@ -201,7 +201,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStb->numOfSmas; ++i) { SSchema *pSchema = &pStb->pSmas[i]; SDB_GET_INT8(pRaw, dataPos, &pSchema->type, STB_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) + SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, STB_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, STB_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER) } diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index fdcc165207..b91c6cd9e3 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -233,7 +233,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { // save sma info int32_t len = tEncodeTSma(NULL, pSmaCfg); - pBuf = taosMemoryCalloc(len, 1); + pBuf = taosMemoryCalloc(1, len); if (pBuf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -285,7 +285,7 @@ static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; tlen += taosEncodeFixedI8(buf, pSchema->type); - tlen += taosEncodeFixedI32(buf, pSchema->colId); + tlen += taosEncodeFixedI16(buf, pSchema->colId); tlen += taosEncodeFixedI32(buf, pSchema->bytes); tlen += taosEncodeString(buf, pSchema->name); } @@ -301,7 +301,7 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; buf = taosDecodeFixedI8(buf, &pSchema->type); - buf = taosDecodeFixedI32(buf, &pSchema->colId); + buf = taosDecodeFixedI16(buf, &pSchema->colId); buf = taosDecodeFixedI32(buf, &pSchema->bytes); buf = taosDecodeStringTo(buf, pSchema->name); } @@ -516,6 +516,7 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { tsize += taosEncodeFixedU64(buf, pTbCfg->ctbCfg.suid); tsize += tdEncodeKVRow(buf, pTbCfg->ctbCfg.pTag); } else if (pTbCfg->type == META_NORMAL_TABLE) { + // TODO } else { ASSERT(0); } @@ -538,6 +539,7 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid)); buf = tdDecodeKVRow(buf, &(pTbCfg->ctbCfg.pTag)); } else if (pTbCfg->type == META_NORMAL_TABLE) { + // TODO } else { ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 37e7ed11ae..8c161e4c8b 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -130,8 +130,8 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int32_t colNeed = 0; while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - int16_t colIdSchema = pColSchema->colId; - int16_t colIdNeed = *(int16_t*)taosArrayGet(pHandle->pColIdList, colNeed); + col_id_t colIdSchema = pColSchema->colId; + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pHandle->pColIdList, colNeed); if (colIdSchema < colIdNeed) { colMeta++; } else if (colIdSchema > colIdNeed) { @@ -159,7 +159,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int j = 0; for (int32_t i = 0; i < colNumNeed; i++) { - int16_t colId = *(int16_t*)taosArrayGet(pHandle->pColIdList, i); + col_id_t colId = *(col_id_t*)taosArrayGet(pHandle->pColIdList, i); while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { j++; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index eca0a9612f..eb8df61051 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1369,7 +1369,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF } } - // Update pBlock membership vairables + // Update pBlock membership variables pBlock->last = isLast; pBlock->offset = offset; pBlock->algorithm = pCfg->compression; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 6eb721fcb1..34dba5d3ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -255,7 +255,7 @@ int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg) { return 0; } -int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { +static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; SSubmitMsgIter msgIter = {0}; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index e0be9ed89a..a0a34b14a4 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -74,6 +74,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: maybe need to clear the request struct taosMemoryFree(vCreateTbReq.stbCfg.pSchema); taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema); + taosMemoryFree(vCreateTbReq.stbCfg.pBSmaCols); + taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam); + taosMemoryFree(vCreateTbReq.dbFName); taosMemoryFree(vCreateTbReq.name); break; } @@ -102,13 +105,18 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name); } taosMemoryFree(pCreateTbReq->name); + taosMemoryFree(pCreateTbReq->dbFName); if (pCreateTbReq->type == TD_SUPER_TABLE) { taosMemoryFree(pCreateTbReq->stbCfg.pSchema); taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema); + taosMemoryFree(pCreateTbReq->stbCfg.pBSmaCols); + taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam); } else if (pCreateTbReq->type == TD_CHILD_TABLE) { taosMemoryFree(pCreateTbReq->ctbCfg.pTag); } else { taosMemoryFree(pCreateTbReq->ntbCfg.pSchema); + taosMemoryFree(pCreateTbReq->ntbCfg.pBSmaCols); + taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam); } } @@ -135,6 +143,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vAlterTbReq); taosMemoryFree(vAlterTbReq.stbCfg.pSchema); taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema); + taosMemoryFree(vAlterTbReq.stbCfg.pBSmaCols); + taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam); + taosMemoryFree(vAlterTbReq.dbFName); taosMemoryFree(vAlterTbReq.name); break; } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 7bbff05b4b..4f894e0aa9 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -172,7 +172,7 @@ void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { int32_t colNum = c->numOfColumns + c->numOfTags; for (int32_t i = 0; i < colNum; ++i) { SSchema *s = &p->schema[i]; - ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes); + ctgDebug("[%d] name:%s, type:%d, colId:%" PRIi16 ", bytes:%d", i, s->name, s->type, s->colId, s->bytes); } } diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index acd021572d..a38d64c58c 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -41,26 +41,26 @@ typedef struct SBoundColumn { } SBoundColumn; typedef struct { - uint16_t schemaColIdx; - uint16_t boundIdx; - uint16_t finalIdx; + col_id_t schemaColIdx; + col_id_t boundIdx; + col_id_t finalIdx; } SBoundIdxInfo; typedef struct SParsedDataColInfo { - int16_t numOfCols; - int16_t numOfBound; + col_id_t numOfCols; + col_id_t numOfBound; uint16_t flen; // TODO: get from STSchema uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) uint16_t extendedVarLen; uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - int32_t * boundedColumns; // bound column idx according to schema - SBoundColumn * cols; + col_id_t *boundColumns; // bound column idx according to schema + SBoundColumn *cols; SBoundIdxInfo *colIdxInfo; int8_t orderStatus; // bound columns } SParsedDataColInfo; typedef struct { - uint8_t memRowType; // default is 0, that is SDataRow + uint8_t rowType; // default is 0, that is SDataRow int32_t rowSize; } SMemRowBuilder; @@ -92,11 +92,11 @@ static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) { (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1); } -static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, - int32_t idx, int32_t *toffset, int32_t *colIdx) { - int32_t schemaIdx = 0; +static FORCE_INLINE void getSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, + int32_t *toffset, col_id_t *colIdx) { + col_id_t schemaIdx = 0; if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundedColumns[idx] - PRIMARYKEY_TIMESTAMP_COL_ID; + schemaIdx = spd->boundColumns[idx] - PRIMARYKEY_TIMESTAMP_COL_ID; if (TD_IS_TP_ROW_T(rowType)) { *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart *colIdx = schemaIdx; @@ -132,7 +132,7 @@ static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks* int32_t schemaIdxCompar(const void *lhs, const void *rhs); int32_t boundIdxCompar(const void *lhs, const void *rhs); -void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols); +void setBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); void destroyBoundColumnInfo(SParsedDataColInfo* pColList); void destroyBlockArrayList(SArray* pDataBlockList); void destroyBlockHashmap(SHashObj* pDataBlockHash); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index b2fc39d064..80a83af922 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -600,9 +600,9 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int typedef struct SMemParam { SRowBuilder* rb; - SSchema* schema; - int32_t toffset; - int32_t colIdx; + SSchema* schema; + int32_t toffset; + col_id_t colIdx; } SMemParam; static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* param) { @@ -623,9 +623,11 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); } else { if (value == NULL) { // it is a null data - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx); + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, + pa->colIdx); } else { - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, pa->colIdx); + tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, + pa->colIdx); } } return TSDB_CODE_SUCCESS; @@ -633,18 +635,18 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p // pSql -> tag1_name, ...) static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { - int32_t nCols = pColList->numOfCols; + int16_t nCols = pColList->numOfCols; pColList->numOfBound = 0; pColList->boundNullLen = 0; - memset(pColList->boundedColumns, 0, sizeof(int32_t) * nCols); + memset(pColList->boundColumns, 0, sizeof(int16_t) * nCols); for (int32_t i = 0; i < nCols; ++i) { pColList->cols[i].valStat = VAL_STAT_NONE; } SToken sToken; bool isOrdered = true; - int32_t lastColIdx = -1; // last column found + int16_t lastColIdx = -1; // last column found while (1) { NEXT_TOKEN(pCxt->pSql, sToken); @@ -652,8 +654,8 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* break; } - int32_t t = lastColIdx + 1; - int32_t index = findCol(&sToken, t, nCols, pSchema); + int16_t t = lastColIdx + 1; + int16_t index = findCol(&sToken, t, nCols, pSchema); if (index < 0 && t > 0) { index = findCol(&sToken, 0, t, pSchema); isOrdered = false; @@ -666,7 +668,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* } lastColIdx = index; pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundedColumns[pColList->numOfBound] = index + PRIMARYKEY_TIMESTAMP_COL_ID; + pColList->boundColumns[pColList->numOfBound] = index + PRIMARYKEY_TIMESTAMP_COL_ID; ++pColList->numOfBound; switch (pSchema[t].type) { case TSDB_DATA_TYPE_BINARY: @@ -689,18 +691,19 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* return TSDB_CODE_TSC_OUT_OF_MEMORY; } SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (uint16_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = (uint16_t)pColList->boundedColumns[i]; + for (int16_t i = 0; i < pColList->numOfBound; ++i) { + pColIdx[i].schemaColIdx = pColList->boundColumns[i]; pColIdx[i].boundIdx = i; } qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); - for (uint16_t i = 0; i < pColList->numOfBound; ++i) { + for (int16_t i = 0; i < pColList->numOfBound; ++i) { pColIdx[i].finalIdx = i; } qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } - memset(&pColList->boundedColumns[pColList->numOfBound], 0, sizeof(int32_t) * (pColList->numOfCols - pColList->numOfBound)); + memset(&pColList->boundColumns[pColList->numOfBound], 0, + sizeof(int16_t) * (pColList->numOfCols - pColList->numOfBound)); return TSDB_CODE_SUCCESS; } @@ -714,8 +717,8 @@ typedef struct SKvParam { static int32_t KvRowAppend(const void *value, int32_t len, void *param) { SKvParam* pa = (SKvParam*) param; - int32_t type = pa->schema->type; - int32_t colId = pa->schema->colId; + int8_t type = pa->schema->type; + int16_t colId = pa->schema->colId; if (TSDB_DATA_TYPE_BINARY == type) { STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len); @@ -747,7 +750,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema* pSchema = &pTagsSchema[pCxt->tags.boundedColumns[i]]; + SSchema* pSchema = &pTagsSchema[pCxt->tags.boundColumns[i]]; param.schema = pSchema; CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); } @@ -813,9 +816,9 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema *pSchema = &schema[spd->boundedColumns[i] - 1]; + SSchema* pSchema = &schema[spd->boundColumns[i] - 1]; param.schema = pSchema; - getMemRowAppendInfo(schema, pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); + getSTSRowAppendInfo(schema, pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, ¶m, &pCxt->msg)); if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index e516053b1e..f70e514b5a 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -43,11 +43,11 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { } } -void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols) { +void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_id_t numOfCols) { pColList->numOfCols = numOfCols; pColList->numOfBound = numOfCols; pColList->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColList->boundedColumns = taosMemoryCalloc(pColList->numOfCols, sizeof(int32_t)); + pColList->boundColumns = taosMemoryCalloc(pColList->numOfCols, sizeof(col_id_t)); pColList->cols = taosMemoryCalloc(pColList->numOfCols, sizeof(SBoundColumn)); pColList->colIdxInfo = NULL; pColList->flen = 0; @@ -73,7 +73,7 @@ void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t default: break; } - pColList->boundedColumns[i] = pSchema[i].colId; + pColList->boundColumns[i] = pSchema[i].colId; } pColList->allNullLen += pColList->flen; pColList->boundNullLen = pColList->allNullLen; // default set allNullLen @@ -103,7 +103,7 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs) { } void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { - taosMemoryFreeClear(pColList->boundedColumns); + taosMemoryFreeClear(pColList->boundColumns); taosMemoryFreeClear(pColList->cols); taosMemoryFreeClear(pColList->colIdxInfo); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2d1882245d..896fab4fdd 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1959,7 +1959,7 @@ typedef struct SVgroupTablesBatch { char dbName[TSDB_DB_NAME_LEN]; } SVgroupTablesBatch; -static void toSchema(const SColumnDefNode* pCol, int32_t colId, SSchema* pSchema) { +static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) { pSchema->colId = colId; pSchema->type = pCol->dataType.type; pSchema->bytes = pCol->dataType.bytes; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 012af26c17..402caeb252 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -87,7 +87,7 @@ private: return meta_; } - int32_t colId_; + col_id_t colId_; int32_t rowsize_; std::shared_ptr meta_; }; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 79a59a5ecb..b08379a186 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -193,7 +193,7 @@ char *jobTaskStatusStr(int32_t status) { return "UNKNOWN"; } -SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name) { +SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name) { SSchema s = {0}; s.type = type; s.bytes = bytes; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 20eb49ed33..1e91c55dc0 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -188,7 +188,7 @@ static int32_t queryConvertTableMetaMsg(STableMetaRsp *pMetaMsg) { } if (pMetaMsg->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - qError("invalid colId[%d] for the first column in table meta rsp msg", pMetaMsg->pSchemas[0].colId); + qError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", pMetaMsg->pSchemas[0].colId); return TSDB_CODE_TSC_INVALID_VALUE; } diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 56267c462b..ebdc3f287d 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -173,7 +173,7 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { return -1; } - int32_t cols = 0; + col_id_t cols = 0; SSchema *pSchema = showRsp.tableMeta.pSchemas; const SSchema *s = tGetTbnameColumnSchema(); From 583f01d4980a4b315697468a504cd07a76f793b7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 27 Mar 2022 12:24:26 +0800 Subject: [PATCH 113/121] refactor --- source/libs/parser/src/parInsert.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 80a83af922..398e5abaa6 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -635,18 +635,18 @@ static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* p // pSql -> tag1_name, ...) static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { - int16_t nCols = pColList->numOfCols; + col_id_t nCols = pColList->numOfCols; pColList->numOfBound = 0; pColList->boundNullLen = 0; - memset(pColList->boundColumns, 0, sizeof(int16_t) * nCols); + memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); for (int32_t i = 0; i < nCols; ++i) { pColList->cols[i].valStat = VAL_STAT_NONE; } SToken sToken; bool isOrdered = true; - int16_t lastColIdx = -1; // last column found + col_id_t lastColIdx = -1; // last column found while (1) { NEXT_TOKEN(pCxt->pSql, sToken); @@ -654,8 +654,8 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* break; } - int16_t t = lastColIdx + 1; - int16_t index = findCol(&sToken, t, nCols, pSchema); + col_id_t t = lastColIdx + 1; + col_id_t index = findCol(&sToken, t, nCols, pSchema); if (index < 0 && t > 0) { index = findCol(&sToken, 0, t, pSchema); isOrdered = false; From 4693f795261ee4bd4b63b4a68e7291d37c015525 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 27 Mar 2022 23:26:21 +0800 Subject: [PATCH 114/121] refactor --- include/common/tmsg.h | 2 +- include/util/tdef.h | 2 +- source/libs/parser/src/parInsert.c | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d447f11523..bdb6181884 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -208,11 +208,11 @@ typedef struct { typedef struct SSubmitBlk { int64_t uid; // table unique id int64_t suid; // stable id - int32_t padding; // TODO just for padding here int32_t sversion; // data schema version int32_t dataLen; // data part length, not including the SSubmitBlk head int32_t schemaLen; // schema length, if length is 0, no schema exists int16_t numOfRows; // total number of rows in current submit block + int16_t padding; // TODO just for padding here char data[]; } SSubmitBlk; diff --git a/include/util/tdef.h b/include/util/tdef.h index 655deb4625..6c5208ec00 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -209,7 +209,7 @@ typedef enum ELogicConditionType { #define TSDB_FUNC_TYPE_AGGREGATE 2 #define TSDB_FUNC_MAX_RETRIEVE 1024 -#define TSDB_INDEX_NAME_LEN 33 // 32 + 1 '\0' +#define TSDB_INDEX_NAME_LEN 65 // 64 + 1 '\0' #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 398e5abaa6..ed67de17e0 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -640,7 +640,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList->numOfBound = 0; pColList->boundNullLen = 0; memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); - for (int32_t i = 0; i < nCols; ++i) { + for (col_id_t i = 0; i < nCols; ++i) { pColList->cols[i].valStat = VAL_STAT_NONE; } @@ -691,19 +691,19 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* return TSDB_CODE_TSC_OUT_OF_MEMORY; } SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (int16_t i = 0; i < pColList->numOfBound; ++i) { + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { pColIdx[i].schemaColIdx = pColList->boundColumns[i]; pColIdx[i].boundIdx = i; } qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); - for (int16_t i = 0; i < pColList->numOfBound; ++i) { + for (col_id_t i = 0; i < pColList->numOfBound; ++i) { pColIdx[i].finalIdx = i; } qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(int16_t) * (pColList->numOfCols - pColList->numOfBound)); + sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); return TSDB_CODE_SUCCESS; } From ded8843367436ab2cb68e2a3658de77893e191b3 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 28 Mar 2022 07:35:57 +0800 Subject: [PATCH 115/121] SSDatablock refactor --- source/dnode/vnode/src/tsdb/tsdbSma.c | 102 +++++++++++++++++------- source/dnode/vnode/test/tsdbSmaTest.cpp | 2 +- 2 files changed, 75 insertions(+), 29 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 9d5e132772..07eafd6df0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -38,9 +38,10 @@ typedef enum { } ESmaStorageLevel; typedef struct { - STsdb *pTsdb; - SDBFile dFile; - int32_t interval; // interval with the precision of DB + STsdb *pTsdb; + SDBFile dFile; + SSDataBlock *pData; // sma data + int32_t interval; // interval with the precision of DB } STSmaWriteH; typedef struct { @@ -98,7 +99,8 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ int32_t nMaxResult); // insert data -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData); +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, SSDataBlock *pData, int64_t interval, + int8_t intervalUnit); static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); @@ -800,9 +802,10 @@ static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *p return TSDB_CODE_SUCCESS; } -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, STSmaDataWrapper *pData) { +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, SSDataBlock *pData, int64_t interval, int8_t intervalUnit) { pSmaH->pTsdb = pTsdb; - pSmaH->interval = tsdbGetIntervalByPrecision(pData->interval, pData->intervalUnit, REPO_CFG(pTsdb)->precision); + pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision); + pSmaH->pData = pData; return TSDB_CODE_SUCCESS; } @@ -857,10 +860,10 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe * @return int32_t */ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); - int64_t indexUid = SMA_TEST_INDEX_UID; + STsdbCfg *pCfg = REPO_CFG(pTsdb); + SSDataBlock *pData = (SSDataBlock *)msg; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); + int64_t indexUid = SMA_TEST_INDEX_UID; if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; @@ -868,15 +871,15 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return terrno; } - if (pData->dataLen <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; + if (pData == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert tSma data failed since pData is NULL", REPO_ID(pTsdb)); + return terrno; } - STSmaWriteH tSmaH = {0}; - - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) { + if (taosArrayGetSize(pData->pDataBlock) <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + tsdbWarn("vgId:%d insert tSma data failed since pDataBlock is empty", REPO_ID(pTsdb)); return TSDB_CODE_FAILED; } @@ -895,6 +898,14 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return TSDB_CODE_FAILED; } + STSma *pSma = pItem->pSma; + + STSmaWriteH tSmaH = {0}; + + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData, pSma->interval, pSma->intervalUnit) != 0) { + return TSDB_CODE_FAILED; + } + char rPath[TSDB_FILENAME_LEN] = {0}; char aPath[TSDB_FILENAME_LEN] = {0}; snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); @@ -907,8 +918,11 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { } // Step 1: Judge the storage level and days - int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); + int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); + + +#if 0 int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file @@ -933,7 +947,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { // Step 3: reset the SSmaStat tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); - +#endif tsdbDestroyTSmaWriteH(&tSmaH); tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_SUCCESS; @@ -999,29 +1013,58 @@ static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, } static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - STSmaDataWrapper *pData = (STSmaDataWrapper *)msg; - SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); + STsdbCfg *pCfg = REPO_CFG(pTsdb); + SSDataBlock *pData = (SSDataBlock *)msg; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); + int64_t indexUid = SMA_TEST_INDEX_UID; if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); + tsdbWarn("vgId:%d insert rSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); return terrno; } - if (pData->dataLen <= 0) { - TASSERT(0); + if (pEnv == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert rSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); + return terrno; + } + + if (pData == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbWarn("vgId:%d insert rSma data failed since pData is NULL", REPO_ID(pTsdb)); + return terrno; + } + + if (taosArrayGetSize(pData->pDataBlock) <= 0) { terrno = TSDB_CODE_INVALID_PARA; + tsdbWarn("vgId:%d insert rSma data failed since pDataBlock is empty", REPO_ID(pTsdb)); return TSDB_CODE_FAILED; } + SSmaStat *pStat = SMA_ENV_STAT(pTsdb->pTSmaEnv); + SSmaStatItem *pItem = NULL; + + tsdbRefSmaStat(pTsdb, pStat); + + if (pStat && pStat->smaStatItems) { + pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); + } + + if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL) || tsdbSmaStatIsDropped(pItem)) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } + + STSma *pSma = pItem->pSma; + STSmaWriteH tSmaH = {0}; - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) { + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData, pSma->interval, pSma->intervalUnit) != 0) { return TSDB_CODE_FAILED; } - int64_t indexUid = SMA_TEST_INDEX_UID; char rPath[TSDB_FILENAME_LEN] = {0}; char aPath[TSDB_FILENAME_LEN] = {0}; snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); @@ -1033,8 +1076,9 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { } // Step 1: Judge the storage level and days - int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit); + int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); + #if 0 int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file @@ -1057,8 +1101,10 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { // Step 3: reset the SSmaStat tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); +#endif tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 86db3af4dc..29a4b7f552 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -280,7 +280,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { } #endif -#if 1 +#if 0 TEST(testCase, tSma_Data_Insert_Query_Test) { // step 1: prepare meta const char *smaIndexName1 = "sma_index_test_1"; From 53e5bfb820af07bfa9b4dc90716b443a269927db Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Mar 2022 10:09:00 +0800 Subject: [PATCH 116/121] minor changes --- source/dnode/mnode/impl/src/mndUser.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index ff34c26c4a..7d3f755cd7 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -21,7 +21,7 @@ #include "mndTrans.h" #include "tbase64.h" -#define TSDB_USER_VER_NUMBER 1 +#define TSDB_USER_VER_NUMBER 1 #define TSDB_USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); @@ -270,7 +270,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate userObj.updateTime = userObj.createdTime; userObj.superUser = pCreate->superUser; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_CREATE_USER, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to create since %s", pCreate->user, terrstr()); return -1; @@ -350,7 +350,7 @@ CREATE_USER_OVER: } static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNodeMsg *pReq) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER,&pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to update since %s", pOld->user, terrstr()); return -1; @@ -511,7 +511,7 @@ ALTER_USER_OVER: } static int32_t mndDropUser(SMnode *pMnode, SNodeMsg *pReq, SUserObj *pUser) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_DROP_USER, &pReq->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_USER, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); return -1; From 4b5c05334fb2b336616be1ce7441c2fc5602d4bf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Mar 2022 10:10:06 +0800 Subject: [PATCH 117/121] minor changes --- source/util/src/tprocess.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index f170be0844..f5ce88179b 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -152,7 +152,7 @@ static void taosProcDestroyBuffer(void *pBuffer, int32_t shmid) { } } -static SProcQueue *taosProcQueueInit(int32_t size) { +static SProcQueue *taosProcInitQueue(int32_t size) { if (size <= 0) size = SHM_DEFAULT_SIZE; int32_t bufSize = CEIL8(size); @@ -353,10 +353,10 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { } pProc->name = pCfg->name; - pProc->pChildQueue = taosProcQueueInit(pCfg->childQueueSize); - pProc->pParentQueue = taosProcQueueInit(pCfg->parentQueueSize); + pProc->pChildQueue = taosProcInitQueue(pCfg->childQueueSize); + pProc->pParentQueue = taosProcInitQueue(pCfg->parentQueueSize); if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { - taosProcQueueCleanup(pProc->pChildQueue); + taosProcCleanupQueue(pProc->pChildQueue); taosMemoryFree(pProc); return NULL; } @@ -450,8 +450,8 @@ void taosProcCleanup(SProcObj *pProc) { if (pProc != NULL) { uDebug("proc:%s, clean up", pProc->name); taosProcStop(pProc); - taosProcQueueCleanup(pProc->pChildQueue); - taosProcQueueCleanup(pProc->pParentQueue); + taosProcCleanupQueue(pProc->pChildQueue); + taosProcCleanupQueue(pProc->pParentQueue); taosMemoryFree(pProc); } } From d882edd037f151357270b6486df0b8946cb23da9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 28 Mar 2022 10:13:24 +0800 Subject: [PATCH 118/121] minor changes --- source/dnode/mgmt/container/src/dndExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/container/src/dndExec.c b/source/dnode/mgmt/container/src/dndExec.c index 24c7ad7968..a7b8ca288b 100644 --- a/source/dnode/mgmt/container/src/dndExec.c +++ b/source/dnode/mgmt/container/src/dndExec.c @@ -139,7 +139,7 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRsp, int32_t msgLen, void *pCont, int32_t contLen) { dTrace("msg:%p, get from parent queue", pRsp); pRsp->pCont = pCont; - dndSendRpcRsp(pWrapper, pRsp); + dndSendRsp(pWrapper, pRsp); taosMemoryFree(pRsp); } From e0f192046c634431798a4989fef45551606efe55 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 28 Mar 2022 11:47:39 +0800 Subject: [PATCH 119/121] multiple level stream schedule --- example/src/tstream.c | 2 +- include/client/taos.h | 4 +-- include/libs/stream/tstream.h | 3 +- source/client/src/tmq.c | 25 +++++++------- source/dnode/mgmt/vnode/src/vmWorker.c | 20 +++++++++++- source/dnode/mnode/impl/src/mndScheduler.c | 1 + source/dnode/vnode/src/tq/tq.c | 16 +++++---- source/dnode/vnode/src/vnd/vnodeQuery.c | 2 ++ source/libs/stream/src/tstream.c | 8 +++-- source/libs/wal/src/walMeta.c | 10 +++--- source/libs/wal/src/walSeek.c | 5 ++- tests/test/c/tmqDemo.c | 38 +++------------------- 12 files changed, 67 insertions(+), 67 deletions(-) diff --git a/example/src/tstream.c b/example/src/tstream.c index 51578bd27b..8ffa932bd2 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -25,7 +25,7 @@ int32_t init_env() { return -1; } - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); if (taos_errno(pRes) != 0) { printf("error in create db, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/client/taos.h b/include/client/taos.h index 55e1d1c422..dc54b89d04 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -217,7 +217,6 @@ DLL_EXPORT void tmq_list_destroy(tmq_list_t *); DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen); -DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message); DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); /* ------------------------TMQ CONSUMER INTERFACE------------------------ */ @@ -258,7 +257,8 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message); DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message); -DLL_EXPORT char *tmq_get_topic_schema(tmq_t *tmq, const char *topic); +DLL_EXPORT void *tmq_get_topic_schema(tmq_t *tmq, const char *topic); +DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message); /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */ DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 177fe39397..4efddde935 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -62,10 +62,11 @@ typedef struct { } STaskExec; typedef struct { - int8_t reserved; + int32_t taskId; } STaskDispatcherInplace; typedef struct { + int32_t taskId; int32_t nodeId; SEpSet epSet; } STaskDispatcherFixedEp; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 2a9b3cdf64..8eaca6853d 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -186,23 +186,23 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } - if (strcmp(key, "connection.ip") == 0) { + if (strcmp(key, "td.connect.ip") == 0) { conf->ip = strdup(value); return TMQ_CONF_OK; } - if (strcmp(key, "connection.user") == 0) { + if (strcmp(key, "td.connect.user") == 0) { conf->user = strdup(value); return TMQ_CONF_OK; } - if (strcmp(key, "connection.pass") == 0) { + if (strcmp(key, "td.connect.pass") == 0) { conf->pass = strdup(value); return TMQ_CONF_OK; } - if (strcmp(key, "connection.port") == 0) { + if (strcmp(key, "td.connect.port") == 0) { conf->port = atoi(value); return TMQ_CONF_OK; } - if (strcmp(key, "connection.db") == 0) { + if (strcmp(key, "td.connect.db") == 0) { conf->db = strdup(value); return TMQ_CONF_OK; } @@ -223,13 +223,13 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { } void tmq_list_destroy(tmq_list_t* list) { - SArray* container = (SArray*)list; + SArray* container = &list->container; /*taosArrayDestroy(container);*/ taosArrayDestroyEx(container, (void (*)(void*))taosMemoryFree); } void tmqClearUnhandleMsg(tmq_t* tmq) { - tmq_message_t* msg; + tmq_message_t* msg = NULL; while (1) { taosGetQitem(tmq->qall, (void**)&msg); if (msg) @@ -807,7 +807,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqClientVg* pVg = pParam->pVg; tmq_t* tmq = pParam->tmq; if (code != 0) { - printf("msg discard %x\n", code); + printf("msg discard, code:%x\n", code); goto WRITE_QUEUE_FAIL; } @@ -877,10 +877,10 @@ WRITE_QUEUE_FAIL: } bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { + printf("call update ep %d\n", epoch); bool set = false; int32_t sz = taosArrayGetSize(pRsp->topics); - if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); - tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); + SArray* newTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); for (int32_t i = 0; i < sz; i++) { SMqClientTopic topic = {0}; SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); @@ -899,8 +899,10 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { taosArrayPush(topic.vgs, &clientVg); set = true; } - taosArrayPush(tmq->clientTopics, &topic); + taosArrayPush(newTopics, &topic); } + if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); + tmq->clientTopics = newTopics; atomic_store_32(&tmq->epoch, epoch); return set; } @@ -1219,6 +1221,7 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese if (rspMsg->msg.head.epoch == atomic_load_32(&tmq->epoch)) { /*printf("epoch match\n");*/ SMqClientVg* pVg = rspMsg->vg; + /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ pVg->currentOffset = rspMsg->msg.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); return rspMsg; diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c index e97d6e7f11..7b6d78a60c 100644 --- a/source/dnode/mgmt/vnode/src/vmWorker.c +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -160,6 +160,24 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf } } +static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { + SVnodeObj *pVnode = pInfo->ahandle; + SNodeMsg *pMsg = NULL; + + for (int32_t i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pMsg); + + dTrace("msg:%p, will be processed in vnode-merge queue", pMsg); + int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg); + if (code != 0) { + vmSendRsp(pVnode->pWrapper, pMsg, code); + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); + } + } +} + static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) { SRpcMsg *pRpc = &pMsg->rpcMsg; int32_t code = -1; @@ -308,7 +326,7 @@ int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue); pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue); - pVnode->pMergeQ = tWWorkerAllocQueue(&pMgmt->mergePool, pVnode, (FItems)vmProcessMergeMsg); + pVnode->pMergeQ = tWWorkerAllocQueue(&pMgmt->mergePool, pVnode, (FItems)vmProcessMergeQueue); pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index a4dfd293de..69ee1a5696 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -185,6 +185,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { pTask->dispatchMsgType = TDMT_VND_TASK_MERGE_EXEC; pTask->dispatchType = TASK_DISPATCH__FIXED; + pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId; pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 91cbc2cff8..4661668cbe 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -42,8 +42,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, STq // TODO: error code of buffer pool } #endif - pTq->tqMeta = - tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, (FTqDelete)taosMemoryFree, 0); + pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, + (FTqDelete)taosMemoryFree, 0); if (pTq->tqMeta == NULL) { taosMemoryFree(pTq); #if 0 @@ -498,12 +498,16 @@ int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen) { } int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg) { - SStreamTaskExecReq* pReq = msg->pCont; - int32_t taskId = pReq->taskId; - SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + char* msgstr = POINTER_SHIFT(msg->pCont, sizeof(SMsgHead)); + + SStreamTaskExecReq req; + tDecodeSStreamTaskExecReq(msgstr, &req); + + int32_t taskId = req.taskId; + SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); ASSERT(pTask); - if (streamExecTask(pTask, &pTq->pVnode->msgCb, pReq->data, STREAM_DATA_TYPE_SSDATA_BLOCK, 0) < 0) { + if (streamExecTask(pTask, &pTq->pVnode->msgCb, req.data, STREAM_DATA_TYPE_SSDATA_BLOCK, 0) < 0) { // TODO } return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 1db17f37cb..94e183f525 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -66,6 +66,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { case TDMT_VND_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_VND_TASK_EXEC: + case TDMT_VND_TASK_PIPE_EXEC: + case TDMT_VND_TASK_MERGE_EXEC: return tqProcessTaskExec(pVnode->pTq, pMsg); case TDMT_VND_STREAM_TRIGGER: return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen); diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index 3ec6603673..31a06a9d9a 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -121,7 +121,7 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { SStreamTaskExecReq req = { .streamId = pTask->streamId, - .taskId = pTask->taskId, + .taskId = pTask->fixedEpDispatcher.taskId, .data = pRes, }; @@ -211,8 +211,9 @@ int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask) { } if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { - if (tEncodeI8(pEncoder, pTask->inplaceDispatcher.reserved) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->inplaceDispatcher.taskId) < 0) return -1; } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { @@ -248,8 +249,9 @@ int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask) { } if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { - if (tDecodeI8(pDecoder, &pTask->inplaceDispatcher.reserved) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->inplaceDispatcher.taskId) < 0) return -1; } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 83c48628a3..36323cdffa 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -19,13 +19,13 @@ #include "tref.h" #include "walInt.h" -int64_t inline walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } +int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } -int64_t inline walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; } +int64_t FORCE_INLINE walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; } -int64_t inline walGetLastVer(SWal* pWal) { return pWal->vers.lastVer; } +int64_t FORCE_INLINE walGetLastVer(SWal* pWal) { return pWal->vers.lastVer; } -static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { +static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); } @@ -46,7 +46,7 @@ void* tmemmem(char* haystack, int hlen, char* needle, int nlen) { return NULL; } -static inline int64_t walScanLogGetLastVer(SWal* pWal) { +static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { ASSERT(pWal->fileInfoSet != NULL); int sz = taosArrayGetSize(pWal->fileInfoSet); ASSERT(sz > 0); diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 140b7ddc32..413dcb47f0 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -74,9 +74,9 @@ int walSetWrite(SWal* pWal) { } int walChangeWrite(SWal* pWal, int64_t ver) { - int code = 0; + int code; TdFilePtr pIdxTFile, pLogTFile; - char fnameStr[WAL_FILE_LEN]; + char fnameStr[WAL_FILE_LEN]; if (pWal->pWriteLogTFile != NULL) { code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { @@ -133,7 +133,6 @@ int walSeekWriteVer(SWal* pWal, int64_t ver) { return -1; } if (ver < pWal->vers.snapshotVer) { - } if (ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { code = walChangeWrite(pWal, ver); diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 1690a5fb3e..fd362e9705 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -314,7 +314,7 @@ int32_t init_env() { } //const char* sql = "select * from tu1"; - sprintf(sqlStr, "create topic test_stb_topic_1 as select * from %s0", g_stConfInfo.stbName); + sprintf(sqlStr, "create topic test_stb_topic_1 as select ts,c0 from %s", g_stConfInfo.stbName); /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sqlStr, strlen(sqlStr));*/ pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { @@ -351,36 +351,6 @@ tmq_list_t* build_topic_list() { return topic_list; } -void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - tmq_resp_err_t err; - - if ((err = tmq_subscribe(tmq, topics))) { - fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); - printf("subscribe err\n"); - return; - } - int32_t cnt = 0; - /*clock_t startTime = clock();*/ - while (running) { - tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1); - if (tmqmessage) { - cnt++; - msg_process(tmqmessage); - tmq_message_destroy(tmqmessage); - /*} else {*/ - /*break;*/ - } - } - /*clock_t endTime = clock();*/ - /*printf("log cnt: %d %f s\n", cnt, (double)(endTime - startTime) / CLOCKS_PER_SEC);*/ - - err = tmq_consumer_close(tmq); - if (err) - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); - else - fprintf(stderr, "%% Consumer closed\n"); -} - void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { static const int MIN_COMMIT_COUNT = 1000; @@ -438,7 +408,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog if (batchCnt != totalMsgs) { printf("%s inserted msgs: %d and consume msgs: %d mismatch %s", GREEN, totalMsgs, batchCnt, NC); - exit(-1); + /*exit(-1);*/ } if (0 == g_stConfInfo.simCase) { @@ -693,8 +663,8 @@ int main(int32_t argc, char *argv[]) { walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); if (walLogSize <= 0) { - printf("vnode2/wal size incorrect!"); - /*exit(-1);*/ + printf("vnode2/wal size incorrect!\n"); + /*exit(-1);*/ } else { if (0 == g_stConfInfo.simCase) { pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); From a942717855cb75931b8617f533ab2a703e6ff326 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 28 Mar 2022 11:56:01 +0800 Subject: [PATCH 120/121] [del walSize check] --- tests/test/c/tmqDemo.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 1690a5fb3e..63a3962d0e 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -691,12 +691,12 @@ int main(int32_t argc, char *argv[]) { float rowsSpeed = totalRows / seconds; float msgsSpeed = totalMsgs / seconds; - walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); - if (walLogSize <= 0) { - printf("vnode2/wal size incorrect!"); - /*exit(-1);*/ - } else { - if (0 == g_stConfInfo.simCase) { + if (0 == g_stConfInfo.simCase) { + walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); + if (walLogSize <= 0) { + printf("%s size incorrect!", g_stConfInfo.vnodeWalPath); + exit(-1); + } else { pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize/(1024 * 1024.0)); } } From 47fe8bd902c5547bf6d041fc28aa71fdedb983ae Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 28 Mar 2022 13:32:13 +0800 Subject: [PATCH 121/121] [modify] --- tests/script/tsim/tmq/basic.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tsim/tmq/basic.sim b/tests/script/tsim/tmq/basic.sim index 876cf7e266..1eeec46d53 100644 --- a/tests/script/tsim/tmq/basic.sim +++ b/tests/script/tsim/tmq/basic.sim @@ -45,7 +45,7 @@ print cmd===> system_content ../../debug/tests/test/c/tmq_demo -sim 1 -b 100 -c system_content ../../debug/tests/test/c/tmq_demo -sim 1 -b 100 -c ../../sim/tsim/cfg -w ../../sim/dnode1/data/vnode/vnode4/wal print cmd result----> $system_content if $system_content != @{consume success: 100}@ then - print not match in pos000 + return -1 endi sql show databases